diff --git a/.clang-tidy b/.clang-tidy index 8ed34ebb0f..39e6d96b87 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -71,6 +71,19 @@ # -modernize-type-traits # -performance-avoid-endl # +# TODO Code Quality WORKAROUND ROCm 6.2 +# -readability-container-size-empty +# -bugprone-inc-dec-in-conditions +# -readability-avoid-nested-conditional-operator +# -performance-enum-size +# -readability-redundant-member-init +# -bugprone-multi-level-implicit-pointer-conversion +# -clang-analyzer-optin.core.EnumCastOutOfRange +# -cppcoreguidelines-macro-to-enum +# -cppcoreguidelines-owning-memory +# -readability-avoid-return-with-void-value +# -bugprone-unused-local-non-trivial-variable +# Checks: >- *, -abseil-*, @@ -78,9 +91,12 @@ Checks: >- -android-cloexec-fopen, -bugprone-easily-swappable-parameters, -bugprone-exception-escape, + -bugprone-inc-dec-in-conditions, -bugprone-lambda-function-name, -bugprone-macro-parentheses, + -bugprone-multi-level-implicit-pointer-conversion, -bugprone-narrowing-conversions, + -bugprone-unused-local-non-trivial-variable, -bugprone-use-after-move, -cert-env33-c, -cert-err33-c, @@ -90,6 +106,7 @@ Checks: >- -cert-msc51-cpp, -clang-analyzer-alpha.core.CastToStruct, -clang-analyzer-cplusplus.NewDeleteLeaks, + -clang-analyzer-optin.core.EnumCastOutOfRange, -clang-analyzer-optin.performance.Padding, -clang-diagnostic-extern-c-compat, -clang-diagnostic-unused-command-line-argument, @@ -100,11 +117,13 @@ Checks: >- -cppcoreguidelines-avoid-magic-numbers, -cppcoreguidelines-explicit-virtual-functions, -cppcoreguidelines-init-variables, + -cppcoreguidelines-macro-to-enum, -cppcoreguidelines-macro-usage, -cppcoreguidelines-misleading-capture-default-by-value, -cppcoreguidelines-missing-std-forward, -cppcoreguidelines-narrowing-conversions, -cppcoreguidelines-non-private-member-variables-in-classes, + -cppcoreguidelines-owning-memory, -cppcoreguidelines-prefer-member-initializer, -cppcoreguidelines-pro-bounds-array-to-pointer-decay, -cppcoreguidelines-pro-bounds-constant-array-index, @@ -168,11 +187,15 @@ Checks: >- -modernize-concat-nested-namespaces, -modernize-unary-static-assert, -performance-avoid-endl, + -performance-enum-size, + -readability-avoid-nested-conditional-operator, -performance-no-automatic-move, -performance-unnecessary-copy-initialization, -performance-unnecessary-value-param, + -readability-avoid-return-with-void-value, -readability-avoid-unconditional-preprocessor-if, -readability-container-data-pointer, + -readability-container-size-empty, -readability-convert-member-functions-to-static, -readability-else-after-return, -readability-function-cognitive-complexity, @@ -182,6 +205,7 @@ Checks: >- -readability-named-parameter, -readability-qualified-auto, -readability-redundant-declaration, + -readability-redundant-member-init, -readability-redundant-string-init, -readability-simplify-boolean-expr, -readability-suspicious-call-argument, diff --git a/.gitattributes b/.gitattributes index 8e66d7251e..475660a7c4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,5 @@ *.db.bz2 binary +*.db.txt.bz2 binary *.fdb.txt.bz2 binary *.kdb.bz2 filter=lfs diff=lfs merge=lfs -text *.ktn.model binary diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1b79ec9b8c..eb8c43eb70 100755 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,4 @@ -* @JehandadKhan @junliume +* @BrianHarrisonAMD @junliume @BradPepersAMD # Documentation files docs/ @ROCm/rocm-documentation *.md @ROCm/rocm-documentation diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..380c163c3f --- /dev/null +++ b/.gitignore @@ -0,0 +1,67 @@ +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch +*.ipch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# vim tags +tags +.tags +.*.swp + +# Editors +.vscode + +# build-in-source directory + +# emacs temporary/backup files +.\#* +\#*\# +*~ + +# GDB temporary files +.gdb_history +install.dir* + +# documentation artifacts +_build/ +_images/ +_static/ +_templates/ +_toc.yml +_doxygen/ + +# JetBrains IDE +.idea/ +cmake-build*/ +build*/ + +# Python virtualenv +.venv/ + +# Python cache +__pycache__/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 40f7136c24..06d18bd816 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,24 @@ # Change Log for MIOpen Full documentation for MIOpen is available [here](https://rocm.docs.amd.com/projects/MIOpen/en/latest/) +## MIOpen 3.3.0 for ROCm 6.3.0 +### Added + +* [RNN] LSTM fwd +* [Mha] Mask is added for Forward pass +* [GLU] Gated Linear Unit (this is an experimental feature) +* [PReLU] Implemented PReLU bwd (this is an experimental feature) +### Optimized + +- MI300 TunaNet Update: CK FWD and WRW Solvers Updated +### Resolved issues -## MIOpen-3.2.0 for ROCm 6.2.0 +- Fixed unset stream when calling `hipMemsetAsync` +- Fixed a memory leak issue caused by an incorrect transpose in find 2.0 (see PR #3285 on GitHub) +- Fixed a `memcopy` data race by replacing `hipMemcpy` with `hipMemcpyWithStream` + + +## MIOpen 3.2.0 for ROCm 6.2.0 ### Added - [Conv] bilinear (alpha beta) solvers - [Conv] enable bf16 for ck-based solvers @@ -12,7 +28,6 @@ Full documentation for MIOpen is available [here](https://rocm.docs.amd.com/proj - [RNN] multi-stream as default solution. - TunaNetv2.0 for MI300 - Added adam and amp adam optimizer - ### Fixed - Memory access fault caused by GemmBwdRest - Context configuration in GetWorkSpaceSize @@ -21,7 +36,7 @@ Full documentation for MIOpen is available [here](https://rocm.docs.amd.com/proj ### Performance - Find: Improve precision of benchmarking -## MIOpen-3.1.0 for ROCm 6.1.0 +## MIOpen 3.1.0 for ROCm 6.1.0 ### Added - CK-based 2d/3d convolution solvers to support nchw/ncdhw layout - Fused solver for Fwd Convolution with Residual, Bias and activation @@ -40,7 +55,7 @@ Full documentation for MIOpen is available [here](https://rocm.docs.amd.com/proj ### Removed - clamping to MAX from CastTensor used in Bwd and WrW convolution -## MIOpen-3.0.0 for ROCm 6.0.0 +## MIOpen 3.0.0 for ROCm 6.0.0 - This release adds 3D convolution, enablement of fp8 convolution, NHWC batch norm, RNN padding support. It also removes INI8x4 support and fix minor issues and bugs. ### Notes @@ -63,7 +78,7 @@ INI8x4 support and fix minor issues and bugs. - Remove INT8x4 support - Remove target ids from kdb args -## MIOpen-2.21.0 for ROCm 5.7.0 +## MIOpen 2.21.0 for ROCm 5.7.0 ### Added - AI Heuristic for Immediate Mode Fallback - CK group forward convolution integration @@ -73,7 +88,7 @@ INI8x4 support and fix minor issues and bugs. ### Changed - Update fdb data to use solver keys [MI100][MI200] -## MIOpen-2.20.0 for ROCm 5.6.0 +## MIOpen 2.20.0 for ROCm 5.6.0 ### Added - AI Based Heuristic for Kernel Parameter Prediction - LSTM multi-stream solver @@ -83,7 +98,7 @@ INI8x4 support and fix minor issues and bugs. - Winograd kernel failure - Perf DB updates for gfx908 and gfx90a -## MIOpen-2.19.0 for ROCm 5.5.0 +## MIOpen 2.19.0 for ROCm 5.5.0 ### Added - ROCm 5.5 support for gfx1101 (Navi32) ### Changed diff --git a/CMakeLists.txt b/CMakeLists.txt index c78430e195..99c205eba8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -116,8 +116,11 @@ set(MIOPEN_ENABLE_SQLITE_BACKOFF On CACHE BOOL "") option( BUILD_DEV "Build for development only" OFF) option(MIOPEN_ENABLE_FIN "Enable the fin driver for MIOpen" OFF) +option(MIOPEN_ENABLE_FIN_INTERFACE "Enable internal interface for the fin" ${MIOPEN_ENABLE_FIN}) option(MIOPEN_STRIP_SYMBOLS "Strip symbols in release mode" ON) +option(MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM "Workaround: Use boost::filesystem instead of std::filesystem" OFF) +message(STATUS "MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM ${MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM}") # Strip symbols for release if(MIOPEN_STRIP_SYMBOLS AND NOT WIN32 AND NOT APPLE) @@ -125,7 +128,7 @@ if(MIOPEN_STRIP_SYMBOLS AND NOT WIN32 AND NOT APPLE) set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -s") endif() -rocm_setup_version(VERSION 3.2.0) +rocm_setup_version(VERSION 3.3.0) list( APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ) include(TargetFlags) @@ -147,9 +150,9 @@ else() set(MIOPEN_DEFAULT_BACKEND "OpenCL") endif() -if(NOT WIN32) +if(NOT WIN32 AND NOT MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM) include(CheckCXXLinkerFlag) - check_cxx_linker_flag(stdc++fs HAS_LIB_STD_FILESYSTEM) + check_cxx_linker_flag(-lstdc++fs HAS_LIB_STD_FILESYSTEM) endif() list(APPEND CMAKE_PREFIX_PATH ${CMAKE_INSTALL_PREFIX} ${CMAKE_INSTALL_PREFIX}/llvm ${CMAKE_INSTALL_PREFIX}/hip /opt/rocm /opt/rocm/llvm /opt/rocm/hip) @@ -227,10 +230,14 @@ if( MIOPEN_BACKEND STREQUAL "OpenCL") set (HIP_CXX_COMPILER ${MIOPEN_HIP_COMPILER}) endif() +# HIP SDK on Windows does not detect platform correctly, defaulting to "amd" +if(NOT HIP_PLATFORM AND CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(HIP_PLATFORM "amd") +endif() # HIP is always required find_package(hip REQUIRED PATHS /opt/rocm) -message(STATUS "Build with HIP ${hip_VERSION}") +message(STATUS "Build with HIP ${hip_VERSION} ${hip_DIR}") # Override HIP version in config.h, if necessary. # The variables set by find_package() can't be overwritten, @@ -321,7 +328,7 @@ add_compile_definitions($<$:HIP_COMPILER_FLAGS=${HIP_COMPI # HIP if( MIOPEN_BACKEND STREQUAL "HIP" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_BACKEND STREQUAL "HIPNOGPU") if(MIOPEN_USE_COMPOSABLEKERNEL) - find_package(composable_kernel 1.0.0 COMPONENTS device_other_operations device_gemm_operations device_conv_operations device_reduction_operations) + find_package(composable_kernel 1.0.0 COMPONENTS device_other_operations device_gemm_operations device_conv_operations device_reduction_operations device_mha_operations) endif() if( MIOPEN_BACKEND STREQUAL "HIPNOGPU") set(MIOPEN_MODE_NOGPU 1) @@ -351,7 +358,7 @@ if( MIOPEN_BACKEND STREQUAL "HIP" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN_B set(MIOPEN_USE_ROCBLAS ON CACHE BOOL "") if(MIOPEN_USE_ROCBLAS) find_package(rocblas REQUIRED PATHS /opt/rocm) - message(STATUS "Build with rocblas ${rocblas_VERSION}") + message(STATUS "Build with rocblas ${rocblas_VERSION} ${rocblas_DIR}") else() message(STATUS "Build without rocblas") endif() @@ -420,7 +427,7 @@ if(MIOPEN_USE_MLIR) endif() endif() endif() - message(STATUS "Build with rocMLIR::rockCompiler ${rocMLIR_VERSION}") + message(STATUS "Build with rocMLIR::rockCompiler ${rocMLIR_VERSION} ${rocMLIR_DIR}") endif() # Update HIP Runtime Package Dependency @@ -449,7 +456,7 @@ message(STATUS "AMDGCN assembler: ${MIOPEN_AMDGCN_ASSEMBLER}") if(MIOPEN_USE_COMGR) find_package(amd_comgr REQUIRED CONFIG) - message(STATUS "Build with comgr ${amd_comgr_VERSION}") + message(STATUS "Build with amd_comgr ${amd_comgr_VERSION} ${amd_comgr_DIR}") set(MIOPEN_PACKAGE_REQS "${MIOPEN_PACKAGE_REQS}, comgr") endif() @@ -458,7 +465,7 @@ if(MIOPEN_USE_HIPRTC) message(FATAL_ERROR "HIPRTC can be used only together with COMGR") endif() find_package(hiprtc REQUIRED) - message(STATUS "Build with HIPRTC ${hiprtc_VERSION}") + message(STATUS "Build with hiprtc ${hiprtc_VERSION} ${hiprtc_DIR}") endif() option(Boost_USE_STATIC_LIBS "Use boost static libraries" ON) @@ -467,8 +474,10 @@ if(MIOPEN_BUILD_DRIVER) # boost core is a header only component that can't be found by find_package # list(APPEND BOOST_COMPONENTS core) endif() + +# The FindBoost module has been removed since CMake 3.30. Use the Boost 1.83 configuration file instead. add_definitions(-DBOOST_ALL_NO_LIB=1) -find_package(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS}) +find_package(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS} CONFIG) find_path(HALF_INCLUDE_DIR half/half.hpp) message(STATUS "HALF_INCLUDE_DIR: ${HALF_INCLUDE_DIR}") @@ -489,7 +498,9 @@ endif() if(MIOPEN_ENABLE_AI_KERNEL_TUNING OR MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK) find_package(frugally-deep CONFIG REQUIRED) + message(STATUS "Build with frugally-deep ${frugally-deep_VERSION} ${frugally-deep_DIR}") find_package(Eigen3 REQUIRED) + message(STATUS "Build with Eigen3 ${Eigen3_VERSION} ${Eigen3_DIR}") endif() if(WIN32) @@ -551,7 +562,7 @@ endif() if(MIOPEN_BUILD_DRIVER) # PR #2785 MIOpenDriver to use rocrand to init buffers find_package(rocrand REQUIRED) - message(STATUS "rocrand_VERSION=${rocrand_VERSION}") + message(STATUS "Build with rocrand ${rocrand_VERSION} ${rocrand_DIR}") set(MIOPEN_PACKAGE_REQS "${MIOPEN_PACKAGE_REQS}, rocrand") endif() @@ -574,7 +585,7 @@ endif() set(KERNELS_SOURCE_DIR ${PROJECT_SOURCE_DIR}/src/kernels) -find_program(UNZIPPER lbzcat bzcat REQUIRED) +find_program(UNZIPPER bzip2 REQUIRED) file(MAKE_DIRECTORY ${KERNELS_BINARY_DIR}) add_custom_target(generate_kernels ALL) @@ -589,7 +600,7 @@ endif() function(unpack_db db_bzip2_file) get_filename_component(__fname ${db_bzip2_file} NAME_WLE) add_custom_command(OUTPUT ${KERNELS_BINARY_DIR}/${__fname} - COMMAND ${UNZIPPER} -k ${db_bzip2_file} > ${KERNELS_BINARY_DIR}/${__fname}) + COMMAND ${UNZIPPER} -dc -k ${db_bzip2_file} > ${KERNELS_BINARY_DIR}/${__fname}) string(REPLACE "." "_" __tname ${__fname}) add_custom_target(generate_${__tname} ALL DEPENDS ${KERNELS_BINARY_DIR}/${__fname}) @@ -609,7 +620,7 @@ function(unpack_db db_bzip2_file) set(__fname ${__fname} PARENT_SCOPE) endfunction() -file(GLOB PERF_DB_BZIP_FILES CONFIGURE_DEPENDS "${KERNELS_SOURCE_DIR}/*.db.bz2") +file(GLOB PERF_DB_BZIP_FILES CONFIGURE_DEPENDS "${KERNELS_SOURCE_DIR}/*.db.bz2" "${KERNELS_SOURCE_DIR}/*.db.txt.bz2") file(GLOB FIND_DB_BZIP_FILES CONFIGURE_DEPENDS "${KERNELS_SOURCE_DIR}/*.fdb.txt.bz2") foreach(DB_BZIP_FILE ${PERF_DB_BZIP_FILES} ${FIND_DB_BZIP_FILES}) @@ -642,8 +653,8 @@ function(install_kdb FILE_NAME COMPONENT_NAME) endfunction() # Both the lists below should be in sync always -set(KDB_BZ2_FILES gfx90a.kdb.bz2 gfx1030.kdb.bz2 gfx908.kdb.bz2 gfx906.kdb.bz2 gfx900.kdb.bz2) -set(COMPONENT_LST gfx90akdb gfx1030kdb gfx908kdb gfx906kdb gfx900kdb) +set(KDB_BZ2_FILES gfx942.kdb.bz2 gfx90a.kdb.bz2 gfx1030.kdb.bz2 gfx908.kdb.bz2 gfx906.kdb.bz2 gfx900.kdb.bz2) +set(COMPONENT_LST gfx942kdb gfx90akdb gfx1030kdb gfx908kdb gfx906kdb gfx900kdb) if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.17) foreach(__file __component IN ZIP_LISTS KDB_BZ2_FILES COMPONENT_LST) @@ -671,7 +682,7 @@ endif() # End KDB package creation if(NOT MIOPEN_TEST_DISCRETE) -list(APPEND CPACK_COMPONENTS_ALL client) +list(APPEND CPACK_COMPONENTS_ALL clients) endif() rocm_create_package( diff --git a/Dockerfile b/Dockerfile index 89c8065f48..790b0c83cf 100755 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,6 @@ RUN dpkg --add-architecture i386 # Install preliminary dependencies RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ - "linux-headers-$(uname -r)" "linux-modules-extra-$(uname -r)" \ apt-utils \ ca-certificates \ curl \ @@ -15,17 +14,22 @@ DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ gnupg2 \ wget +# This stage fails quite often and not always necessary +# RUN apt-get update && \ +# DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ +# "linux-headers-$(uname -r)" "linux-modules-extra-$(uname -r)" + #Add gpg keys ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn RUN curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor -o /etc/apt/trusted.gpg.d/rocm-keyring.gpg -RUN wget https://repo.radeon.com/amdgpu-install/6.1/ubuntu/jammy/amdgpu-install_6.1.60100-1_all.deb --no-check-certificate +RUN wget https://repo.radeon.com/amdgpu-install/6.2.2/ubuntu/jammy/amdgpu-install_6.2.60202-1_all.deb --no-check-certificate RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \ - ./amdgpu-install_6.1.60100-1_all.deb + ./amdgpu-install_6.2.60202-1_all.deb # Add rocm repository -RUN export ROCM_APT_VER=6.1;\ +RUN export ROCM_APT_VER=6.2.2;\ echo $ROCM_APT_VER &&\ sh -c 'echo deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/rocm-keyring.gpg] https://repo.radeon.com/amdgpu/$ROCM_APT_VER/ubuntu jammy main > /etc/apt/sources.list.d/amdgpu.list' &&\ sh -c 'echo deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/rocm-keyring.gpg] https://repo.radeon.com/rocm/apt/$ROCM_APT_VER jammy main > /etc/apt/sources.list.d/rocm.list' @@ -84,7 +88,9 @@ ADD requirements.txt /requirements.txt ADD dev-requirements.txt /dev-requirements.txt # Install dependencies # TODO: Add --std=c++14 -ARG GPU_ARCH=";" +# GPU_ARCH can be defined in docker build process +ARG GPU_ARCHS=gfx908;gfx90a;gfx942;gfx1100 +# install to /opt/rocm will cause permission issue ARG PREFIX=/usr/local ARG USE_FIN="OFF" ARG CCACHE_SECONDARY_STORAGE="" @@ -101,15 +107,36 @@ RUN ccache -s # purge existing composable kernel installed with ROCm # hence cannot use autoremove since it will remove more components +# even purge will remove some other components which is not ideal RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get purge -y --allow-unauthenticated \ - composablekernel-dev + composablekernel-dev \ + miopen-hip + +# TODO: it should be able to automatically get commit hash from requirements.txt +ARG CK_COMMIT=fb948120d2d674607e70d0e7587dcb249c0e74c7 +RUN wget -O ck.tar.gz https://www.github.com/ROCm/composable_kernel/archive/${CK_COMMIT}.tar.gz && \ + tar zxvf ck.tar.gz &&\ + cd composable_kernel-${CK_COMMIT} && \ + mkdir build && cd build && \ + CXX=/opt/rocm/bin/amdclang++ cmake \ + -D CMAKE_PREFIX_PATH=/opt/rocm \ + -D CMAKE_CXX_COMPILER_LAUNCHER="${COMPILER_LAUNCHER}" \ + -D CMAKE_BUILD_TYPE=Release \ + -D GPU_ARCHS="gfx908;gfx90a;gfx942;gfx1100" \ + -D CMAKE_CXX_FLAGS=" -O3 " .. && \ + make -j $(nproc) install + +# Composable Kernel installed separated from rbuild to take in values from GPU_ARCHS +# this can minimize build time +RUN sed -i '/composable_kernel/d' /requirements.txt + ARG COMPILER_LAUNCHER="" # rbuild is used to trigger build of requirements.txt, dev-requirements.txt RUN if [ "$USE_FIN" = "ON" ]; then \ - rbuild prepare -s fin -d $PREFIX -DCMAKE_CXX_COMPILER_LAUNCHER="${COMPILER_LAUNCHER}"; \ + rbuild prepare -s fin -d $PREFIX -DGPU_ARCHS="${GPU_ARCHS}" -DCMAKE_CXX_COMPILER_LAUNCHER="${COMPILER_LAUNCHER}"; \ else \ - rbuild prepare -s develop -d $PREFIX -DCMAKE_CXX_COMPILER_LAUNCHER="${COMPILER_LAUNCHER}"; \ + rbuild prepare -s develop -d $PREFIX -DGPU_ARCHS="${GPU_ARCHS}" -DCMAKE_CXX_COMPILER_LAUNCHER="${COMPILER_LAUNCHER}"; \ fi RUN ccache -s @@ -120,4 +147,7 @@ RUN pip3 install -r /doc-requirements.txt # Composable Kernel requires this version cmake RUN pip3 install --upgrade cmake==3.27.5 +# groupadd can add one group a time RUN groupadd -f render +RUN groupadd -f video +RUN usermod -a -G render,video root \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index 424c1d8a2f..87a48be3b0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,21 +33,27 @@ def cmake_build(Map conf=[:]){ def debug_flags = "-g -fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize-recover=undefined -Wno-option-ignored " + conf.get("extradebugflags", "") def build_envs = "CTEST_PARALLEL_LEVEL=4 " + conf.get("build_env","") def prefixpath = conf.get("prefixpath","/opt/rocm") + def build_type_debug = (conf.get("build_type",'release') == 'debug') + def mlir_args = " -DMIOPEN_USE_MLIR=" + conf.get("mlir_build", "ON") + // WORKAROUND_ISSUE_3192 Disabling MLIR for debug builds since MLIR generates sanitizer errors. + if (build_type_debug) + { + mlir_args = " -DMIOPEN_USE_MLIR=OFF" + } + def setup_args = mlir_args + " -DMIOPEN_GPU_SYNC=Off " + conf.get("setup_flags","") def build_fin = conf.get("build_fin", "OFF") setup_args = setup_args + " -DCMAKE_PREFIX_PATH=${prefixpath} " - def build_type_debug = (conf.get("build_type",'release') == 'debug') - //cmake_env can overwrite default CXX variables. def cmake_envs = "CXX=${compiler} CXXFLAGS='-Werror' " + conf.get("cmake_ex_env","") def package_build = (conf.get("package_build","") == "true") if (package_build == true) { - make_targets = "miopen_gtest package" + make_targets = "miopen_gtest package miopen_gtest_check" setup_args = " -DMIOPEN_TEST_DISCRETE=OFF " + setup_args } @@ -72,9 +78,7 @@ def cmake_build(Map conf=[:]){ test_flags = " --disable-verification-cache " + test_flags } - if(conf.get("codecov", false)){ //Need - setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags} -fprofile-arcs -ftest-coverage' -DCODECOV_TEST=On " + setup_args - }else if(build_type_debug){ + if(build_type_debug){ setup_args = " -DCMAKE_BUILD_TYPE=debug -DCMAKE_CXX_FLAGS_DEBUG='${debug_flags}'" + setup_args }else{ setup_args = " -DCMAKE_BUILD_TYPE=release" + setup_args @@ -194,9 +198,9 @@ def getDockerImage(Map conf=[:]) { env.DOCKER_BUILDKIT=1 def prefixpath = conf.get("prefixpath", "/opt/rocm") // one image for each prefix 1: /usr/local 2:/opt/rocm - def gpu_arch = "gfx900;gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx1100;gfx1101;gfx1102" // prebuilt dockers should have all the architectures enabled so one image can be used for all stages + def gpu_arch = "gfx908;gfx90a;gfx942;gfx1100" // prebuilt dockers should have all the architectures enabled so one image can be used for all stages def mlir_build = conf.get("mlir_build", "ON") // always ON - def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg GPU_ARCH='${gpu_arch}' --build-arg USE_MLIR='${mlir_build}' " + def dockerArgs = "--build-arg BUILDKIT_INLINE_CACHE=1 --build-arg PREFIX=${prefixpath} --build-arg GPU_ARCHS='\"${gpu_arch}\"' --build-arg USE_MLIR='${mlir_build}' " if(env.CCACHE_HOST) { def check_host = sh(script:"""(printf "PING\r\n";) | nc -N ${env.CCACHE_HOST} 6379 """, returnStdout: true).trim() @@ -240,7 +244,6 @@ def buildHipClangJob(Map conf=[:]){ show_node_info() miopenCheckout() env.HSA_ENABLE_SDMA=0 - env.CODECOV_TOKEN="aec031be-7673-43b5-9840-d8fb71a2354e" env.DOCKER_BUILDKIT=1 def image def dockerOpts="--device=/dev/kfd --device=/dev/dri --group-add video --group-add render --cap-add=SYS_PTRACE --security-opt seccomp=unconfined" @@ -250,12 +253,11 @@ def buildHipClangJob(Map conf=[:]){ def variant = env.STAGE_NAME - def codecov = conf.get("codecov", false) def needs_gpu = conf.get("needs_gpu", true) def lfs_pull = conf.get("lfs_pull", false) def retimage - gitStatusWrapper(credentialsId: "${env.status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'MIOpen') { + gitStatusWrapper(credentialsId: "${env.miopen_git_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'MIOpen') { try { (retimage, image) = getDockerImage(conf) if (needs_gpu) { @@ -284,24 +286,13 @@ def buildHipClangJob(Map conf=[:]){ } withDockerContainer(image: image, args: dockerOpts + ' -v=/var/jenkins/:/var/jenkins') { - timeout(time: 210, unit:'MINUTES') + timeout(time: 420, unit:'MINUTES') { if (lfs_pull) { sh "git lfs pull --exclude=" } cmake_build(conf) - - if (codecov) { - sh ''' - cd build - lcov --directory . --capture --output-file $(pwd)/coverage.info - lcov --remove $(pwd)/coverage.info '/usr/*' --output-file $(pwd)/coverage.info - lcov --list $(pwd)/coverage.info - curl -s https://codecov.io/bash | bash - echo "Uploaded" - ''' - } } } } @@ -410,10 +401,9 @@ def CheckPerfDbValid(Map conf=[:]){ /// BuildType := { Release* | Debug | Install } [ BuildTypeModifier ] /// * BuildTypeModifier := { NOCOMGR | Embedded | Static | Normal-Find | Fast-Find /// NOCK | NOMLIR | Tensile | Tensile-Latest | Package | ... } -/// TestSet := { All | Smoke* | | Build-only } [ Codecov ] +/// TestSet := { All | Smoke* | | Build-only } /// * "All" corresponds to "cmake -DMIOPEN_TEST_ALL=On". /// * "Smoke" (-DMIOPEN_TEST_ALL=Off) is the default and usually not specified. -/// * "Codecov" is optional code coverage analysis. /// * "Performance Dataset" is a performance test with a specified dataset. /// Target := { gfx908 | gfx90a | Vega20 | Vega10 | Vega* | gfx1030 } [ Xnack+ ] /// * "Vega" (gfx906 or gfx900) is the default and usually not specified. @@ -550,7 +540,7 @@ pipeline { NOMLIR_flags = " -DMIOPEN_USE_MLIR=Off" } triggers{ - + cron(env.BRANCH_NAME == env.NIGHTLY_BRANCH ? env.NIGHTLY_SCHEDULE : '') } stages{ @@ -558,7 +548,7 @@ pipeline { when { expression { params.BUILD_DOCKER && params.TARGET_NOGPU && params.DATATYPE_NA } } - agent{ label rocmnode("nogpu") } + agent{ label rocmnode("gfx90a") } steps{ getDockerImage() } @@ -648,7 +638,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(make_targets: Smoke_targets) + buildHipClangJobAndReboot(make_targets: Smoke_targets, build_install: "true") } } stage('Fp32 Hip Debug gfx90a') { @@ -661,7 +651,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets) + buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, build_install: "true") } } stage('Fp32 Hip Debug gfx908') { @@ -674,7 +664,7 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets) + buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, build_install: "true") } } stage('Fp32 Hip Debug gfx94X') { @@ -687,7 +677,7 @@ pipeline { } agent{ label rocmnode("gfx94X") } steps{ - buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, needs_reboot:false) + buildHipClangJobAndReboot(build_type: 'debug', make_targets: Smoke_targets, needs_reboot:false, build_install: "true") } } } @@ -711,7 +701,7 @@ pipeline { NOCOMGR_build_cmd = "CTEST_PARALLEL_LEVEL=4 MIOPEN_LOG_LEVEL=5 make -j\$(nproc) check" } steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOCOMGR_flags, build_cmd: NOCOMGR_build_cmd, test_flags: ' --verbose ') + buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOCOMGR_flags, build_cmd: NOCOMGR_build_cmd, test_flags: ' --verbose ', build_install: "true") } } stage('Fp32 Hip Debug NOMLIR gfx90a') { @@ -728,7 +718,7 @@ pipeline { NOMLIR_build_cmd = "CTEST_PARALLEL_LEVEL=4 MIOPEN_LOG_LEVEL=5 make -j\$(nproc) check" } steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOMLIR_flags, build_cmd: NOMLIR_build_cmd, test_flags: ' --verbose ') + buildHipClangJobAndReboot( build_type: 'debug', setup_flags: NOMLIR_flags, build_cmd: NOMLIR_build_cmd, test_flags: ' --verbose ', build_install: "true") } } stage('Fp32 Hip Debug NOCK gfx90a Build-Only') { @@ -741,7 +731,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: "-DMIOPEN_USE_COMPOSABLEKERNEL=Off", make_targets: "") + buildHipClangJobAndReboot( build_type: 'debug', setup_flags: "-DMIOPEN_USE_COMPOSABLEKERNEL=Off", make_targets: "", build_install: "true") } } stage('Fp32 Hip Debug Embedded Vega20') { @@ -757,7 +747,7 @@ pipeline { Embedded_flags = "-DMIOPEN_EMBED_DB='gfx906_60'" } steps{ - buildHipClangJobAndReboot( build_type: 'debug', setup_flags: Embedded_flags, build_env: extra_log_env, test_flags: ' --verbose ') + buildHipClangJobAndReboot( build_type: 'debug', setup_flags: Embedded_flags, build_env: extra_log_env, test_flags: ' --verbose ', build_install: "true") } } stage('Fp32 Hip Static gfx90a') { @@ -770,7 +760,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot( setup_flags: "-DBUILD_SHARED_LIBS=Off", mlir_build: 'OFF') + buildHipClangJobAndReboot( setup_flags: "-DBUILD_SHARED_LIBS=Off", mlir_build: 'OFF', build_install: "true") } } stage('Fp32 Hip Normal-Find gfx90a') { @@ -787,7 +777,7 @@ pipeline { execute_cmd = "bin/test_conv2d --disable-verification-cache" } steps{ - buildHipClangJobAndReboot(make_targets: make_targets, execute_cmd: execute_cmd, find_mode: "Normal") + buildHipClangJobAndReboot(make_targets: make_targets, execute_cmd: execute_cmd, find_mode: "Normal", build_install: "true") } } stage('Fp32 Hip Fast-Find gfx90a') { @@ -804,7 +794,7 @@ pipeline { execute_cmd = "MIOPEN_FIND_MODE=2 CTEST_PARALLEL_LEVEL=4 bin/test_conv2d --disable-verification-cache" } steps{ - buildHipClangJobAndReboot( make_targets: make_targets, execute_cmd: execute_cmd) + buildHipClangJobAndReboot( make_targets: make_targets, execute_cmd: execute_cmd, build_install: "true") } } stage('Fp32 Hip gfx90a') { @@ -830,7 +820,22 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(make_targets: Smoke_targets, setup_flags: "-DMIOPEN_USE_SQLITE_PERF_DB=On") + buildHipClangJobAndReboot(make_targets: Smoke_targets, setup_flags: "-DMIOPEN_USE_SQLITE_PERF_DB=On", build_install: "true") + } + } + stage('Fp32 Hip Fin Interface gfx90a') { + when { + beforeAgent true + expression { params.TARGET_GFX90A } + } + options { + retry(2) + } + agent{ label rocmnode("gfx90a") } + steps{ + buildHipClangJobAndReboot(setup_flags: "-DMIOPEN_ENABLE_FIN_INTERFACE=On", + make_targets: "test_unit_FinInterface", + execute_cmd: "bin/test_unit_FinInterface") } } } @@ -850,7 +855,7 @@ pipeline { } agent{ label rocmnode("vega20") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets) + buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, build_install: "true") } } stage('Bf16 Hip Vega20') { @@ -863,7 +868,7 @@ pipeline { } agent{ label rocmnode("vega20") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets) + buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, build_install: "true") } } stage('Fp16 Hip gfx908') { @@ -876,7 +881,7 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets) + buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, build_install: "true") } } stage('Bf16 Hip gfx908') { @@ -889,7 +894,7 @@ pipeline { } agent{ label rocmnode("gfx908") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets) + buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, build_install: "true") } } stage('Fp16 Hip gfx90a') { @@ -902,7 +907,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets) + buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, build_install: "true") } } stage('Bf16 Hip gfx90a') { @@ -915,7 +920,7 @@ pipeline { } agent{ label rocmnode("gfx90a") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets) + buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, build_install: "true") } } stage('Fp16 Hip gfx94X') { @@ -928,7 +933,7 @@ pipeline { } agent{ label rocmnode("gfx94X") } steps{ - buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, needs_reboot:false) + buildHipClangJobAndReboot( setup_flags: Fp16_flags, make_targets: Smoke_targets, needs_reboot:false, build_install: "true") } } stage('Bf16 Hip gfx94X') { @@ -941,7 +946,7 @@ pipeline { } agent{ label rocmnode("gfx94X") } steps{ - buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, needs_reboot:false) + buildHipClangJobAndReboot(setup_flags: Bf16_flags, make_targets: Smoke_targets, needs_reboot:false, build_install: "true") } } } @@ -969,7 +974,7 @@ pipeline { buildHipClangJobAndReboot(lfs_pull: true, setup_flags: "-DMIOPEN_TEST_DBSYNC=1", make_targets: 'test_db_sync', - execute_cmd: 'MIOPEN_TEST_DBSYNC=1 ./bin/test_db_sync', + execute_cmd: './bin/test_db_sync', needs_gpu:false, needs_reboot:false, build_install: "true") @@ -988,7 +993,7 @@ pipeline { buildHipClangJobAndReboot(lfs_pull: true, setup_flags: "-DMIOPEN_TEST_DBSYNC=1", make_targets: 'test_db_sync', - execute_cmd: 'MIOPEN_TEST_DBSYNC=1 ./bin/test_db_sync', + execute_cmd: './bin/test_db_sync', needs_gpu:false, needs_reboot:false, build_install: "true") @@ -1007,7 +1012,7 @@ pipeline { buildHipClangJobAndReboot(lfs_pull: true, setup_flags: "-DMIOPEN_TEST_DBSYNC=1", make_targets: 'test_db_sync', - execute_cmd: 'MIOPEN_TEST_DBSYNC=1 ./bin/test_db_sync', + execute_cmd: './bin/test_db_sync', needs_gpu:false, needs_reboot:false, build_install: "true") diff --git a/LICENSE.txt b/LICENSE.txt index 157ccd1ec2..a6b96e6aab 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2017 - 2022 Advanced Micro Devices, Inc. All rights Reserved. +Copyright (c) 2017 - 2024 Advanced Micro Devices, Inc. All rights Reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 5858223141..f707fd6da1 100755 --- a/README.md +++ b/README.md @@ -2,9 +2,10 @@ MIOpen is AMD's library for high-performance machine learning primitives. -You can find sources and binaries in our [GitHub repository](https://github.com/ROCm/MIOpen) and -our documentation on -[ROCm docs](https://rocm.docs.amd.com/projects/MIOpen/en/latest/index.html). +You can find sources and binaries in our [GitHub repository](https://github.com/ROCm/MIOpen). + +> [!NOTE] +> The published MIOpen documentation is available at [MIOpen](https://rocm.docs.amd.com/projects/MIOpen/en/latest/index.html) in an organized, easy-to-read format, with search and a table of contents. The documentation source files reside in the MIOpen/docs folder of this repository. As with all ROCm projects, the documentation is open source. For more information, see [Contribute to ROCm documentation](https://rocm.docs.amd.com/en/latest/contribute/contributing.html). MIOpen supports these programming models (backends): @@ -77,11 +78,10 @@ kernels don't reduce startup time on subsequent runs. To install the kernels package for your GPU architecture, use the following command: ``` shell -apt-get install miopenkernels-- +apt-get install miopen-hip-kdb ``` -Where ```` is the GPU architecture (e.g., `gfx900`, `gfx906`, `gfx1030` ) and `` is the -number of CUs available in the GPU (e.g., `56`, `64`). +Where ```` is the GPU architecture (e.g., `gfx900`, `gfx906`, `gfx1030` ). >[!NOTE] >Not installing these packages doesn't impact the functioning of MIOpen, since MIOpen compiles diff --git a/addkernels/CMakeLists.txt b/addkernels/CMakeLists.txt index 30c54ee491..88d6c7424f 100644 --- a/addkernels/CMakeLists.txt +++ b/addkernels/CMakeLists.txt @@ -32,4 +32,14 @@ if(HAS_LIB_STD_FILESYSTEM) target_link_libraries(addkernels PRIVATE stdc++fs) endif() +# This hack is for MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM. +# We can't use config.h because it is generated after addkernels. +target_compile_definitions(addkernels PRIVATE -DMIOPEN_HACK_DO_NOT_INCLUDE_CONFIG_H=1) +if(MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM) + target_compile_definitions(addkernels PRIVATE -DMIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM=1) + target_link_libraries(addkernels PRIVATE Boost::filesystem) +else() + target_compile_definitions(addkernels PRIVATE -DMIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM=0) +endif() + clang_tidy_check(addkernels) diff --git a/cmake/CheckCXXLinkerFlag.cmake b/cmake/CheckCXXLinkerFlag.cmake index 7d9e82420e..6ae5f1cf6f 100644 --- a/cmake/CheckCXXLinkerFlag.cmake +++ b/cmake/CheckCXXLinkerFlag.cmake @@ -55,6 +55,7 @@ include (CheckCXXSourceCompiles) function(check_cxx_linker_flag _flag _var) set (_source "int main() { return 0; }") - check_cxx_source_compiles("${_source}" _result ${check_cxx_linker_flag_patterns}) - set(${_var} "${_result}" PARENT_SCOPE) + set(CMAKE_REQUIRED_LINK_OPTIONS "${_flag}") + check_cxx_source_compiles("${_source}" "_${_var}" ${check_cxx_linker_flag_patterns}) + set(${_var} "${_${_var}}" PARENT_SCOPE) endfunction() diff --git a/cmake/EnableCompilerWarnings.cmake b/cmake/EnableCompilerWarnings.cmake index 03a26b0c09..b64c717020 100644 --- a/cmake/EnableCompilerWarnings.cmake +++ b/cmake/EnableCompilerWarnings.cmake @@ -105,7 +105,8 @@ set(__clang_cxx_compile_options -Wno-unused-value -Wno-unused-parameter -Wno-missing-noreturn - -Wno-tautological-constant-out-of-range-compare) + -Wno-tautological-constant-out-of-range-compare + -Wno-c++20-extensions) if(WIN32) list(APPEND __clang_cxx_compile_options -fdelayed-template-parsing diff --git a/cmake/hip-config.cmake b/cmake/hip-config.cmake deleted file mode 100644 index 279de67bba..0000000000 --- a/cmake/hip-config.cmake +++ /dev/null @@ -1,122 +0,0 @@ -get_filename_component(PACKAGE_PREFIX_DIR "${CMAKE_CURRENT_LIST_DIR}/../../../" ABSOLUTE) - -macro(set_and_check _var _file) - set(${_var} "${_file}") - if(NOT EXISTS "${_file}") - message(FATAL_ERROR "File or directory ${_file} referenced by variable ${_var} does not exist !") - endif() -endmacro() - -macro(check_required_components _NAME) - foreach(comp ${${_NAME}_FIND_COMPONENTS}) - if(NOT ${_NAME}_${comp}_FOUND) - if(${_NAME}_FIND_REQUIRED_${comp}) - set(${_NAME}_FOUND FALSE) - endif() - endif() - endforeach() -endmacro() - -#################################################################################### - -include(CMakeFindDependencyMacro OPTIONAL RESULT_VARIABLE _CMakeFindDependencyMacro_FOUND) -if (NOT _CMakeFindDependencyMacro_FOUND) - macro(find_dependency dep) - if (NOT ${dep}_FOUND) - set(cmake_fd_version) - if (${ARGC} GREATER 1) - set(cmake_fd_version ${ARGV1}) - endif() - set(cmake_fd_exact_arg) - if(${CMAKE_FIND_PACKAGE_NAME}_FIND_VERSION_EXACT) - set(cmake_fd_exact_arg EXACT) - endif() - set(cmake_fd_quiet_arg) - if(${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY) - set(cmake_fd_quiet_arg QUIET) - endif() - set(cmake_fd_required_arg) - if(${CMAKE_FIND_PACKAGE_NAME}_FIND_REQUIRED) - set(cmake_fd_required_arg REQUIRED) - endif() - find_package(${dep} ${cmake_fd_version} - ${cmake_fd_exact_arg} - ${cmake_fd_quiet_arg} - ${cmake_fd_required_arg} - ) - string(TOUPPER ${dep} cmake_dep_upper) - if (NOT ${dep}_FOUND AND NOT ${cmake_dep_upper}_FOUND) - set(${CMAKE_FIND_PACKAGE_NAME}_NOT_FOUND_MESSAGE "${CMAKE_FIND_PACKAGE_NAME} could not be found because dependency ${dep} could not be found.") - set(${CMAKE_FIND_PACKAGE_NAME}_FOUND False) - return() - endif() - set(cmake_fd_version) - set(cmake_fd_required_arg) - set(cmake_fd_quiet_arg) - set(cmake_fd_exact_arg) - endif() - endmacro() -endif() - -set(HIP_COMPILER "clang") -set(HIP_RUNTIME "VDI") - -set_and_check( hip_INCLUDE_DIR "${PACKAGE_PREFIX_DIR}/include" ) -set_and_check( hip_INCLUDE_DIRS "${hip_INCLUDE_DIR}" ) -set_and_check( hip_LIB_INSTALL_DIR "${PACKAGE_PREFIX_DIR}/lib" ) -set_and_check( hip_BIN_INSTALL_DIR "${PACKAGE_PREFIX_DIR}/bin" ) - -set_and_check(hip_HIPCC_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipcc") -set_and_check(hip_HIPCONFIG_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipconfig") - -get_filename_component(HIP_CLANG_ROOT "${CMAKE_CXX_COMPILER}" PATH) -get_filename_component(HIP_CLANG_ROOT "${HIP_CLANG_ROOT}" PATH) -file(GLOB HIP_CLANG_INCLUDE_SEARCH_PATHS ${HIP_CLANG_ROOT}/lib/clang/*/include) -find_path(HIP_CLANG_INCLUDE_PATH stddef.h - HINTS - ${HIP_CLANG_INCLUDE_SEARCH_PATHS} - NO_DEFAULT_PATH) - -find_dependency(amd_comgr) -find_dependency(AMDDeviceLibs) -set(AMDGPU_TARGETS "gfx900;gfx906" CACHE STRING "AMD GPU targets to compile for") -set(GPU_TARGETS "${AMDGPU_TARGETS}" CACHE STRING "GPU targets to compile for") - -include( "${CMAKE_CURRENT_LIST_DIR}/hip-targets.cmake" ) - -set_property(TARGET hip::device APPEND PROPERTY - INTERFACE_COMPILE_OPTIONS -x hip --hip-device-lib-path=${AMD_DEVICE_LIBS_PREFIX}/lib -) - -set_property(TARGET hip::device APPEND PROPERTY - INTERFACE_LINK_LIBRARIES --hip-device-lib-path=${AMD_DEVICE_LIBS_PREFIX}/lib --hip-link -) - -set_property(TARGET hip::device APPEND PROPERTY - INTERFACE_INCLUDE_DIRECTORIES "${HIP_CLANG_INCLUDE_PATH}" -) - -set_property(TARGET hip::device APPEND PROPERTY - INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${HIP_CLANG_INCLUDE_PATH}" -) - -foreach(GPU_TARGET ${GPU_TARGETS}) - set_property(TARGET hip::device APPEND PROPERTY - INTERFACE_COMPILE_OPTIONS "--cuda-gpu-arch=${GPU_TARGET}" - ) - set_property(TARGET hip::device APPEND PROPERTY - INTERFACE_LINK_LIBRARIES "--cuda-gpu-arch=${GPU_TARGET}" - ) -endforeach() - -set( hip_LIBRARIES hip::host hip::device) -set( hip_LIBRARY ${hip_LIBRARIES}) - -set(HIP_INCLUDE_DIR ${hip_INCLUDE_DIR}) -set(HIP_INCLUDE_DIRS ${hip_INCLUDE_DIRS}) -set(HIP_LIB_INSTALL_DIR ${hip_LIB_INSTALL_DIR}) -set(HIP_BIN_INSTALL_DIR ${hip_BIN_INSTALL_DIR}) -set(HIP_LIBRARIES ${hip_LIBRARIES}) -set(HIP_LIBRARY ${hip_LIBRARY}) -set(HIP_HIPCC_EXECUTABLE ${hip_HIPCC_EXECUTABLE}) -set(HIP_HIPCONFIG_EXECUTABLE ${hip_HIPCONFIG_EXECUTABLE}) diff --git a/dev-requirements.txt b/dev-requirements.txt index 3d5b4d4bdc..009e2f93a5 100755 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ -ROCm/rocm-recipes@329203d79f9fe77ae5d0d742af0966bc57f4dfc8 +ROCm/rocm-recipes@92c6695449c85887962f45509b376f2eb0d284f7 -f requirements.txt danmar/cppcheck@2.12.1 diff --git a/docs/conceptual/perfdb.rst b/docs/conceptual/perfdb.rst index 3ecb3c2a5a..b41a944fa4 100644 --- a/docs/conceptual/perfdb.rst +++ b/docs/conceptual/perfdb.rst @@ -55,6 +55,16 @@ You can override the latter two conditions by enforcing the search using the ``- MIOPEN_FIND_ENFORCE`` environment variable. You can also use this variable to remove values from User PerfDb, as described in the following section. +To optimize performance, MIOpen provides several find modes to accelerate find API calls. +These modes include: + +* normal find +* fast find +* hybrid find +* dynamic hybrid find + +For more information about MIOpen find modes, see :ref:`Find modes `. + Using MIOPEN_FIND_ENFORCE ---------------------------------------------------------------------------------------------------------- diff --git a/docs/conceptual/porting-guide.rst b/docs/conceptual/porting-guide.rst index a61059eba4..2c50bba940 100644 --- a/docs/conceptual/porting-guide.rst +++ b/docs/conceptual/porting-guide.rst @@ -543,8 +543,6 @@ Convolution operations int *returnedAlgoCount, cudnnConvolutionFwdAlgoPerf_t *perfResults) - .. code-block:: cpp - cudnnStatus_t cudnnFindConvolutionForwardAlgorithmEx( cudnnHandle_t handle, @@ -561,8 +559,6 @@ Convolution operations void *workSpace, size_t workSpaceSizeInBytes) - .. code-block:: cpp - cudnnStatus_t cudnnGetConvolutionForwardAlgorithm( cudnnHandle_t handle, @@ -636,8 +632,6 @@ Convolution operations int *returnedAlgoCount, cudnnConvolutionBwdFilterAlgoPerf_t *perfResults) - .. code-block:: cpp - cudnnStatus_t cudnnFindConvolutionBackwardFilterAlgorithmEx( cudnnHandle_t handle, @@ -654,8 +648,6 @@ Convolution operations void *workSpace, size_t workSpaceSizeInBytes) - .. code-block:: cpp - cudnnStatus_t cudnnGetConvolutionBackwardFilterAlgorithm( cudnnHandle_t handle, @@ -705,8 +697,6 @@ Convolution operations int *returnedAlgoCount, cudnnConvolutionBwdDataAlgoPerf_t *perfResults) - .. code-block:: cpp - cudnnStatus_t cudnnFindConvolutionBackwardDataAlgorithmEx( cudnnHandle_t handle, @@ -723,8 +713,6 @@ Convolution operations void *workSpace, size_t workSpaceSizeInBytes) - .. code-block:: cpp - cudnnStatus_t cudnnGetConvolutionBackwardDataAlgorithm( cudnnHandle_t handle, diff --git a/docs/how-to/find-and-immediate.rst b/docs/how-to/find-and-immediate.rst index d94eb01a55..70e4198c66 100644 --- a/docs/how-to/find-and-immediate.rst +++ b/docs/how-to/find-and-immediate.rst @@ -214,6 +214,7 @@ OpenCL). MIOpenGEMM only contains support for FP32. The HIP backend uses rocBLAS as its fallback path, which contains a more robust set of datatypes. +.. _find_modes: Find modes ============================================================ diff --git a/docs/install/install.rst b/docs/install/install.rst index a24a5a6bf9..077701ffd2 100644 --- a/docs/install/install.rst +++ b/docs/install/install.rst @@ -65,10 +65,9 @@ To install the kernels package for your GPU architecture, use the following comm .. code:: shell - apt-get install miopenkernels-- + apt-get install miopen-hip-kdb -Where ```` is the GPU architecture (e.g., ``gfx900``, ``gfx906``, ``gfx1030`` ) and ```` is the -number of CUs available in the GPU (e.g., ``56``, ``64``). +Where ```` is the GPU architecture (e.g., ``gfx900``, ``gfx906``, ``gfx1030`` ). .. note:: diff --git a/docs/license.md b/docs/license.md index bfc65acd03..a9fe13a7b2 100644 --- a/docs/license.md +++ b/docs/license.md @@ -1,3 +1,10 @@ +--- +myst: + html_meta: + "description": "MIOpen licensing information" + "keywords": "MIOpen, ROCm, API, documentation, license" +--- + # License ```{include} ../LICENSE.txt diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 02496548e8..c2b74eabee 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -33,3 +33,9 @@ The MIOpen API library is structured as follows: * :doc:`Cat <../doxygen/html/group__cat>` (experimental) * :doc:`SGD <../doxygen/html/group___s_g_d>` (experimental) * :doc:`ReduceExtreme <../doxygen/html/group__ReduceExtreme>` (experimental) + * :doc:`Getitem <../doxygen/html/group__getitem>` (experimental) + * :doc:`ReduceCalculation <../doxygen/html/group__ReduceCalculation>` (experimental) + * :doc:`RotaryPositionalEmbeddings <../doxygen/html/group__RotaryPositionalEmbeddings>` (experimental) + * :doc:`ReLU <../doxygen/html/group___re_l_u>` (experimental) + * :doc:`Kthvalue <../doxygen/html/group__kthvalue>` (experimental) + * :doc:`GLU <../doxygen/html/group__glu>` (experimental) diff --git a/docs/sphinx/requirements.in b/docs/sphinx/requirements.in index a88668ba54..ed04a6309d 100644 --- a/docs/sphinx/requirements.in +++ b/docs/sphinx/requirements.in @@ -1 +1 @@ -rocm-docs-core[api_reference]==1.5.0 +rocm-docs-core[api_reference]==1.12.0 diff --git a/docs/sphinx/requirements.txt b/docs/sphinx/requirements.txt index 8713c61807..1c99759f10 100644 --- a/docs/sphinx/requirements.txt +++ b/docs/sphinx/requirements.txt @@ -31,8 +31,12 @@ click==8.1.7 # sphinx-external-toc click-log==0.4.0 # via doxysphinx -cryptography==42.0.7 +contourpy==1.3.1 + # via matplotlib +cryptography==43.0.1 # via pyjwt +cycler==0.12.1 + # via matplotlib deprecated==1.2.14 # via pygithub docutils==0.21.2 @@ -41,10 +45,12 @@ docutils==0.21.2 # myst-parser # pydata-sphinx-theme # sphinx -doxysphinx==3.3.8 +doxysphinx==3.3.12 # via rocm-docs-core fastjsonschema==2.19.1 # via rocm-docs-core +fonttools==4.55.0 + # via matplotlib gitdb==4.0.11 # via gitpython gitpython==3.1.43 @@ -57,9 +63,11 @@ jinja2==3.1.4 # via # myst-parser # sphinx +kiwisolver==1.4.7 + # via matplotlib libsass==0.22.0 # via doxysphinx -lxml==4.9.4 +lxml==5.2.1 # via doxysphinx markdown-it-py==3.0.0 # via @@ -67,6 +75,8 @@ markdown-it-py==3.0.0 # myst-parser markupsafe==2.1.5 # via jinja2 +matplotlib==3.9.2 + # via doxysphinx mdit-py-plugins==0.4.1 # via myst-parser mdurl==0.1.2 @@ -76,11 +86,17 @@ mpire==2.10.2 myst-parser==3.0.1 # via rocm-docs-core numpy==1.26.4 - # via doxysphinx + # via + # contourpy + # doxysphinx + # matplotlib packaging==24.0 # via + # matplotlib # pydata-sphinx-theme # sphinx +pillow==11.0.0 + # via matplotlib pycparser==2.22 # via cffi pydata-sphinx-theme==0.15.2 @@ -95,14 +111,18 @@ pygments==2.18.0 # mpire # pydata-sphinx-theme # sphinx -pyjson5==1.6.6 +pyjson5==1.6.7 # via doxysphinx pyjwt[crypto]==2.8.0 # via pygithub pynacl==1.5.0 # via pygithub -pyparsing==3.1.2 - # via doxysphinx +pyparsing==3.2.0 + # via + # doxysphinx + # matplotlib +python-dateutil==2.9.0.post0 + # via matplotlib pyyaml==6.0.1 # via # myst-parser @@ -112,8 +132,10 @@ requests==2.32.2 # via # pygithub # sphinx -rocm-docs-core[api-reference]==1.5.0 +rocm-docs-core[api-reference]==1.12.0 # via -r requirements.in +six==1.16.0 + # via python-dateutil smmap==5.0.1 # via gitdb snowballstemmer==2.2.0 @@ -155,7 +177,7 @@ sphinxcontrib-serializinghtml==1.1.10 # via sphinx tomli==2.0.1 # via sphinx -tqdm==4.66.4 +tqdm==4.67.1 # via mpire typing-extensions==4.11.0 # via diff --git a/driver/CMakeLists.txt b/driver/CMakeLists.txt index de1a0edd2d..60d6fe6ce6 100644 --- a/driver/CMakeLists.txt +++ b/driver/CMakeLists.txt @@ -43,15 +43,22 @@ add_executable(MIOpenDriver dm_dropout.cpp dm_fusion.cpp dm_gemm.cpp + dm_getitem.cpp + dm_glu.cpp dm_groupnorm.cpp + dm_kthvalue.cpp dm_layernorm.cpp dm_lrn.cpp + dm_multimarginloss.cpp dm_pool.cpp + dm_prelu.cpp dm_reduce.cpp dm_reduceextreme.cpp + dm_reducecalculation.cpp dm_rnn.cpp + dm_rope.cpp + dm_softmarginloss.cpp dm_softmax.cpp - dm_sum.cpp dm_t5layernorm.cpp dm_tensorop.cpp dm_transformers_adam_w.cpp diff --git a/driver/InputFlags.cpp b/driver/InputFlags.cpp index 41f872b0e8..98b3d85b6a 100644 --- a/driver/InputFlags.cpp +++ b/driver/InputFlags.cpp @@ -254,7 +254,7 @@ TensorParameters InputFlags::GetValueTensor(const std::string& long_name) const const auto& input = MapInputs.at(FindShortName(long_name)); const auto components = miopen::SplitDelim(input.value.c_str(), ','); - if(components.size() < 1) + if(components.empty()) return {}; auto parse = [](auto line) { @@ -292,6 +292,165 @@ TensorParameters InputFlags::GetValueTensor(const std::string& long_name) const MIOPEN_THROW("Too many tensor descriptor parameters."); } + +TensorParametersUint64 InputFlags::GetValueTensorUint64(const std::string& long_name) const +{ + const auto& input = MapInputs.at(FindShortName(long_name)); + const auto components = miopen::SplitDelim(input.value.c_str(), ','); + + if(components.size() < 1) + return {}; + + auto parse = [](auto line) { + auto ret = std::vector{}; + const auto strs = miopen::SplitDelim(line, 'x'); + for(auto&& str : strs) + { + auto elem = uint64_t{}; + auto ss = std::istringstream{str}; + ss >> elem; + + if(ss.bad() || ss.fail()) + MIOPEN_THROW("Invalid tensor component " + str + " in " + line + "."); + + ret.push_back(elem); + } + return ret; + }; + + auto lens = parse(components[0]); + + if(components.size() == 1) + return {lens}; + + auto layout = std::string{}; + auto strides = std::vector{}; + + if(std::isdigit(components[1][0])) + strides = parse(components[1]); + else + layout = components[1]; + + if(components.size() == 2) + return {lens, strides, layout}; + + MIOPEN_THROW("Too many tensor descriptor parameters."); +} + +std::vector InputFlags::GetValueVectorInt(const std::string& long_name) const +{ + const auto& input = MapInputs.at(FindShortName(long_name)); + + auto ret = std::vector{}; + const auto strs = miopen::SplitDelim(input.value.c_str(), ','); + + for(auto&& str : strs) + { + auto elem = int32_t{}; + auto ss = std::istringstream{str}; + ss >> elem; + + if(ss.bad() || ss.fail()) + MIOPEN_THROW("Invalid tensor component " + str + " in " + input.value.c_str() + "."); + + ret.push_back(elem); + } + + return ret; +} + +std::vector InputFlags::GetValueVectorUint64(const std::string& long_name) const +{ + const auto& input = MapInputs.at(FindShortName(long_name)); + + auto ret = std::vector{}; + const auto strs = miopen::SplitDelim(input.value.c_str(), ','); + + for(auto&& str : strs) + { + auto elem = uint64_t{}; + auto ss = std::istringstream{str}; + ss >> elem; + + if(ss.bad() || ss.fail()) + MIOPEN_THROW("Invalid tensor component " + str + " in " + input.value.c_str() + "."); + + ret.push_back(elem); + } + + return ret; +} + +std::vector> +InputFlags::GetValue2dVectorInt(const std::string& long_name) const +{ + const auto& input = MapInputs.at(FindShortName(long_name)); + const auto components = miopen::SplitDelim(input.value.c_str(), ','); + auto output = std::vector>{}; + + if(components.size() < 1) + return {}; + + auto parse = [](auto line) { + auto ret = std::vector{}; + const auto strs = miopen::SplitDelim(line, 'x'); + for(auto&& str : strs) + { + auto elem = int32_t{}; + auto ss = std::istringstream{str}; + ss >> elem; + + if(ss.bad() || ss.fail()) + MIOPEN_THROW("Invalid tensor component " + str + " in " + line + "."); + + ret.push_back(elem); + } + return ret; + }; + + for(auto&& component : components) + { + output.push_back(parse(component)); + } + + return output; +} + +std::vector> +InputFlags::GetValue2dVectorUint64(const std::string& long_name) const +{ + const auto& input = MapInputs.at(FindShortName(long_name)); + const auto components = miopen::SplitDelim(input.value.c_str(), ','); + auto output = std::vector>{}; + + if(components.size() < 1) + return {}; + + auto parse = [](auto line) { + auto ret = std::vector{}; + const auto strs = miopen::SplitDelim(line, 'x'); + for(auto&& str : strs) + { + auto elem = uint64_t{}; + auto ss = std::istringstream{str}; + ss >> elem; + + if(ss.bad() || ss.fail()) + MIOPEN_THROW("Invalid tensor component " + str + " in " + line + "."); + + ret.push_back(elem); + } + return ret; + }; + + for(auto&& component : components) + { + output.push_back(parse(component)); + } + + return output; +} + void InputFlags::SetValue(const std::string& long_name, const std::string& new_value) { char short_name = FindShortName(long_name); diff --git a/driver/InputFlags.hpp b/driver/InputFlags.hpp index 557a895b11..43f7c3a206 100644 --- a/driver/InputFlags.hpp +++ b/driver/InputFlags.hpp @@ -63,6 +63,25 @@ struct TensorParameters void CalculateStrides(); }; +struct TensorParametersUint64 +{ + std::vector lengths = {}; + std::vector strides = {}; + std::string layout = ""; + + TensorParametersUint64 FillMissing(const TensorParametersUint64& other) const + { + return { + (lengths.empty() ? other.lengths : lengths), + (strides.empty() ? other.strides : strides), + (layout.empty() ? other.layout : layout), + }; + } + + uint64_t SetTensordDescriptor(miopenTensorDescriptor_t result, miopenDataType_t data_type); + void CalculateStrides(); +}; + class InputFlags { std::map MapInputs; @@ -90,6 +109,11 @@ class InputFlags uint64_t GetValueUint64(const std::string& _long_name) const; double GetValueDouble(const std::string& _long_name) const; TensorParameters GetValueTensor(const std::string& long_name) const; + TensorParametersUint64 GetValueTensorUint64(const std::string& long_name) const; + std::vector GetValueVectorInt(const std::string& long_name) const; + std::vector GetValueVectorUint64(const std::string& long_name) const; + std::vector> GetValue2dVectorInt(const std::string& long_name) const; + std::vector> GetValue2dVectorUint64(const std::string& long_name) const; void SetValue(const std::string& long_name, const std::string& new_value); void StoreOptionalFlagValue(char short_name, const std::string& input_value); diff --git a/driver/README.md b/driver/README.md index 9cf5d704b5..d4d3bc1636 100644 --- a/driver/README.md +++ b/driver/README.md @@ -40,7 +40,7 @@ The supported base arguments: Summary of base_args meant for different datatypes and different operations: -![DatatypeSupport](../docs/data/driverTableCrop.png) +![DatatypeSupport](../docs/data/how-to/driverTableCrop.png) ## Executing MIOpenDriver diff --git a/driver/addlayernorm_driver.hpp b/driver/addlayernorm_driver.hpp index 4741d2d820..73cdd6110d 100644 --- a/driver/addlayernorm_driver.hpp +++ b/driver/addlayernorm_driver.hpp @@ -152,7 +152,6 @@ class AddLayerNormDriver : public Driver private: InputFlags inflags; - int forw; int dim_size; miopenTensorDescriptor_t inputDesc; @@ -176,8 +175,8 @@ class AddLayerNormDriver : public Driver std::vector weight; std::vector bias; std::vector out; - std::vector mean; - std::vector rstd; + std::vector mean; + std::vector rstd; std::vector outhost; std::vector meanhost; std::vector rstdhost; @@ -259,7 +258,7 @@ int AddLayerNormDriver::AddCmdLineArgs() inflags.AddInputFlag("eps", 'e', "0.00001", "Alpha (Default=0.00001)", "double"); inflags.AddInputFlag("normalized_dim", 'o', "3", "Nomalized Dim (Default=3)", "int"); inflags.AddInputFlag( - "mode", 'm', "0", "elemwise affine mode (0), weight and bias mode (1) (Default=0)", "int"); + "mode", 'm', "2", "elemwise affine mode (2), weight and bias mode (3) (Default=0)", "int"); inflags.AddInputFlag("iter", 'i', "10", "Number of Iterations (Default=10)", "int"); inflags.AddInputFlag("verify", 'V', "1", "Verify Each Layer (Default=1)", "int"); @@ -291,16 +290,16 @@ int AddLayerNormDriver::AllocateBuffersAndCopy() weight_dev = std::unique_ptr(new GPUMem(ctx, weight_sz, sizeof(Tgpu))); bias_dev = std::unique_ptr(new GPUMem(ctx, bias_sz, sizeof(Tgpu))); out_dev = std::unique_ptr(new GPUMem(ctx, out_sz, sizeof(Tgpu))); - mean_dev = std::unique_ptr(new GPUMem(ctx, mean_sz, sizeof(Tref))); - rstd_dev = std::unique_ptr(new GPUMem(ctx, rstd_sz, sizeof(Tref))); + mean_dev = std::unique_ptr(new GPUMem(ctx, mean_sz, sizeof(Tgpu))); + rstd_dev = std::unique_ptr(new GPUMem(ctx, rstd_sz, sizeof(Tgpu))); in = std::vector(in_sz, Tgpu0val); in2 = std::vector(in2_sz, Tgpu0val); weight = std::vector(weight_sz, Tgpu0val); bias = std::vector(bias_sz, Tgpu0val); out = std::vector(out_sz, Tgpu0val); - mean = std::vector(mean_sz, Tref0val); - rstd = std::vector(rstd_sz, Tref0val); + mean = std::vector(mean_sz, Tgpu0val); + rstd = std::vector(rstd_sz, Tgpu0val); outhost = std::vector(out_sz, Tref0val); meanhost = std::vector(mean_sz, Tref0val); rstdhost = std::vector(rstd_sz, Tref0val); diff --git a/driver/bn_driver.hpp b/driver/bn_driver.hpp index 4b94ac42d8..bfa1b91aef 100644 --- a/driver/bn_driver.hpp +++ b/driver/bn_driver.hpp @@ -33,12 +33,16 @@ #include "tensor_driver.hpp" #include "timer.hpp" #include "util_driver.hpp" +#include "rocrand_wrapper.hpp" #include "../test/verify.hpp" +#include "../test/random.hpp" +#include "../test/fusionHost.hpp" #include #include #include +#include "miopen/batch_norm.hpp" #include #include @@ -56,26 +60,21 @@ #define ERRTOL_FP32 1e-4 #define ERRTOL_FP16 0.5e-3 #define RMSTOL_FP32 1e-4 -#define RMSTOL_FP16 0.5e-3 +#define RMSTOL_FP16 2e-3 #define MIO_DRIVER_BN_REFERENCE_COMPUTE_3D_AS_2D 1 // Resolves issue #1974 //#define BN_RUNFOR_PROFILER -template +template class BatchNormDriver : public Driver { public: - BatchNormDriver() : Driver() - { - miopenCreateTensorDescriptor(&inputTensor); - miopenCreateTensorDescriptor(&outputTensor); - miopenCreateTensorDescriptor(&biasScaleTensor); - miopenCreateTensorDescriptor(&dxOutputTensor); - miopenCreateTensorDescriptor(&dyInputTensor); - - data_type = (sizeof(Tgpu) == 4) ? miopenFloat : miopenHalf; - } + BatchNormDriver() : Driver() { data_type = (sizeof(TInput) == 4) ? miopenFloat : miopenHalf; } int AddCmdLineArgs() override; int ParseCmdLineArgs(int argc, char* argv[]) override; @@ -99,25 +98,25 @@ class BatchNormDriver : public Driver void runGPUFwdTrain(Tref epsilon, Tref eAF, float alpha, float beta); void runGPUBwd(Tref epsilon, float alpha, float beta); - void runCPUFwdInference( - Tref epsilon, int batch_sz, int channels, int height, int width, int depth = 0); - void runCPUFwdTrain( - Tref epsilon, Tref eAF, int batch_sz, int channels, int height, int width, int depth = 0); + void runCPUFwdInference(Tref epsilon); + void runCPUFwdTrain(Tref epsilon, Tref eAF); int VerifyBackward() override; int VerifyForward() override; - ~BatchNormDriver() override - { - miopenDestroyTensorDescriptor(outputTensor); - miopenDestroyTensorDescriptor(inputTensor); - miopenDestroyTensorDescriptor(biasScaleTensor); - miopenDestroyTensorDescriptor(dxOutputTensor); - miopenDestroyTensorDescriptor(dyInputTensor); - } + // Helper function to check the Layout type short names + bool ChkLayout_ShortName(); + // function to validate the Layout type parameters. + // layout parameter value to std (NCHW/NHWC/NCDHW/NDHWC) values, + // defined in MIOpen lib. + void ValidateLayoutInputParameters(std::string layout_type); + + ~BatchNormDriver() override {} private: miopenBatchNormMode_t bn_mode; + miopenActivationMode_t activ_mode = miopenActivationRELU; + bool saveMeanVar; bool bsaveMeanVar; bool keepRunningMeanVar; @@ -126,67 +125,57 @@ class BatchNormDriver : public Driver int forw; int back; + bool isFwdInfer = false; + bool isFwdTrain = false; + bool isBwd = false; + InputFlags inflags; bool isDepthSpecified = false; - miopenTensorDescriptor_t inputTensor; - miopenTensorDescriptor_t biasScaleTensor; - miopenTensorDescriptor_t outputTensor; - - // Backwards - miopenTensorDescriptor_t dyInputTensor; - miopenTensorDescriptor_t dxOutputTensor; - - std::unique_ptr dyin_dev; // this is the output of fwd - std::unique_ptr in_dev; - std::unique_ptr out_dev; - std::unique_ptr scale_dev; - std::unique_ptr bias_dev; - - std::unique_ptr dxout_dev; - std::unique_ptr dscale_dev; - std::unique_ptr dbias_dev; - - std::unique_ptr runningMean_dev; - std::unique_ptr runningVariance_dev; - std::unique_ptr saveMean_dev; - std::unique_ptr saveInvVariance_dev; - - std::vector dyin; // output of forward - std::vector in; - std::vector out; - std::vector out_host; - std::vector dxout; - std::vector dxout_host; - - std::vector scale; - std::vector scale_host; - std::vector bias; - std::vector bias_host; - - std::vector dscale; - std::vector dscale_host; - std::vector dbias; - std::vector dbias_host; - - std::vector runningMean; - std::vector runningVariance; - std::vector runningMean_host; - std::vector runningVariance_host; - - std::vector saveMean; - std::vector saveInvVariance; - - std::vector saveMean_host; - std::vector saveInvVariance_host; - - int createSaveBuffers(); - int createRunningBuffers(); + GpumemTensor in; + GpumemTensor out; // for forward the output maches input type. + tensor out_ref; + + // forward + GpumemTensor scale; + GpumemTensor bias; + + // forward inference + GpumemTensor estMean; + GpumemTensor estVariance; + + GpumemTensor savedMean; + tensor savedMean_ref; + + // forward training + GpumemTensor savedVariance; + GpumemTensor runMean; + GpumemTensor runVariance; + // ref + tensor savedVariance_ref; + tensor runMean_ref; + tensor runVariance_ref; + + // backward needed different type for bwd. + GpumemTensor out_bwd; + + GpumemTensor bnScale; + GpumemTensor dScale; + GpumemTensor dBias; + // savedMean declared above as TAcc as well + GpumemTensor savedInvVar; + GpumemTensor dy; + + tensor dBias_ref; + tensor dScale_ref; + Tref maxval; + + miopenTensorLayout_t bn_layout; }; -template -int BatchNormDriver::ParseCmdLineArgs(int argc, char* argv[]) +template +int BatchNormDriver::ParseCmdLineArgs(int argc, char* argv[]) { inflags.Parse(argc, argv); @@ -198,56 +187,93 @@ int BatchNormDriver::ParseCmdLineArgs(int argc, char* argv[]) return miopenStatusSuccess; } -template -int BatchNormDriver::GetandSetData() +template +int BatchNormDriver::GetandSetData() { + std::vector in_len = GetInputTensorLengthsFromCmdLine(); SetBNParametersFromCmdLineArgs(); - std::vector in_len = GetInputTensorLengthsFromCmdLine(); + in.AllocOnHost(tensor{bn_layout, in_len}); + for(size_t i = 0; i < in.GetVector().size(); i++) + { + in.GetVector()[i] = prng::gen_canonical(); + } - std::vector sb_len; - if(bn_mode == miopenBNPerActivation) + auto derivedBnDesc = miopen::TensorDescriptor{}; + miopen::DeriveBNTensorDescriptor(derivedBnDesc, in.GetTensor().desc, bn_mode); + + if(isFwdInfer || isFwdTrain) { - // 1xCxHxW | in_len.size = 4 - sb_len.push_back(1); - sb_len.push_back(in_len[1]); - sb_len.push_back(in_len[2]); - sb_len.push_back(in_len[3]); + out.AllocOnHost(tensor{bn_layout, in_len}); + scale.AllocOnHost(tensor{bn_layout, derivedBnDesc.GetLengths()}); + bias.AllocOnHost(tensor{bn_layout, derivedBnDesc.GetLengths()}); - // 1xCxDxHxW | in_len.size = 5 - if(in_len.size() == 5) + for(int i = 0; i < scale.GetVector().size(); i++) { - sb_len.push_back(in_len[4]); + scale.GetVector()[i] = prng::gen_canonical(); + bias.GetVector()[i] = prng::gen_canonical(); } } - else if(bn_mode == miopenBNSpatial) - { // 1xCx1x1 - sb_len.push_back(1); - sb_len.push_back(in_len[1]); - sb_len.push_back(1); - sb_len.push_back(1); + if(isFwdInfer) + { + estMean.AllocOnHost(tensor{bn_layout, derivedBnDesc.GetLengths()}); + estVariance.AllocOnHost(tensor{bn_layout, derivedBnDesc.GetLengths()}); - // 1xCx1x1x1 - if(in_len.size() == 5) + auto gen_value_emean = [](auto...) { return prng::gen_descreet_unsigned(1e-2, 100); }; + estMean.InitHostData(estMean.GetTensor().desc.GetElementSize(), true, gen_value_emean); + } + else if(isFwdTrain) + { + savedMean.AllocOnHost(tensor{bn_layout, derivedBnDesc.GetLengths()}); + savedVariance.AllocOnHost(tensor{bn_layout, derivedBnDesc.GetLengths()}); + runMean.AllocOnHost(tensor{bn_layout, derivedBnDesc.GetLengths()}); + runVariance.AllocOnHost(tensor{bn_layout, derivedBnDesc.GetLengths()}); + + for(int i = 0; i < runVariance.GetVector().size(); i++) { - sb_len.push_back(1); + runMean.GetVector()[i] = prng::gen_canonical(); + runVariance.GetVector()[i] = prng::gen_canonical(); } } + else if(isBwd) + { + out_bwd.AllocOnHost(tensor{bn_layout, in_len}); - SetTensorNd(inputTensor, in_len, data_type); - SetTensorNd(biasScaleTensor, sb_len, ((sizeof(Tmix) == 4) ? miopenFloat : miopenHalf)); - SetTensorNd(outputTensor, in_len, data_type); + bnScale.AllocOnHost(tensor{bn_layout, derivedBnDesc.GetLengths()}); + dy.AllocOnHost(tensor{bn_layout, in_len}); - // backwards - SetTensorNd(dyInputTensor, in_len, data_type); - SetTensorNd(dxOutputTensor, in_len, data_type); + auto gen_var_bwd = [](auto...) { + return static_cast(1e-2 * (prng::gen_0_to_B(100) + 1)); + }; + + dy.InitHostData(dy.GetTensor().desc.GetElementSize(), true, gen_var_bwd); + + dScale.AllocOnHost(tensor{bn_layout, derivedBnDesc.GetLengths()}); + dBias.AllocOnHost(tensor{bn_layout, derivedBnDesc.GetLengths()}); + savedMean.AllocOnHost(tensor{bn_layout, derivedBnDesc.GetLengths()}); + savedInvVar.AllocOnHost(tensor{bn_layout, derivedBnDesc.GetLengths()}); + + auto gen_value = [](auto...) { return prng::gen_descreet_unsigned(1e-2, 100); }; + bnScale.InitHostData(bnScale.GetTensor().desc.GetElementSize(), true, gen_value); + + auto gen_in_var = [](auto...) { + return static_cast(1e-2 * (prng::gen_0_to_B(100) + 1)); + }; + savedMean.InitHostData(savedMean.GetTensor().desc.GetElementSize(), true, gen_in_var); + savedInvVar.InitHostData(savedInvVar.GetTensor().desc.GetElementSize(), true, gen_in_var); + } + else + { + std::cout << "\nUnknown batch norm state!\n"; + exit(EXIT_FAILURE); + } return miopenStatusSuccess; } -template -int BatchNormDriver::AddCmdLineArgs() +template +int BatchNormDriver::AddCmdLineArgs() { inflags.AddInputFlag( "forw", @@ -265,6 +291,10 @@ int BatchNormDriver::AddCmdLineArgs() inflags.AddInputFlag("in_h", 'H', "32", "Input Height (Default=32)", "int"); inflags.AddInputFlag("in_w", 'W', "32", "Input Width (Default=32)", "int"); inflags.AddInputFlag("in_d", 'D', "0", "Input Depth (Default=0)", "int"); + + inflags.AddInputFlag( + "layout", 'L', "", "Layout (Default=NCHW for 2d conv, NCDHW for 3d conv)", "string", true); + inflags.AddInputFlag("alpha", 'A', "1.0", "Alpha (Default=1.0)", "float"); inflags.AddInputFlag("beta", 'B', "0.", "Beta (Default=0.)", "float"); inflags.AddInputFlag("iter", 'i', "1", "Number of Iterations (Default=1)", "int"); @@ -294,8 +324,9 @@ int BatchNormDriver::AddCmdLineArgs() return miopenStatusSuccess; } -template -std::vector BatchNormDriver::GetInputTensorLengthsFromCmdLine() +template +std::vector +BatchNormDriver::GetInputTensorLengthsFromCmdLine() { int in_n = inflags.GetValueInt("batchsize"); int in_c = inflags.GetValueInt("in_channels"); @@ -317,13 +348,86 @@ std::vector BatchNormDriver::GetInputTensorLengthsFromCmd } } -template -int BatchNormDriver::SetBNParametersFromCmdLineArgs() +template +bool BatchNormDriver::ChkLayout_ShortName() +{ + // check for short name of layout type + if(inflags.FindShortName("layout") == 'L') + { + // do noting + // found valid short names + return true; + } + else + { + std::cerr << "Error:Invalid Short Name for layout!" << std::endl; + exit(EXIT_FAILURE); + } +} + +template +void BatchNormDriver::ValidateLayoutInputParameters( + std::string layout_value) +{ + if(!ChkLayout_ShortName()) + { + std::cerr << "Invalid Layout Short Name = " << inflags.FindShortName("layout") << std::endl; + exit(EXIT_FAILURE); + } + if((layout_value.compare("NCHW") != 0) && (layout_value.compare("NHWC") != 0) && + (layout_value.compare("NCDHW") != 0) && (layout_value.compare("NDHWC") != 0)) + { + std::cerr << "Invalid Layout Parameter Value - " << layout_value << std::endl; + exit(EXIT_FAILURE); + } +} + +template +int BatchNormDriver::SetBNParametersFromCmdLineArgs() { // double bnAlpha = inflags.GetValueDouble("alpha"); // double bnBeta = inflags.GetValueDouble("beta"); + const std::string default_layout = isDepthSpecified ? "NCDHW" : "NCHW"; + + // inflags value is empty, default value is used + // if it is supplied via cmd line, check the value. + if(inflags.GetValueStr("layout").empty()) + { + inflags.SetValue("layout", default_layout); + } + else + { + std::string layoutValue = inflags.GetValueStr("layout"); + ValidateLayoutInputParameters(layoutValue); + inflags.SetValue("layout", layoutValue); + } + + std::string layout = inflags.GetValueStr("layout"); + + if(layout == "NCHW") + { + bn_layout = miopenTensorNCHW; + } + else if(layout == "NHWC") + { + bn_layout = miopenTensorNHWC; + } + else if(layout == "NCDHW") + { + bn_layout = miopenTensorNCDHW; + } + else if(layout == "NDHWC") + { + bn_layout = miopenTensorNDHWC; + } + else + { + std::cout << "Cannot handle layout : " << layout << "\n"; + exit(EXIT_FAILURE); // NOLINT (concurrency-mt-unsafe) + } + // batch norm mode type if(inflags.GetValueInt("mode") == 0) { @@ -395,242 +499,100 @@ int BatchNormDriver::SetBNParametersFromCmdLineArgs() forw = 1; } - return miopenStatusSuccess; -} - -template -int BatchNormDriver::createSaveBuffers() -{ - - status_t status = STATUS_SUCCESS; - DEFINE_CONTEXT(ctx); -#if MIOPEN_BACKEND_OPENCL - clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); -#endif - - size_t sb_sz = GetTensorSize(biasScaleTensor); - - if(saveMeanVar) + if(forw == 1) { - // GPU allocation - saveMean_dev = std::unique_ptr(new GPUMem(ctx, sb_sz, sizeof(Tmix))); - saveInvVariance_dev = std::unique_ptr(new GPUMem(ctx, sb_sz, sizeof(Tmix))); - - if(back == 1) - { - // GPU host allocation - saveMean = std::vector(sb_sz, static_cast(0)); - saveInvVariance = std::vector(sb_sz, static_cast(0)); - - // CPU allocation - saveMean_host = std::vector(sb_sz, static_cast(0)); - saveInvVariance_host = std::vector(sb_sz, static_cast(0)); - - // Populate - for(int i = 0; i < sb_sz; i++) - { - saveMean[i] = prng::gen_canonical(); - saveMean_host[i] = static_cast(saveMean[i]); - saveInvVariance[i] = prng::gen_canonical(); - saveInvVariance_host[i] = static_cast(saveInvVariance[i]); - } - } - else - { - // GPU host allocation - saveMean = std::vector(sb_sz, static_cast(0)); - saveInvVariance = std::vector(sb_sz, static_cast(0)); - - // CPU allocation - saveMean_host = std::vector(sb_sz, static_cast(0)); - saveInvVariance_host = std::vector(sb_sz, static_cast(0)); - } - // GPU data transfer - status |= saveMean_dev->ToGPU(q, saveMean.data()); - status |= saveInvVariance_dev->ToGPU(q, saveInvVariance.data()); + isFwdTrain = true; + } + else if(forw == 2) + { + isFwdInfer = true; } else { - saveMean_dev = nullptr; - saveInvVariance_dev = nullptr; + isBwd = true; } - if(status != STATUS_SUCCESS) - printf("Error copying data to GPU\n"); - return miopenStatusSuccess; } -template -int BatchNormDriver::createRunningBuffers() +template +int BatchNormDriver::AllocateBuffersAndCopy() { status_t status = STATUS_SUCCESS; DEFINE_CONTEXT(ctx); #if MIOPEN_BACKEND_OPENCL clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); #endif - size_t sb_sz = GetTensorSize(biasScaleTensor); + status |= in.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&in.GetTensor().desc)); - if(keepRunningMeanVar) + if(isFwdInfer || isFwdTrain) { - // GPU allocation - runningMean_dev = std::unique_ptr(new GPUMem(ctx, sb_sz, sizeof(Tmix))); - runningVariance_dev = std::unique_ptr(new GPUMem(ctx, sb_sz, sizeof(Tmix))); - - if(forw == 2) - { - // GPU host allocation - runningMean = std::vector(sb_sz, static_cast(0)); - runningVariance = std::vector(sb_sz, static_cast(0)); - - // CPU allocation - runningMean_host = std::vector(sb_sz, static_cast(0)); - runningVariance_host = std::vector(sb_sz, static_cast(0)); - - // Populate - for(int i = 0; i < sb_sz; i++) - { - runningMean[i] = prng::gen_canonical(); - runningMean_host[i] = static_cast(runningMean[i]); - runningVariance[i] = prng::gen_canonical(); - runningVariance_host[i] = static_cast(runningVariance[i]); - } - } - else - { - // GPU host allocation - runningMean = std::vector(sb_sz, static_cast(0)); - runningVariance = std::vector(sb_sz, static_cast(0)); - - // CPU allocation - runningMean_host = std::vector(sb_sz, static_cast(0)); - runningVariance_host = std::vector(sb_sz, static_cast(0)); - } - - // GPU data transfer - status |= runningMean_dev->ToGPU(q, runningMean.data()); - status |= runningVariance_dev->ToGPU(q, runningVariance.data()); + status |= out.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&out.GetTensor().desc)); + out_ref = + tensor{out.GetTensor().desc.GetLayout_t(), out.GetTensor().desc.GetLengths()}; + status |= scale.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&scale.GetTensor().desc)); + status |= bias.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&bias.GetTensor().desc)); } - else + if(isFwdInfer) { - runningMean_dev = nullptr; - runningVariance_dev = nullptr; + status |= estMean.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&estMean.GetTensor().desc)); + status |= + estVariance.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&estVariance.GetTensor().desc)); } - if(status != STATUS_SUCCESS) - printf("Error copying data to GPU\n"); - - return miopenStatusSuccess; -} - -template -int BatchNormDriver::AllocateBuffersAndCopy() -{ - status_t status = STATUS_SUCCESS; - DEFINE_CONTEXT(ctx); -#if MIOPEN_BACKEND_OPENCL - clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); -#endif - - size_t in_sz = GetTensorSize(inputTensor); - size_t sb_sz = GetTensorSize(biasScaleTensor); - - if(forw) + if(isFwdTrain) { + status |= + savedMean.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&savedMean.GetTensor().desc)); + status |= savedVariance.AllocOnDeviceAndInit( + q, ctx, GetTensorSize(&savedVariance.GetTensor().desc)); + status |= runMean.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&runMean.GetTensor().desc)); + status |= + runVariance.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&runVariance.GetTensor().desc)); - size_t out_sz = GetTensorSize(outputTensor); - - // GPU allocation - in_dev = std::unique_ptr(new GPUMem(ctx, in_sz, sizeof(Tgpu))); - scale_dev = std::unique_ptr(new GPUMem(ctx, sb_sz, sizeof(Tmix))); - bias_dev = std::unique_ptr(new GPUMem(ctx, sb_sz, sizeof(Tmix))); - out_dev = std::unique_ptr(new GPUMem(ctx, out_sz, sizeof(Tgpu))); - - // GPU host allocation - in = std::vector(in_sz, static_cast(0)); - out = std::vector(out_sz, static_cast(0)); - scale = std::vector(sb_sz, static_cast(0)); - bias = std::vector(sb_sz, static_cast(0)); + savedMean_ref = tensor{savedMean.GetTensor().desc.GetLayout_t(), + savedMean.GetTensor().desc.GetLengths()}; - // CPU allocation - out_host = std::vector(out_sz, static_cast(0)); - scale_host = std::vector(sb_sz, static_cast(0)); - bias_host = std::vector(sb_sz, static_cast(0)); + savedVariance_ref = tensor{savedVariance.GetTensor().desc.GetLayout_t(), + savedVariance.GetTensor().desc.GetLengths()}; - // Data initialization - for(int i = 0; i < in_sz; i++) - { - in[i] = prng::gen_canonical(); - } - status |= in_dev->ToGPU(q, in.data()); - - // Using random beta and gamma - for(int i = 0; i < sb_sz; i++) - { - scale[i] = prng::gen_canonical(); - scale_host[i] = static_cast(scale[i]); - bias[i] = prng::gen_canonical(); - bias_host[i] = static_cast(bias[i]); - } - status |= scale_dev->ToGPU(q, scale.data()); - status |= bias_dev->ToGPU(q, bias.data()); - status |= out_dev->ToGPU(q, out.data()); + runMean_ref = tensor{runMean.GetTensor().desc.GetLayout_t(), + runMean.GetTensor().desc.GetLengths()}; - if(forw == 1) - { // training - status |= createRunningBuffers(); - status |= createSaveBuffers(); - } - else if(forw == 2) - { // inference - status |= createRunningBuffers(); - } - } // end forward - - if(back == 1) + runVariance_ref = tensor{runVariance.GetTensor().desc.GetLayout_t(), + runVariance.GetTensor().desc.GetLengths()}; + } + if(isBwd) { + status |= out_bwd.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&out_bwd.GetTensor().desc)); - size_t out_sz = GetTensorSize(dxOutputTensor); + out_ref = tensor{out_bwd.GetTensor().desc.GetLayout_t(), + out_bwd.GetTensor().desc.GetLengths()}; - // GPU allocation - in_dev = std::unique_ptr(new GPUMem(ctx, in_sz, sizeof(Tgpu))); - dyin_dev = std::unique_ptr(new GPUMem(ctx, in_sz, sizeof(Tgpu))); - dxout_dev = std::unique_ptr(new GPUMem(ctx, out_sz, sizeof(Tgpu))); - dscale_dev = std::unique_ptr(new GPUMem(ctx, sb_sz, sizeof(Tmix))); - dbias_dev = std::unique_ptr(new GPUMem(ctx, sb_sz, sizeof(Tmix))); - scale_dev = std::unique_ptr(new GPUMem(ctx, sb_sz, sizeof(Tmix))); + status |= bnScale.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&bnScale.GetTensor().desc)); + status |= dy.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&dy.GetTensor().desc)); - // GPU host allocation - in = std::vector(in_sz, static_cast(0)); - dyin = std::vector(in_sz, static_cast(0)); - dxout = std::vector(out_sz, static_cast(0)); - dscale = std::vector(sb_sz, static_cast(0)); - dbias = std::vector(sb_sz, static_cast(0)); - scale = std::vector(sb_sz, static_cast(0)); + status |= dScale.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&dScale.GetTensor().desc)); + status |= dBias.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&dBias.GetTensor().desc)); + status |= + savedMean.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&savedMean.GetTensor().desc)); + status |= + savedInvVar.AllocOnDeviceAndInit(q, ctx, GetTensorSize(&savedInvVar.GetTensor().desc)); - // CPU allocation - dxout_host = std::vector(out_sz, static_cast(0)); - dscale_host = std::vector(sb_sz, static_cast(0)); - dbias_host = std::vector(sb_sz, static_cast(0)); + dScale_ref = tensor{dScale.GetTensor().desc.GetLayout_t(), + dScale.GetTensor().desc.GetLengths()}; - // Populate - for(int i = 0; i < sb_sz; i++) - { - scale[i] = prng::gen_canonical(); - } - status |= scale_dev->ToGPU(q, scale.data()); - status |= dscale_dev->ToGPU(q, dscale.data()); - status |= dbias_dev->ToGPU(q, dbias.data()); + dBias_ref = + tensor{dBias.GetTensor().desc.GetLayout_t(), dBias.GetTensor().desc.GetLengths()}; + } - for(int i = 0; i < in_sz; i++) - { - dyin[i] = prng::gen_canonical(); - in[i] = prng::gen_canonical(); - } - status |= dyin_dev->ToGPU(q, dyin.data()); - status |= in_dev->ToGPU(q, in.data()); - status |= dxout_dev->ToGPU(q, dxout.data()); + for(size_t i = 0; i < runMean.GetVector().size(); ++i) + { + runMean_ref.data[i] = static_cast(runMean.GetVector()[i]); + } - status |= createSaveBuffers(); + for(size_t i = 0; i < runVariance.GetVector().size(); ++i) + { + runVariance_ref.data[i] = static_cast(runVariance.GetVector()[i]); } if(status != STATUS_SUCCESS) @@ -639,158 +601,181 @@ int BatchNormDriver::AllocateBuffersAndCopy() return miopenStatusSuccess; } -template -void BatchNormDriver::runGPUFwdInference(Tref epsilon, float alpha, float beta) +template +void BatchNormDriver::runGPUFwdInference(Tref epsilon, + float alpha, + float beta) { if(keepRunningMeanVar) { // use precalculated mean and variance - miopenBatchNormalizationForwardInference(GetHandle(), - bn_mode, - &alpha, - &beta, - inputTensor, - in_dev->GetMem(), - outputTensor, - out_dev->GetMem(), - biasScaleTensor, - scale_dev->GetMem(), - bias_dev->GetMem(), - runningMean_dev->GetMem(), - runningVariance_dev->GetMem(), - epsilon); + miopenBatchNormalizationForwardInference_V2(GetHandle(), + bn_mode, + &alpha, + &beta, + &in.GetTensor().desc, + in.GetDevicePtr(), + &out.GetTensor().desc, + out.GetDevicePtr(), + &scale.GetTensor().desc, + &bias.GetTensor().desc, + &estMean.GetTensor().desc, + &estVariance.GetTensor().desc, + scale.GetDevicePtr(), + bias.GetDevicePtr(), + estMean.GetDevicePtr(), + estVariance.GetDevicePtr(), + epsilon); } else { // recalculate mean and variance - miopenBatchNormalizationForwardInference(GetHandle(), - bn_mode, - &alpha, - &beta, - inputTensor, - in_dev->GetMem(), - outputTensor, - out_dev->GetMem(), - biasScaleTensor, - scale_dev->GetMem(), - bias_dev->GetMem(), - nullptr, - nullptr, - epsilon); + miopenBatchNormalizationForwardInference_V2(GetHandle(), + bn_mode, + &alpha, + &beta, + &in.GetTensor().desc, + in.GetDevicePtr(), + &out.GetTensor().desc, + out.GetDevicePtr(), + &scale.GetTensor().desc, + &bias.GetTensor().desc, + &estMean.GetTensor().desc, + &estVariance.GetTensor().desc, + scale.GetDevicePtr(), + bias.GetDevicePtr(), + nullptr, + nullptr, + epsilon); } return; } -template -void BatchNormDriver::runGPUFwdTrain(Tref epsilon, - Tref eAF, - float alpha, - float beta) +template +void BatchNormDriver::runGPUFwdTrain(Tref epsilon, + Tref eAF, + float alpha, + float beta) { if(saveMeanVar && keepRunningMeanVar) { - miopenBatchNormalizationForwardTraining(GetHandle(), - bn_mode, - &alpha, - &beta, - inputTensor, - in_dev->GetMem(), - outputTensor, - out_dev->GetMem(), - biasScaleTensor, - scale_dev->GetMem(), - bias_dev->GetMem(), - eAF, - runningMean_dev->GetMem(), - runningVariance_dev->GetMem(), - epsilon, - saveMean_dev->GetMem(), - saveInvVariance_dev->GetMem()); + miopenBatchNormalizationForwardTraining_V2(GetHandle(), + bn_mode, + &alpha, + &beta, + &in.GetTensor().desc, + in.GetDevicePtr(), + &out.GetTensor().desc, + out.GetDevicePtr(), + &scale.GetTensor().desc, + &bias.GetTensor().desc, + &savedMean.GetTensor().desc, + &savedVariance.GetTensor().desc, + scale.GetDevicePtr(), + bias.GetDevicePtr(), + eAF, + runMean.GetDevicePtr(), + runVariance.GetDevicePtr(), + epsilon, + savedMean.GetDevicePtr(), + savedVariance.GetDevicePtr()); } else if(saveMeanVar) { - miopenBatchNormalizationForwardTraining(GetHandle(), - bn_mode, - &alpha, - &beta, - inputTensor, - in_dev->GetMem(), - outputTensor, - out_dev->GetMem(), - biasScaleTensor, - scale_dev->GetMem(), - bias_dev->GetMem(), - eAF, - nullptr, - nullptr, - epsilon, - saveMean_dev->GetMem(), - saveInvVariance_dev->GetMem()); + miopenBatchNormalizationForwardTraining_V2(GetHandle(), + bn_mode, + &alpha, + &beta, + &in.GetTensor().desc, + in.GetDevicePtr(), + &out.GetTensor().desc, + out.GetDevicePtr(), + &scale.GetTensor().desc, + &bias.GetTensor().desc, + &savedMean.GetTensor().desc, + &savedVariance.GetTensor().desc, + scale.GetDevicePtr(), + bias.GetDevicePtr(), + eAF, + nullptr, + nullptr, + epsilon, + savedMean.GetDevicePtr(), + savedVariance.GetDevicePtr()); } else if(keepRunningMeanVar) { - miopenBatchNormalizationForwardTraining(GetHandle(), - bn_mode, - &alpha, - &beta, - inputTensor, - in_dev->GetMem(), - outputTensor, - out_dev->GetMem(), - biasScaleTensor, - scale_dev->GetMem(), - bias_dev->GetMem(), - eAF, - runningMean_dev->GetMem(), - runningVariance_dev->GetMem(), - epsilon, - nullptr, - nullptr); + miopenBatchNormalizationForwardTraining_V2(GetHandle(), + bn_mode, + &alpha, + &beta, + &in.GetTensor().desc, + in.GetDevicePtr(), + &out.GetTensor().desc, + out.GetDevicePtr(), + &scale.GetTensor().desc, + &bias.GetTensor().desc, + &savedMean.GetTensor().desc, + &savedVariance.GetTensor().desc, + scale.GetDevicePtr(), + bias.GetDevicePtr(), + eAF, + runMean.GetDevicePtr(), + runVariance.GetDevicePtr(), + epsilon, + nullptr, + nullptr); } else { - miopenBatchNormalizationForwardTraining(GetHandle(), - bn_mode, - &alpha, - &beta, - inputTensor, - in_dev->GetMem(), - outputTensor, - out_dev->GetMem(), - biasScaleTensor, - scale_dev->GetMem(), - bias_dev->GetMem(), - eAF, - nullptr, - nullptr, - epsilon, - nullptr, - nullptr); + miopenBatchNormalizationForwardTraining_V2(GetHandle(), + bn_mode, + &alpha, + &beta, + &in.GetTensor().desc, + in.GetDevicePtr(), + &out.GetTensor().desc, + out.GetDevicePtr(), + &scale.GetTensor().desc, + &bias.GetTensor().desc, + &savedMean.GetTensor().desc, + &savedVariance.GetTensor().desc, + scale.GetDevicePtr(), + bias.GetDevicePtr(), + eAF, + nullptr, + nullptr, + epsilon, + nullptr, + nullptr); } #ifdef BN_RUNFOR_PROFILER - miopenBatchNormalizationForwardTraining(GetHandle(), - bn_mode, - &alpha, - &beta, - inputTensor, - in_dev->GetMem(), - outputTensor, - out_dev->GetMem(), - biasScaleTensor, - scale_dev->GetMem(), - bias_dev->GetMem(), - eAF, - nullptr, - nullptr, - epsilon, - nullptr, - nullptr); + miopenBatchNormalizationForwardTraining_V2(GetHandle(), + bn_mode, + &alpha, + &beta, + &in.GetTensor().desc, + in.GetDevicePtr(), + &out.GetTensor().desc, + out.GetDevicePtr(), + &scale.GetTensor().desc, + &bias.GetTensor().desc, + &savedMean.GetTensor().desc, + &savedVariance.GetTensor().desc, + scale.GetDevicePtr(), + bias.GetDevicePtr(), + eAF, + nullptr, + nullptr, + epsilon, + nullptr, + nullptr); #endif } -template -int BatchNormDriver::RunForwardGPU() +template +int BatchNormDriver::RunForwardGPU() { float alpha = static_cast(1), beta = static_cast(0); @@ -866,9 +851,9 @@ int BatchNormDriver::RunForwardGPU() avgtime / (iters - 1), iters - 1); int in_n, in_c, in_h, in_w; - std::tie(in_n, in_c, in_h, in_w) = miopen::tien<4>(miopen::deref(inputTensor).GetLengths()); + std::tie(in_n, in_c, in_h, in_w) = miopen::tien<4>(in.GetTensor().desc.GetLengths()); size_t M = in_n * in_c * in_h * in_w; - size_t dataSz = (M + 2 * in_c) * miopen::GetTypeSize(miopen::deref(inputTensor).GetType()); + size_t dataSz = (M + 2 * in_c) * miopen::GetTypeSize(in.GetTensor().desc.GetType()); float rdCnt = -1.0; float wrCnt = 1.0; if(forw == 1) @@ -889,42 +874,45 @@ int BatchNormDriver::RunForwardGPU() return miopenStatusSuccess; } -template -void BatchNormDriver::runCPUFwdInference( - Tref epsilon, int batch_sz, int channels, int height, int width, int depth) +template +void BatchNormDriver::runCPUFwdInference(Tref epsilon) { + int size{0}; + miopenGetTensorDescriptorSize(&in.GetTensor().desc, &size); + + if(size == 5) + { + in.GetTensor().desc = miopen::BuildReshaped4DTensorDescriptor(in.GetTensor().desc); + out_ref.desc = miopen::BuildReshaped4DTensorDescriptor(out_ref.desc); + scale.GetTensor().desc = miopen::BuildReshaped4DTensorDescriptor(scale.GetTensor().desc); + bias.GetTensor().desc = miopen::BuildReshaped4DTensorDescriptor(bias.GetTensor().desc); + estMean.GetTensor().desc = + miopen::BuildReshaped4DTensorDescriptor(estMean.GetTensor().desc); + estVariance.GetTensor().desc = + miopen::BuildReshaped4DTensorDescriptor(estVariance.GetTensor().desc); + } if(bn_mode == miopenBNPerActivation) { // 1xCxHxW - miopenBNFwdInferPerActivationRunHost(/* alpha, beta, */ batch_sz, - channels, - (isDepthSpecified ? depth : 1), - height, - width, - in.data(), - out_host.data(), - scale_host.data(), - bias_host.data(), - epsilon, - keepRunningMeanVar, - runningMean_host.data(), - runningVariance_host.data()); + // handle 3d case + batchNormPerActivHostInference(in.GetTensor(), + out_ref, + scale.GetTensor(), + bias.GetTensor(), + epsilon, + estMean.GetTensor(), + estVariance.GetTensor()); } else if(bn_mode == miopenBNSpatial) { // 1xCx1x1 - miopenBNFwdInferSpatialRunHost(/* alpha, beta, */ batch_sz, - channels, - (isDepthSpecified ? depth : 1), - height, - width, - in.data(), - out_host.data(), - scale_host.data(), - bias_host.data(), - epsilon, - keepRunningMeanVar, - runningMean_host.data(), - runningVariance_host.data()); + + batchNormSpatialHostInference(in.GetTensor(), + out_ref, + scale.GetTensor(), + bias.GetTensor(), + epsilon, + estMean.GetTensor(), + estVariance.GetTensor()); } else { @@ -935,60 +923,66 @@ void BatchNormDriver::runCPUFwdInference( return; } -template -void BatchNormDriver::runCPUFwdTrain( - Tref epsilon, Tref eAF, int batch_sz, int channels, int height, int width, int depth) +template +void BatchNormDriver::runCPUFwdTrain(Tref epsilon, Tref eAF) { - + int size{0}; + miopenGetTensorDescriptorSize(&in.GetTensor().desc, &size); + if(size == 5) + { + in.GetTensor().desc = miopen::BuildReshaped4DTensorDescriptor(in.GetTensor().desc); + out_ref.desc = miopen::BuildReshaped4DTensorDescriptor(out_ref.desc); + scale.GetTensor().desc = miopen::BuildReshaped4DTensorDescriptor(scale.GetTensor().desc); + bias.GetTensor().desc = miopen::BuildReshaped4DTensorDescriptor(bias.GetTensor().desc); + savedMean_ref.desc = miopen::BuildReshaped4DTensorDescriptor(savedMean_ref.desc); + savedVariance_ref.desc = miopen::BuildReshaped4DTensorDescriptor(savedVariance_ref.desc); + runMean_ref.desc = miopen::BuildReshaped4DTensorDescriptor(runMean_ref.desc); + runVariance_ref.desc = miopen::BuildReshaped4DTensorDescriptor(runVariance_ref.desc); + } if(bn_mode == miopenBNPerActivation) { // 1xCxHxW - miopenBNFwdTrainPerActivationRunHost(/* alpha, beta, */ batch_sz, - channels, -#if MIO_DRIVER_BN_REFERENCE_COMPUTE_3D_AS_2D - 1, - height * (isDepthSpecified ? depth : 1), -#else - (isDepthSpecified ? depth : 1), - height, -#endif - width, - in.data(), - out_host.data(), - scale_host.data(), - bias_host.data(), - epsilon, - saveMeanVar, - keepRunningMeanVar, - saveMean_host.data(), - saveInvVariance_host.data(), - runningMean_host.data(), - runningVariance_host.data(), - eAF); + + batchNormPerActHostFwdTrain(in.GetTensor(), + out_ref, + scale.GetTensor(), + bias.GetTensor(), + static_cast(epsilon), + static_cast(eAF), + savedMean_ref, + savedVariance_ref, + runMean_ref, + runVariance_ref); } else if(bn_mode == miopenBNSpatial) { // 1xCx1x1 - miopenBNFwdTrainSpatialRunHost(/* alpha, beta, */ batch_sz, - channels, -#if MIO_DRIVER_BN_REFERENCE_COMPUTE_3D_AS_2D - 1, - height * (isDepthSpecified ? depth : 1), -#else - (isDepthSpecified ? depth : 1), - height, -#endif - width, - in.data(), - out_host.data(), - scale_host.data(), - bias_host.data(), - epsilon, - saveMeanVar, - keepRunningMeanVar, - saveMean_host.data(), - saveInvVariance_host.data(), - runningMean_host.data(), - runningVariance_host.data(), - eAF); + + if(forw == 2 && !keepRunningMeanVar) + { + tensor empty_tensor; + batchNormSpatialHostFwdTrain(in.GetTensor(), + out_ref, + scale.GetTensor(), + bias.GetTensor(), + static_cast(epsilon), + static_cast(eAF), + empty_tensor, // savedMean_ref + empty_tensor, // savedVariance_ref + empty_tensor, // runMean_ref + empty_tensor); // runVariance_ref + } + else + { + batchNormSpatialHostFwdTrain(in.GetTensor(), + out_ref, + scale.GetTensor(), + bias.GetTensor(), + static_cast(epsilon), + static_cast(eAF), + savedMean_ref, + savedVariance_ref, + runMean_ref, + runVariance_ref); + } } else { @@ -998,47 +992,38 @@ void BatchNormDriver::runCPUFwdTrain( } } -template -int BatchNormDriver::RunForwardCPU() +template +int BatchNormDriver::RunForwardCPU() { - int nIn = 0, cIn = 0, dIn = 0, hIn = 0, wIn = 0; - - if(isDepthSpecified) - miopenGet5dTensorDescriptorLengths(inputTensor, &nIn, &cIn, &dIn, &hIn, &wIn); - else - miopenGet4dTensorDescriptorLengths(inputTensor, &nIn, &cIn, &hIn, &wIn); - - int batch_sz = nIn; - int channels = cIn; - int height = hIn; - int width = wIn; - int depth = dIn; - // T alpha = 0., beta = 0.; Tref epsilon = static_cast(EPSILON); Tref eAF = static_cast(1.0); - if(forw == 1) + if(forw == 1 || (forw == 2 && !keepRunningMeanVar)) { // training only for(int i = 0; i < inflags.GetValueInt("iter"); i++) { eAF = static_cast(1.0) / (static_cast(i) + static_cast(1.0)); - runCPUFwdTrain( - epsilon, eAF, /* alpha, beta,*/ batch_sz, channels, height, width, depth); + runCPUFwdTrain(epsilon, eAF /* alpha, beta,*/); } } - else if(forw == 2) - { // inference only - runCPUFwdInference(epsilon, /* alpha, beta,*/ batch_sz, channels, height, width, depth); + else if(forw == 2 && keepRunningMeanVar) + { + // inference only + runCPUFwdInference(epsilon); + } + else + { + printf("Unsupported forward cpu run state.\nExiting...\n\n"); + exit(EXIT_FAILURE); // NOLINT (concurrency-mt-unsafe) } return miopenStatusSuccess; } -template -int BatchNormDriver::RunBackwardGPU() +template +int BatchNormDriver::RunBackwardGPU() { - if(!back) return miopenStatusSuccess; @@ -1058,47 +1043,53 @@ int BatchNormDriver::RunBackwardGPU() if(saveMeanVar) { - miopenBatchNormalizationBackward(GetHandle(), - bn_mode, - &alphaDataDiff, - &betaDataDiff, - &alphaParamDiff, - &betaParamDiff, - inputTensor, - in_dev->GetMem(), - dyInputTensor, - dyin_dev->GetMem(), - dxOutputTensor, - dxout_dev->GetMem(), - biasScaleTensor, - scale_dev->GetMem(), - dscale_dev->GetMem(), - dbias_dev->GetMem(), - epsilon, - saveMean_dev->GetMem(), - saveInvVariance_dev->GetMem()); + miopenBatchNormalizationBackward_V2(GetHandle(), + bn_mode, + &alphaDataDiff, + &betaDataDiff, + &alphaParamDiff, + &betaParamDiff, + &in.GetTensor().desc, + in.GetDevicePtr(), + &dy.GetTensor().desc, + dy.GetDevicePtr(), + &out_bwd.GetTensor().desc, + out_bwd.GetDevicePtr(), + &bnScale.GetTensor().desc, + &dBias.GetTensor().desc, + &savedMean.GetTensor().desc, + &savedInvVar.GetTensor().desc, + bnScale.GetDevicePtr(), + dScale.GetDevicePtr(), + dBias.GetDevicePtr(), + epsilon, + savedMean.GetDevicePtr(), + savedInvVar.GetDevicePtr()); } else { - miopenBatchNormalizationBackward(GetHandle(), - bn_mode, - &alphaDataDiff, - &betaDataDiff, - &alphaParamDiff, - &betaParamDiff, - inputTensor, - in_dev->GetMem(), - dyInputTensor, - dyin_dev->GetMem(), - dxOutputTensor, - dxout_dev->GetMem(), - biasScaleTensor, - scale_dev->GetMem(), - dscale_dev->GetMem(), - dbias_dev->GetMem(), - epsilon, - nullptr, - nullptr); + miopenBatchNormalizationBackward_V2(GetHandle(), + bn_mode, + &alphaDataDiff, + &betaDataDiff, + &alphaParamDiff, + &betaParamDiff, + &in.GetTensor().desc, + in.GetDevicePtr(), + &dy.GetTensor().desc, + dy.GetDevicePtr(), + &out_bwd.GetTensor().desc, + out_bwd.GetDevicePtr(), + &bnScale.GetTensor().desc, + &dBias.GetTensor().desc, + &savedMean.GetTensor().desc, + &savedInvVar.GetTensor().desc, + bnScale.GetDevicePtr(), + dScale.GetDevicePtr(), + dBias.GetDevicePtr(), + epsilon, + nullptr, + nullptr); } miopen::deref(GetHandle()).Finish(); @@ -1120,13 +1111,11 @@ int BatchNormDriver::RunBackwardGPU() avgtime += time; int in_n, in_c, in_h, in_w; - std::tie(in_n, in_c, in_h, in_w) = - miopen::tien<4>(miopen::deref(inputTensor).GetLengths()); - size_t M = in_n * in_c * in_h * in_w; - size_t dataSz = - (M + 2 * in_c) * miopen::GetTypeSize(miopen::deref(inputTensor).GetType()); - float rdCnt = 2.0; - float wrCnt = 1.0; + std::tie(in_n, in_c, in_h, in_w) = miopen::tien<4>(in.GetTensor().desc.GetLengths()); + size_t M = in_n * in_c * in_h * in_w; + size_t dataSz = (M + 2 * in_c) * miopen::GetTypeSize(in.GetTensor().desc.GetType()); + float rdCnt = 2.0; + float wrCnt = 1.0; // layer, flopCnt, reads, writes, GFLOPS, GB/s, timeMs printf("stats: bnormb, 0, %zu, %zu, 0, %f, %f\n", dataSz, @@ -1152,18 +1141,18 @@ int BatchNormDriver::RunBackwardGPU() return miopenStatusSuccess; } -template -int BatchNormDriver::VerifyForward() +template +int BatchNormDriver::VerifyForward() { // jump out since we are forcing forward off when doing backwards. if(!forw) return miopenStatusSuccess; - const Tref maxrms = static_cast((sizeof(Tgpu) == 4) ? RMSTOL_FP32 : RMSTOL_FP16); + const Tref maxrms = static_cast((sizeof(TInput) == 4) ? RMSTOL_FP32 : RMSTOL_FP16); #if(MIO_BN_DEBUG == 1) - const Tref tolerance = static_cast((sizeof(Tgpu) == 4) ? ERRTOL_FP32 : ERRTOL_FP16); + const Tref tolerance = static_cast((sizeof(TInput) == 4) ? ERRTOL_FP32 : ERRTOL_FP16); Tref diff = static_cast(0.); #endif @@ -1176,27 +1165,28 @@ int BatchNormDriver::VerifyForward() if(keepRunningMeanVar) { // copy back for verification - runningMean_dev->FromGPU(GetStream(), runningMean.data()); - runningVariance_dev->FromGPU(GetStream(), runningVariance.data()); + runMean.CopyFromDeviceToHost(GetStream()); + runVariance.CopyFromDeviceToHost(GetStream()); + + auto errorRunMean = miopen::rms_range(runMean_ref.data, runMean.GetVector()); - auto errorRunMean = miopen::rms_range(runningMean_host, runningMean); if(!std::isfinite(errorRunMean) || errorRunMean > maxrms) { std::cout << "Forward train batch norm verification FAILED on running mean: " << errorRunMean << std::endl; anError = true; #if(MIO_BN_DEBUG == 1) - for(int i = 0; i < runningMean.size() && i < runningMean_host.size() && + for(int i = 0; i < runMean.GetVector().size() && i < runMean_ref.data.size() && i < MIO_BN_MAX_DEBUGLOOP; i++) { - diff = fabs(Tmix(fabs(runningMean[i]) - fabs(runningMean_host[i]))); + diff = fabs(TAcc(fabs(runMean.GetVector()[i]) - fabs(runMean_ref.data[i]))); if(!std::isfinite(diff) || diff > tolerance) { - std::cout << "rm[" << i << "]: " << runningMean[i]; - std::cout << ", rm_host[" << i << "]: " << runningMean_host[i]; - std::cout << ", diff[" << i - << "]: " << Tmix(fabs(runningMean[i]) - fabs(runningMean_host[i])) + std::cout << "rm[" << i << "]: " << runMean.GetVector()[i]; + std::cout << ", rm_host[" << i << "]: " << runMean_ref.data[i]; + std::cout << ", diff[" << i << "]: " + << TAcc(fabs(runMean.GetVector()[i]) - fabs(runMean_ref.data[i])) << std::endl; } } @@ -1208,24 +1198,26 @@ int BatchNormDriver::VerifyForward() << errorRunMean << ')' << std::endl; } - auto errorRunVar = miopen::rms_range(runningVariance_host, runningVariance); + auto errorRunVar = miopen::rms_range(runVariance_ref.data, runVariance.GetVector()); if(!std::isfinite(errorRunVar) || errorRunVar > maxrms) { std::cout << "Forward train batch norm verification FAILED on running variance: " << errorRunVar << std::endl; anError = true; #if(MIO_BN_DEBUG == 1) - for(int i = 0; i < runningVariance.size() && i < runningVariance_host.size() && - i < MIO_BN_MAX_DEBUGLOOP; + for(int i = 0; i < runVariance.GetVector().size() && + i < runVariance_ref.data.size() && i < MIO_BN_MAX_DEBUGLOOP; i++) { - diff = fabs(Tmix(fabs(runningVariance[i]) - fabs(runningVariance_host[i]))); + diff = fabs( + TAcc(fabs(runVariance.GetVector()[i]) - fabs(runVariance_ref.data[i]))); if(!std::isfinite(diff) || diff > tolerance) { - std::cout << "rv[" << i << "]: " << runningVariance[i]; - std::cout << ", rv_host[" << i << "]: " << runningVariance_host[i]; + std::cout << "rv[" << i << "]: " << runVariance.GetVector()[i]; + std::cout << ", rv_host[" << i << "]: " << runVariance_ref.data[i]; std::cout << ", diff[" << i << "]: " - << Tmix(fabs(runningVariance[i]) - fabs(runningVariance_host[i])) + << TAcc(fabs(runVariance.GetVector()[i]) - + fabs(runVariance_ref.data[i])) << std::endl; } } @@ -1240,28 +1232,29 @@ int BatchNormDriver::VerifyForward() if(saveMeanVar) { // copy back for verification - saveMean_dev->FromGPU(GetStream(), saveMean.data()); - saveInvVariance_dev->FromGPU(GetStream(), saveInvVariance.data()); + savedMean.CopyFromDeviceToHost(GetStream()); + savedVariance.CopyFromDeviceToHost(GetStream()); maxval = static_cast(0.0); - auto errorSaveMean = miopen::rms_range(saveMean_host, saveMean); + auto errorSaveMean = miopen::rms_range(savedMean_ref.data, savedMean.GetVector()); if(!std::isfinite(errorSaveMean) || errorSaveMean > maxrms) { std::cout << "Forward train batch norm verification FAILED on saved mean: " << errorSaveMean << std::endl; anError = true; #if(MIO_BN_DEBUG == 1) - for(int i = 0; - i < saveMean.size() && i < saveMean_host.size() && i < MIO_BN_MAX_DEBUGLOOP; + for(int i = 0; i < savedMean.GetVector().size() && i < savedMean_ref.data.size() && + i < MIO_BN_MAX_DEBUGLOOP; i++) { - diff = fabs(Tmix(fabs(saveMean[i]) - fabs(saveMean_host[i]))); + diff = fabs(TAcc(fabs(savedMean.GetVector()[i]) - fabs(savedMean_ref.data[i]))); maxval = maxval < diff ? diff : maxval; if(!std::isfinite(diff) || diff > tolerance) { - std::cout << "sm[" << i << "]: " << saveMean[i]; - std::cout << ", sm_host[" << i << "]: " << saveMean_host[i]; - std::cout << ", diff[" << i - << "]: " << Tmix(fabs(saveMean[i]) - fabs(saveMean_host[i])) + std::cout << "sm[" << i << "]: " << savedMean.GetVector()[i]; + std::cout << ", sm_host[" << i << "]: " << savedMean_ref.data[i]; + std::cout << ", diff[" << i << "]: " + << TAcc(fabs(savedMean.GetVector()[i]) - + fabs(savedMean_ref.data[i])) << std::endl; } } @@ -1274,7 +1267,8 @@ int BatchNormDriver::VerifyForward() << errorSaveMean << ')' << std::endl; } - auto errorSaveVar = miopen::rms_range(saveInvVariance_host, saveInvVariance); + auto errorSaveVar = + miopen::rms_range(savedVariance_ref.data, savedVariance.GetVector()); if(!std::isfinite(errorSaveVar) || errorSaveVar > maxrms) { std::cout @@ -1282,17 +1276,19 @@ int BatchNormDriver::VerifyForward() << errorSaveVar << std::endl; anError = true; #if(MIO_BN_DEBUG == 1) - for(int i = 0; i < saveInvVariance.size() && i < saveInvVariance_host.size() && - i < MIO_BN_MAX_DEBUGLOOP; + for(int i = 0; i < savedVariance.GetVector().size() && + i < savedVariance_ref.data.size() && i < MIO_BN_MAX_DEBUGLOOP; i++) { - diff = fabs(Tmix(fabs(saveInvVariance[i]) - fabs(saveInvVariance_host[i]))); + diff = fabs( + TAcc(fabs(savedVariance.GetVector()[i]) - fabs(savedVariance_ref.data[i]))); if(!std::isfinite(diff) || diff > tolerance) { - std::cout << "sv[" << i << "]: " << saveInvVariance[i]; - std::cout << ", sv_host[" << i << "]: " << saveInvVariance_host[i]; + std::cout << "sv[" << i << "]: " << savedVariance.GetVector()[i]; + std::cout << ", sv_host[" << i << "]: " << savedVariance_ref.data[i]; std::cout << ", diff[" << i << "]: " - << Tmix(fabs(saveInvVariance[i]) - fabs(saveInvVariance_host[i])) + << TAcc(fabs(savedVariance.GetVector()[i]) - + fabs(savedVariance_ref.data[i])) << std::endl; } } @@ -1307,38 +1303,41 @@ int BatchNormDriver::VerifyForward() } // end if(saveMeanVar) } - // Check output tensor error - out_dev->FromGPU(GetStream(), out.data()); - maxval = static_cast(0.0); - auto errorOut = miopen::rms_range(out_host, out); + out.CopyFromDeviceToHost(GetStream()); + + maxval = static_cast(0.0); + + auto errorOut = miopen::rms_range(out_ref.data, out.GetVector()); if(!std::isfinite(errorOut) || errorOut > maxrms) { std::cout << "Forward batch norm verification FAILED on output: " << errorOut << std::endl; anError = true; #if(MIO_BN_DEBUG == 1) unsigned int count = 0; - for(int i = 0; i < out.size() && i < out_host.size(); i++) + for(int i = 0; i < out.GetVector().size() && i < out_ref.data.size(); i++) { - if(std::isnan(out[i])) + if(std::isnan(out.GetVector()[i])) { - std::cout << "out[" << i << "] produced a nan: " << out[i] << std::endl; + std::cout << "out[" << i << "] produced a nan: " << out.GetVector()[i] << std::endl; } - if(std::isnan(out_host[i])) + if(std::isnan(out_ref.data[i])) { - std::cout << "out_host[" << i << "] produced a nan: " << out_host[i] << std::endl; + std::cout << "out_ref[" << i << "] produced a nan: " << out_ref.data[i] + << std::endl; } - diff = Tref(fabs(out[i]) - fabs(out_host[i])); + diff = Tref(fabs(out.GetVector()[i]) - fabs(out_ref.data[i])); maxval = maxval < diff ? diff : maxval; if(!std::isfinite(diff) || diff > tolerance) { - std::cout << "out[" << i << "]: " << out[i]; - std::cout << ", out_host[" << i << "]: " << out_host[i]; - std::cout << ", diff[" << i << "]: " << Tref(out[i] - out_host[i]) << std::endl; + std::cout << "out[" << i << "]: " << out.GetVector()[i]; + std::cout << ", out_ref.data[" << i << "]: " << out_ref.data[i]; + std::cout << ", diff[" << i << "]: " << Tref(out.GetVector()[i] - out_ref.data[i]) + << std::endl; count++; } } - std::cout << "Number of elements: " << out.size() << std::endl; + std::cout << "Number of elements: " << out.GetVector().size() << std::endl; std::cout << "Number of bad elements: " << count << std::endl; std::cout << "max difference in output: " << maxval << std::endl; #endif @@ -1358,68 +1357,84 @@ int BatchNormDriver::VerifyForward() return miopenStatusSuccess; } -template -int BatchNormDriver::RunBackwardCPU() +template +int BatchNormDriver::RunBackwardCPU() { if(!back) return miopenStatusSuccess; - int nIn = 0, cIn = 0, dIn = 0, hIn = 0, wIn = 0; - if(isDepthSpecified) - miopenGet5dTensorDescriptorLengths(inputTensor, &nIn, &cIn, &dIn, &hIn, &wIn); - else - miopenGet4dTensorDescriptorLengths(inputTensor, &nIn, &cIn, &hIn, &wIn); - - int batch_sz = nIn; - int channels = cIn; - int height = hIn; - int width = wIn; - int depth = dIn; - // T alphaDiff = 1, betaDiff = 0; // T alphaParam = 1, betaParam = 0; - Tref epsilon = static_cast(EPSILON); + double alpha = static_cast(1), beta = static_cast(0), + gamma = static_cast(1); + + // float alphaDataDiff = static_cast(1), betaDataDiff = static_cast(0); + // float alphaParamDiff = static_cast(1), betaParamDiff = static_cast(0); + int size{0}; + miopenGetTensorDescriptorSize(&in.GetTensor().desc, &size); + if(size == 5) + { + in.GetTensor().desc = miopen::BuildReshaped4DTensorDescriptor(in.GetTensor().desc); + dy.GetTensor().desc = miopen::BuildReshaped4DTensorDescriptor(dy.GetTensor().desc); + out_bwd.GetTensor().desc = + miopen::BuildReshaped4DTensorDescriptor(out_bwd.GetTensor().desc); + out_ref.desc = miopen::BuildReshaped4DTensorDescriptor(out_ref.desc); + bnScale.GetTensor().desc = + miopen::BuildReshaped4DTensorDescriptor(bnScale.GetTensor().desc); + dBias.GetTensor().desc = miopen::BuildReshaped4DTensorDescriptor(dBias.GetTensor().desc); + dScale_ref.desc = miopen::BuildReshaped4DTensorDescriptor(dScale_ref.desc); + dBias_ref.desc = miopen::BuildReshaped4DTensorDescriptor(dBias_ref.desc); + savedMean.GetTensor().desc = + miopen::BuildReshaped4DTensorDescriptor(savedMean.GetTensor().desc); + savedInvVar.GetTensor().desc = + miopen::BuildReshaped4DTensorDescriptor(savedInvVar.GetTensor().desc); + } if(bn_mode == miopenBNPerActivation) - { // 1xCxHxW - miopenBNBwdPerActivationRunHost(/* alphaDiff, betaDiff, alphaParam, - betaParam, */ - batch_sz, - channels, - (isDepthSpecified ? depth : 1), - height, - width, - in.data(), - dyin.data(), - dxout_host.data(), - scale.data(), - dscale_host.data(), - dbias_host.data(), - epsilon, - saveMeanVar, - saveMean_host.data(), - saveInvVariance_host.data()); + { + // 1xCxHxW + batchNormActivSpatialHostBwdTrain(activ_mode, + gamma, + beta, + alpha, + in.GetTensor(), + dy.GetTensor(), + out.GetTensor(), + out_ref, + bnScale.GetTensor(), + dBias.GetTensor(), + dScale_ref, + dBias_ref, + savedMean.GetTensor(), + savedInvVar.GetTensor()); } else if(bn_mode == miopenBNSpatial) - { // 1xCx1x1 - miopenBNBwdSpatialRunHost(/* alphaDiff, betaDiff, alphaParam, betaParam, - */ - batch_sz, - channels, - (isDepthSpecified ? depth : 1), - height, - width, - in.data(), - dyin.data(), - dxout_host.data(), - scale.data(), - dscale_host.data(), - dbias_host.data(), - epsilon, - saveMeanVar, - saveMean_host.data(), - saveInvVariance_host.data()); + { // 1xCx1x1 + if(saveMeanVar) + { + + batchNormSpatialHostBwdTrain(in.GetTensor(), + dy.GetTensor(), + out_ref, + bnScale.GetTensor(), + dScale_ref, + dBias_ref, + savedMean.GetTensor(), + savedInvVar.GetTensor()); + } + else + { + tensor empty_tensor; + batchNormSpatialHostBwdTrain(in.GetTensor(), + dy.GetTensor(), + out_ref, + bnScale.GetTensor(), + dScale_ref, + dBias_ref, + empty_tensor, + empty_tensor); + } } else { @@ -1431,45 +1446,49 @@ int BatchNormDriver::RunBackwardCPU() return miopenStatusSuccess; } -template -int BatchNormDriver::VerifyBackward() +template +int BatchNormDriver::VerifyBackward() { if(!back) return miopenStatusSuccess; - const Tref maxrms = static_cast(((sizeof(Tgpu) == 4) ? RMSTOL_FP32 : RMSTOL_FP16) * 1000); - bool anError = false; + const Tref maxrms = + static_cast(((sizeof(TInput) == 4) ? RMSTOL_FP32 : RMSTOL_FP16) * 1000); + bool anError = false; RunBackwardCPU(); - dxout_dev->FromGPU(GetStream(), dxout.data()); - dscale_dev->FromGPU(GetStream(), dscale.data()); - dbias_dev->FromGPU(GetStream(), dbias.data()); + out_bwd.CopyFromDeviceToHost(GetStream()); + dScale.CopyFromDeviceToHost(GetStream()); + dBias.CopyFromDeviceToHost(GetStream()); + #if(MIO_BN_DEBUG == 1) const Tref tolerance = - static_cast(1000 * (sizeof(Tgpu) == 4) ? ERRTOL_FP32 : ERRTOL_FP16); + static_cast(1000 * (sizeof(TInput) == 4) ? ERRTOL_FP32 : ERRTOL_FP16); Tref diff = static_cast(0.0); #endif maxval = static_cast(0.0); - auto errordxout = miopen::rms_range(dxout_host, dxout); + auto errordxout = miopen::rms_range(out_ref.data, out_bwd.GetVector()); if(!std::isfinite(errordxout) || errordxout > maxrms) { std::cout << "Backwards prop batch norm verification FAILED on dx: " << errordxout << std::endl; anError = true; #if(MIO_BN_DEBUG == 1) - for(int i = 0; i < dxout.size() && i < MIO_BN_MAX_DEBUGLOOP; i++) + for(int i = 0; i < out_ref.data.size() && i < MIO_BN_MAX_DEBUGLOOP; i++) { - diff = fabs(Tgpu(fabs(dxout[i]) - fabs(dxout_host[i]))); + diff = fabs(TOut(fabs(out_ref.data[i]) - fabs(out_bwd.GetVector()[i]))); maxval = maxval < diff ? diff : maxval; if(!std::isfinite(diff) || diff > tolerance) { - std::cout << "dxout[" << i << "]: " << dxout[i]; - std::cout << "\tdxout_host[" << i << "]: " << dxout_host[i]; - std::cout << "\tdiff[" << i << "]: " << Tgpu(fabs(dxout[i]) - fabs(dxout_host[i])); + std::cout << "out_ref[" << i << "]: " << out_ref.data[i]; + std::cout << "\tout_bwd.GetVector()[" << i << "]: " << out_bwd.GetVector()[i]; + std::cout << "\tdiff[" << i + << "]: " << TOut(fabs(out_ref.data[i]) - fabs(out_bwd.GetVector()[i])); std::cout << "\tratioH: " - << fabs(fabs(dxout[i]) - fabs(dxout_host[i])) / fabs(dxout_host[i]) + << fabs(fabs(out_ref.data[i]) - fabs(out_bwd.GetVector()[i])) / + fabs(out_bwd.GetVector()[i]) << std::endl; } } @@ -1483,25 +1502,26 @@ int BatchNormDriver::VerifyBackward() } maxval = static_cast(0.0); - auto errordscale = miopen::rms_range(dscale_host, dscale); + auto errordscale = miopen::rms_range(dScale_ref.data, dScale.GetVector()); if(!std::isfinite(errordscale) || errordscale > maxrms) { std::cout << "Backwards prop batch norm verification FAILED on dscale: " << errordscale << std::endl; anError = true; #if(MIO_BN_DEBUG == 1) - for(int i = 0; i < dscale.size() && i < MIO_BN_MAX_DEBUGLOOP; i++) + for(int i = 0; i < dScale.GetVector().size() && i < MIO_BN_MAX_DEBUGLOOP; i++) { - diff = fabs(Tmix(fabs(dscale[i]) - fabs(dscale_host[i]))); - maxval = maxval < diff ? diff : maxval; + auto diff = fabs(TAcc(fabs(dScale.GetVector()[i]) - fabs(dScale_ref.data[i]))); + maxval = maxval < diff ? diff : maxval; if(!std::isfinite(diff) || diff > tolerance) { - std::cout << "dscale[" << i << "]: " << dscale[i]; - std::cout << "\tdscale_host[" << i << "]: " << dscale_host[i]; + std::cout << "dscale[" << i << "]: " << dScale.GetVector()[i]; + std::cout << "\tdscale_host[" << i << "]: " << dScale_ref.data[i]; std::cout << "\tdiff[" << i - << "]: " << Tmix(fabs(dscale[i]) - fabs(dscale_host[i])); + << "]: " << TAcc(fabs(dScale.GetVector()[i]) - fabs(dScale_ref.data[i])); std::cout << "\tratioH: " - << fabs(fabs(dscale[i]) - fabs(dscale_host[i])) / fabs(dscale_host[i]) + << fabs(fabs(dScale.GetVector()[i]) - fabs(dScale_ref.data[i])) / + fabs(dScale_ref.data[i]) << std::endl; } } @@ -1514,23 +1534,25 @@ int BatchNormDriver::VerifyBackward() << ')' << std::endl; } - auto errordbias = miopen::rms_range(dbias_host, dbias); + auto errordbias = miopen::rms_range(dBias_ref.data, dBias.GetVector()); if(!std::isfinite(errordbias) || errordbias > maxrms) { std::cout << "Backwards prop batch norm verification FAILED on dbias: " << errordbias << std::endl; anError = true; #if(MIO_BN_DEBUG == 1) - for(int i = 0; i < dbias.size() && i < MIO_BN_MAX_DEBUGLOOP; i++) + for(int i = 0; i < dBias.GetVector().size() && i < MIO_BN_MAX_DEBUGLOOP; i++) { - diff = fabs(Tmix(fabs(dbias[i]) - fabs(dbias_host[i]))); + diff = fabs(TAcc(fabs(dBias.GetVector()[i]) - fabs(dBias_ref.data[i]))); if(!std::isfinite(diff) || diff > tolerance) { - std::cout << "dbias[" << i << "]: " << dbias[i]; - std::cout << "\tdbias_host[" << i << "]: " << dbias_host[i]; - std::cout << "\tdiff[" << i << "]: " << Tmix(fabs(dbias[i]) - fabs(dbias_host[i])); + std::cout << "dbias[" << i << "]: " << dBias.GetVector()[i]; + std::cout << "\tdbias_host[" << i << "]: " << dBias_ref.data[i]; + std::cout << "\tdiff[" << i + << "]: " << TAcc(fabs(dBias.GetVector()[i]) - fabs(dBias_ref.data[i])); std::cout << "\tratioH: " - << fabs(fabs(dbias[i]) - fabs(dbias_host[i])) / fabs(dbias_host[i]) + << fabs(fabs(dBias.GetVector()[i]) - fabs(dBias_ref.data[i])) / + fabs(dBias_ref.data[i]) << std::endl; } } diff --git a/driver/cat_driver.hpp b/driver/cat_driver.hpp index 51eb16b1c7..3254b5f3bc 100644 --- a/driver/cat_driver.hpp +++ b/driver/cat_driver.hpp @@ -183,8 +183,8 @@ template int CatDriver::AddCmdLineArgs() { inflags.AddInputFlag("forw", 'F', "1", "Run only Forward Cat (Default=1)", "int"); - inflags.AddTensorFlag("input1", '1', "", "input1 tensor descriptor"); - inflags.AddTensorFlag("input2", '2', "", "input2 tensor descriptor"); + inflags.AddTensorFlag("input1", '1', "2x32x128x128x128", "input1 tensor descriptor"); + inflags.AddTensorFlag("input2", '2', "2x32x128x128x128", "input2 tensor descriptor"); inflags.AddTensorFlag("input3", '3', "", "input3 tensor descriptor"); inflags.AddTensorFlag("input4", '4', "", "input4 tensor descriptor"); inflags.AddTensorFlag("input5", '5', "", "input5 tensor descriptor"); diff --git a/driver/conv_driver.hpp b/driver/conv_driver.hpp index 8f9e836345..967849b18e 100644 --- a/driver/conv_driver.hpp +++ b/driver/conv_driver.hpp @@ -47,7 +47,7 @@ #include #include #include -#include +#include #include #include <../test/cpu_bias.hpp> @@ -180,135 +180,6 @@ static inline miopenDataType_t DataTypeFromShortString(const std::string& type) } } -template -class GpumemTensor -{ - std::unique_ptr dev; - tensor host; - bool is_gpualloc = false; - -public: - void SetGpuallocMode(bool v) { is_gpualloc = v; } - tensor& GetTensor() { return host; } - - void AllocOnHost(miopenTensorDescriptor_t t) - { - host = tensor(miopen::deref(t)); - if(is_gpualloc) // We do not need host data. - { - host.data.clear(); - host.data.shrink_to_fit(); // To free host memory. - } - } - - std::vector& GetVector() - { - if(is_gpualloc) - MIOPEN_THROW("[MIOpenDriver] GpumemTensor::GetVector should not be called in " - "'--gpualloc 1' mode"); - return host.data; - } - - Tgpu* GetVectorData() { return is_gpualloc ? nullptr : host.data.data(); } - std::size_t GetVectorSize() const { return is_gpualloc ? 0 : host.data.size(); } - - void - InitHostData(const size_t sz, // - const bool do_write, // If set to false, then only generate random data. This is - // necessary to reproduce values in input buffers even if some - // directions are skipped. For example, inputs for Backward - // will be the same for both "-F 0" and "-F 2". - std::function generator) - { - if(is_gpualloc) - { - /// In gpualloc mode, we do not care about reproducibility of results, because - /// validation is not used. Therefore, we do not have to always generate random value - /// (\ref move_rand) - return; - } - - for(size_t i = 0; i < sz; ++i) - { - /// \anchor move_rand - /// Generate random value, even if buffer is unused. This provides the same - /// initialization of input buffers regardless of which kinds of - /// convolutions are currently selectedfor testing (see the "-F" option). - /// Verification cache would be broken otherwise. - auto val = generator(); - if(do_write) - GetVector()[i] = val; - } - } - - status_t AllocOnDevice(stream, context_t ctx, const size_t sz) - { - dev = std::make_unique(ctx, sz, sizeof(Tgpu)); - return STATUS_SUCCESS; - } - - status_t AllocOnDeviceAndInit(stream q, context_t ctx, const size_t sz) - { - AllocOnDevice(q, ctx, sz); - if(is_gpualloc) - { - /// \anchor gpualloc_random_init - /// In gpualloc mode, we do not want to leave input buffers uninitialized, because - /// there could be NaNs and Infs, which may affect the performance (which we are - /// interested to evaluate in this mode). Initialization with all 0's is not the - /// best choice as well, because GPU HW may optimize out computations with 0's and - /// that could affect performance of kernels too. That is why we are using - /// rocrand to initialize input buffers. - /// - /// However we do not care about precision in gpualloc mode, because validation - /// is not used. Therefore, range (0,1] is fine. - return gpumemrand::gen_0_1(static_cast(GetDevicePtr()), sz); - } - return dev->ToGPU(q, GetVectorData()); - } - - template - status_t AllocOnDevice(stream, context_t ctx, const size_t sz, std::vector&) - { - static_assert(std::is_same::value // - || std::is_same::value, // - "Before enabling more types, check thoroughly."); - dev = std::make_unique(ctx, sz, sizeof(T)); - return STATUS_SUCCESS; - } - - template - status_t AllocOnDeviceAndInit(stream q, context_t ctx, const size_t sz, std::vector& init) - { - AllocOnDevice(q, ctx, sz, init); - if(is_gpualloc) - { - /// \ref gpualloc_random_init - return gpumemrand::gen_0_1(static_cast(GetDevicePtr()), sz); - } - return dev->ToGPU(q, init.data()); - } - - status_t CopyFromDeviceToHost(stream q) - { - return is_gpualloc ? STATUS_SUCCESS : dev->FromGPU(q, GetVectorData()); - } - - template - status_t CopyFromDeviceToHost(stream q, tensor& t) - { - return is_gpualloc ? STATUS_SUCCESS : dev->FromGPU(q, t.data.data()); - } - - template - status_t CopyFromDeviceToHost(stream q, std::vector& v) - { - return is_gpualloc ? STATUS_SUCCESS : dev->FromGPU(q, v.data()); - } - - auto GetDevicePtr() -> auto { return dev->GetMem(); } -}; - template class GpumemVector { @@ -932,19 +803,17 @@ int ConvDriver::GetandSetData() SetConvDescriptorFromCmdLineArgs(); std::vector out_len = GetOutputTensorLengths(); - if(miopen::deref(inputTensor).GetLayout_t() == miopenTensorNCHWc4 || - miopen::deref(inputTensor).GetLayout_t() == miopenTensorNCHWc8) + if(miopen::deref(inputTensor).GetLayoutEnum() == miopenTensorNCHWc4 || + miopen::deref(inputTensor).GetLayoutEnum() == miopenTensorNCHWc8) { out_len[1] *= miopen::deref(inputTensor).GetVectorLength(); } - if(miopen::deref(inputTensor).GetLayout_t() == miopenTensorCHWNc4 || - miopen::deref(inputTensor).GetLayout_t() == miopenTensorCHWNc8) + if(miopen::deref(inputTensor).GetLayoutEnum() == miopenTensorCHWNc4 || + miopen::deref(inputTensor).GetLayoutEnum() == miopenTensorCHWNc8) { out_len[0] *= miopen::deref(inputTensor).GetVectorLength(); } - miopenDataType_t y_type = - (data_type == miopenInt8 || data_type == miopenInt8x4) ? miopenInt32 : data_type; - SetTensorNd(outputTensor, out_len, inflags.GetValueStr("out_layout"), y_type); + SetTensorNd(outputTensor, out_len, inflags.GetValueStr("out_layout"), data_type); if(inflags.GetValueStr("out_cast_type") != "-1") { const auto out_cast_type = DataTypeFromShortString(inflags.GetValueStr("out_cast_type")); @@ -1799,7 +1668,7 @@ void ConvDriver::PrintForwardTime(const float kernel_total_time, const auto num_dim = miopen::deref(inputTensor).GetNumDims() - 2; if(num_dim != 2 && num_dim != 3) { - printf("stats: for conv%dd\n", num_dim); + printf("stats: for conv%ud\n", num_dim); return; } @@ -1829,7 +1698,7 @@ void ConvDriver::PrintForwardTime(const float kernel_total_time, printf("stats: name, n, c, ho, wo, x, y, k, flopCnt, bytesRead, bytesWritten, GFLOPs, " "GB/s, timeMs\n"); - printf("stats: %s%dx%du%d, %u, %u, %u, %u, %u, %u, %u, %zu, %zu, %zu, %.0f, %.0f, %f\n", + printf("stats: %s%dx%du%d, %d, %d, %d, %d, %d, %d, %d, %zu, %zu, %zu, %.0f, %.0f, %f\n", "fwd-conv", wei_h, wei_w, @@ -1874,7 +1743,7 @@ void ConvDriver::PrintForwardTime(const float kernel_total_time, printf("stats: name , n, c, do, ho, wo, z, y, x, k, flopCnt, bytesRead, bytesWritten, " "GFLOPs, " "GB/s, timeMs\n"); - printf("stats: %s%dx%dx%du%d, %u, %u, %u, %u, %u, %u, %u, %u, %u, %zu, %zu, %zu, " + printf("stats: %s%dx%dx%du%d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %zu, %zu, %zu, " "%.0f, %.0f, %f\n", "fwd-conv", wei_d, @@ -2717,7 +2586,7 @@ void ConvDriver::PrintBackwardDataTime(float kernel_total_time, floa const auto num_dim = miopen::deref(inputTensor).GetNumDims() - 2; if(num_dim != 2 && num_dim != 3) { - printf("stats: for conv%dd\n", num_dim); + printf("stats: for conv%ud\n", num_dim); return; } @@ -2747,7 +2616,7 @@ void ConvDriver::PrintBackwardDataTime(float kernel_total_time, floa printf("stats: name, n, c, ho, wo, x, y, k, flopCnt, bytesRead, bytesWritten, GFLOPs, " "GB/s, timeMs\n"); - printf("stats: %s%dx%du%d, %u, %u, %u, %u, %u, %u, %u, %zu, %zu, %zu, %.0f, %.0f, %f\n", + printf("stats: %s%dx%du%d, %d, %d, %d, %d, %d, %d, %d, %zu, %zu, %zu, %.0f, %.0f, %f\n", "bwdd-conv", wei_h, wei_w, @@ -2792,7 +2661,7 @@ void ConvDriver::PrintBackwardDataTime(float kernel_total_time, floa printf( "stats: name, n, c, do, ho, wo, z, x, y, k, flopCnt, bytesRead, bytesWritten, GFLOPs, " "GB/s, timeMs\n"); - printf("stats: %s%dx%dx%du%d, %u, %u, %u, %u, %u, %u, %u, %u, %u %zu, %zu, %zu, %.0f, " + printf("stats: %s%dx%dx%du%d, %d, %d, %d, %d, %d, %d, %d, %d, %d %zu, %zu, %zu, %.0f, " "%.0f, %f\n", "bwdd-conv", wei_d, @@ -2928,7 +2797,7 @@ void ConvDriver::PrintBackwardWrwTime(float kernel_total_time, float const auto num_dim = miopen::deref(inputTensor).GetNumDims() - 2; if(num_dim != 2 && num_dim != 3) { - printf("stats: for conv%dd\n", num_dim); + printf("stats: for conv%ud\n", num_dim); return; } @@ -2952,7 +2821,7 @@ void ConvDriver::PrintBackwardWrwTime(float kernel_total_time, float printf("stats: name, n, c, ho, wo, x, y, k, flopCnt, bytesRead, bytesWritten, GFLOPs, " "GB/s, timeMs\n"); - printf("stats: %s%dx%du%d, %u, %u, %u, %u, %u, %u, %u, %zu, %zu, %zu, %.0f, %.0f, %f\n", + printf("stats: %s%dx%du%d, %d, %d, %d, %d, %d, %d, %d, %zu, %zu, %zu, %.0f, %.0f, %f\n", "bwdw-conv", wei_h, wei_w, @@ -2991,7 +2860,7 @@ void ConvDriver::PrintBackwardWrwTime(float kernel_total_time, float printf( "stats: name, n, c, do, ho, wo, z, x, y, k, flopCnt, bytesRead, bytesWritten, GFLOPs, " "GB/s, timeMs\n"); - printf("stats: %s%dx%dx%du%d, %u, %u, %u, %u, %u, %u, %u, %u, %u, %zu, %zu, %zu, %.0f, " + printf("stats: %s%dx%dx%du%d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %zu, %zu, %zu, %.0f, " "%.0f, %f\n", "bwdw-conv", wei_d, diff --git a/driver/dm_bnorm.cpp b/driver/dm_bnorm.cpp index c7bab90bb5..22d8bc6429 100644 --- a/driver/dm_bnorm.cpp +++ b/driver/dm_bnorm.cpp @@ -30,8 +30,15 @@ static Driver* makeDriver(const std::string& base_arg) { if(base_arg == "bnorm") return new BatchNormDriver(); + // if(base_arg == "bnormfp16") - return new BatchNormDriver(); + return new BatchNormDriver(); + if(base_arg == "bnormbfp16") + return new BatchNormDriver(); + if(base_arg == "bnormfp16fp32") + return new BatchNormDriver(); + if(base_arg == "bnormbfp16fp32") + return new BatchNormDriver(); return nullptr; } diff --git a/driver/dm_getitem.cpp b/driver/dm_getitem.cpp new file mode 100644 index 0000000000..bfb72be96a --- /dev/null +++ b/driver/dm_getitem.cpp @@ -0,0 +1,40 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#include "getitem_driver.hpp" +#include "registry_driver_maker.hpp" + +static Driver* makeDriver(const std::string& base_arg) +{ + if(base_arg == "getitem") + return new GetitemDriver(); + if(base_arg == "getitemfp16") + return new GetitemDriver(); + if(base_arg == "getitembfp16") + return new GetitemDriver(); + return nullptr; +} + +REGISTER_DRIVER_MAKER(makeDriver); diff --git a/driver/dm_sum.cpp b/driver/dm_glu.cpp similarity index 85% rename from driver/dm_sum.cpp rename to driver/dm_glu.cpp index ef89f1feda..666ccf03da 100644 --- a/driver/dm_sum.cpp +++ b/driver/dm_glu.cpp @@ -24,16 +24,16 @@ * *******************************************************************************/ #include "registry_driver_maker.hpp" -#include "sum_driver.hpp" +#include "glu_driver.hpp" static Driver* makeDriver(const std::string& base_arg) { - if(base_arg == "sum") - return new SumDriver(); - if(base_arg == "sumfp16") - return new SumDriver(); - if(base_arg == "sumbfp16") - return new SumDriver(); + if(base_arg == "glu") + return new GLUDriver(); + if(base_arg == "glufp16") + return new GLUDriver(); + if(base_arg == "glubfp16") + return new GLUDriver(); return nullptr; } diff --git a/driver/dm_kthvalue.cpp b/driver/dm_kthvalue.cpp new file mode 100644 index 0000000000..71adfdc245 --- /dev/null +++ b/driver/dm_kthvalue.cpp @@ -0,0 +1,41 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include "registry_driver_maker.hpp" +#include "kthvalue_driver.hpp" + +static Driver* makeDriver(const std::string& base_arg) +{ + if(base_arg == "kthvalue") + return new KthvalueDriver(); + else if(base_arg == "kthvaluefp16") + return new KthvalueDriver(); + else if(base_arg == "kthvaluebfp16") + return new KthvalueDriver(); + return nullptr; +} + +REGISTER_DRIVER_MAKER(makeDriver); diff --git a/driver/dm_multimarginloss.cpp b/driver/dm_multimarginloss.cpp new file mode 100644 index 0000000000..1c924bb7dc --- /dev/null +++ b/driver/dm_multimarginloss.cpp @@ -0,0 +1,40 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#include "multimarginloss_driver.hpp" +#include "registry_driver_maker.hpp" + +static Driver* makeDriver(const std::string& base_arg) +{ + if(base_arg == "multimarginloss") + return new MultiMarginLossDriver(); + if(base_arg == "multimarginlossfp16") + return new MultiMarginLossDriver(); + if(base_arg == "multimarginlossbfp16") + return new MultiMarginLossDriver(); + return nullptr; +} + +REGISTER_DRIVER_MAKER(makeDriver); diff --git a/driver/dm_prelu.cpp b/driver/dm_prelu.cpp new file mode 100644 index 0000000000..06ec99e8a5 --- /dev/null +++ b/driver/dm_prelu.cpp @@ -0,0 +1,40 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#include "registry_driver_maker.hpp" +#include "prelu_driver.hpp" + +static Driver* makeDriver(const std::string& base_arg) +{ + if(base_arg == "prelu") + return new PReLUDriver(); + if(base_arg == "prelufp16") + return new PReLUDriver(); + if(base_arg == "prelubfp16") + return new PReLUDriver(); + return nullptr; +} + +REGISTER_DRIVER_MAKER(makeDriver); diff --git a/driver/dm_reducecalculation.cpp b/driver/dm_reducecalculation.cpp new file mode 100644 index 0000000000..90475db0d1 --- /dev/null +++ b/driver/dm_reducecalculation.cpp @@ -0,0 +1,40 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#include "reducecalculation_driver.hpp" +#include "registry_driver_maker.hpp" + +static Driver* makeDriver(const std::string& base_arg) +{ + if(base_arg == "reducecalculation") + return new ReduceCalculationDriver(); + if(base_arg == "reducecalculationfp16") + return new ReduceCalculationDriver(); + if(base_arg == "reducecalculationbfp16") + return new ReduceCalculationDriver(); + return nullptr; +} + +REGISTER_DRIVER_MAKER(makeDriver); diff --git a/driver/dm_rope.cpp b/driver/dm_rope.cpp new file mode 100644 index 0000000000..9f9b0c2046 --- /dev/null +++ b/driver/dm_rope.cpp @@ -0,0 +1,40 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#include "registry_driver_maker.hpp" +#include "rope_driver.hpp" + +static Driver* makeDriver(const std::string& base_arg) +{ + if(base_arg == "rope") + return new RoPEDriver(); + if(base_arg == "ropefp16") + return new RoPEDriver(); + if(base_arg == "ropebfp16") + return new RoPEDriver(); + return nullptr; +} + +REGISTER_DRIVER_MAKER(makeDriver); diff --git a/driver/dm_softmarginloss.cpp b/driver/dm_softmarginloss.cpp new file mode 100644 index 0000000000..5beec40a08 --- /dev/null +++ b/driver/dm_softmarginloss.cpp @@ -0,0 +1,40 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#include "softmarginloss_driver.hpp" +#include "registry_driver_maker.hpp" + +static Driver* makeDriver(const std::string& base_arg) +{ + if(base_arg == "softmarginloss") + return new SoftMarginLossDriver(); + if(base_arg == "softmarginlossfp16") + return new SoftMarginLossDriver(); + if(base_arg == "softmarginlossbfp16") + return new SoftMarginLossDriver(); + return nullptr; +} + +REGISTER_DRIVER_MAKER(makeDriver); diff --git a/driver/driver.hpp b/driver/driver.hpp index bf951a54ad..d77d5d02d2 100644 --- a/driver/driver.hpp +++ b/driver/driver.hpp @@ -38,6 +38,9 @@ #include #include #include +#include <../test/tensor_holder.hpp> +#include "util_driver.hpp" +#include "rocrand_wrapper.hpp" using half = half_float::half; using hip_bfloat16 = bfloat16; #include @@ -157,6 +160,140 @@ struct GPUMem #endif }; +template +class GpumemTensor +{ + std::unique_ptr dev; + tensor host; + bool is_gpualloc = false; + +public: + void SetGpuallocMode(bool v) { is_gpualloc = v; } + tensor& GetTensor() { return host; } + + void AllocOnHost(miopenTensorDescriptor_t t) + { + host = tensor(miopen::deref(t)); + if(is_gpualloc) // We do not need host data. + { + host.data.clear(); + host.data.shrink_to_fit(); // To free host memory. + } + } + template + void AllocOnHost(tensor t) + { + AllocOnHost(&t.desc); + } + + std::vector& GetVector() + { + if(is_gpualloc) + MIOPEN_THROW("[MIOpenDriver] GpumemTensor::GetVector should not be called in " + "'--gpualloc 1' mode"); + return host.data; + } + + Tgpu* GetVectorData() { return is_gpualloc ? nullptr : host.data.data(); } + std::size_t GetVectorSize() const { return is_gpualloc ? 0 : host.data.size(); } + + void + InitHostData(const size_t sz, // + const bool do_write, // If set to false, then only generate random data. This is + // necessary to reproduce values in input buffers even if some + // directions are skipped. For example, inputs for Backward + // will be the same for both "-F 0" and "-F 2". + std::function generator) + { + if(is_gpualloc) + { + /// In gpualloc mode, we do not care about reproducibility of results, because + /// validation is not used. Therefore, we do not have to always generate random value + /// (\ref move_rand) + return; + } + + for(size_t i = 0; i < sz; ++i) + { + /// \anchor move_rand + /// Generate random value, even if buffer is unused. This provides the same + /// initialization of input buffers regardless of which kinds of + /// convolutions are currently selectedfor testing (see the "-F" option). + /// Verification cache would be broken otherwise. + auto val = generator(); + if(do_write) + GetVector()[i] = val; + } + } + + status_t AllocOnDevice(stream, context_t ctx, const size_t sz) + { + dev = std::make_unique(ctx, sz, sizeof(Tgpu)); + return STATUS_SUCCESS; + } + + status_t AllocOnDeviceAndInit(stream q, context_t ctx, const size_t sz) + { + AllocOnDevice(q, ctx, sz); + if(is_gpualloc) + { + /// \anchor gpualloc_random_init + /// In gpualloc mode, we do not want to leave input buffers uninitialized, because + /// there could be NaNs and Infs, which may affect the performance (which we are + /// interested to evaluate in this mode). Initialization with all 0's is not the + /// best choice as well, because GPU HW may optimize out computations with 0's and + /// that could affect performance of kernels too. That is why we are using + /// rocrand to initialize input buffers. + /// + /// However we do not care about precision in gpualloc mode, because validation + /// is not used. Therefore, range (0,1] is fine. + return gpumemrand::gen_0_1(static_cast(GetDevicePtr()), sz); + } + return dev->ToGPU(q, GetVectorData()); + } + + template + status_t AllocOnDevice(stream, context_t ctx, const size_t sz, std::vector&) + { + static_assert(std::is_same::value // + || std::is_same::value, // + "Before enabling more types, check thoroughly."); + dev = std::make_unique(ctx, sz, sizeof(T)); + return STATUS_SUCCESS; + } + + template + status_t AllocOnDeviceAndInit(stream q, context_t ctx, const size_t sz, std::vector& init) + { + AllocOnDevice(q, ctx, sz, init); + if(is_gpualloc) + { + /// \ref gpualloc_random_init + return gpumemrand::gen_0_1(static_cast(GetDevicePtr()), sz); + } + return dev->ToGPU(q, init.data()); + } + + status_t CopyFromDeviceToHost(stream q) + { + return is_gpualloc ? STATUS_SUCCESS : dev->FromGPU(q, GetVectorData()); + } + + template + status_t CopyFromDeviceToHost(stream q, tensor& t) + { + return is_gpualloc ? STATUS_SUCCESS : dev->FromGPU(q, t.data.data()); + } + + template + status_t CopyFromDeviceToHost(stream q, std::vector& v) + { + return is_gpualloc ? STATUS_SUCCESS : dev->FromGPU(q, v.data()); + } + + auto GetDevicePtr() -> auto { return dev->GetMem(); } +}; + inline void PadBufferSize(size_t& sz, int datatype_sz) { size_t page_sz = (2 * 1024 * 1024) / datatype_sz; @@ -169,13 +306,15 @@ inline void PadBufferSize(size_t& sz, int datatype_sz) [[noreturn]] inline void Usage() { printf("Usage: ./driver *base_arg* *other_args*\n"); - printf("Supported Base Arguments: conv[fp16|int8|bfp16|fp8|bfp8], CBAInfer[fp16], " - "pool[fp16], lrn[fp16], " + printf("Supported Base Arguments: conv[fp16|int8|bfp16], pool[fp16], lrn[fp16], " "activ[fp16], softmax[fp16], bnorm[fp16], rnn[fp16], gemm[fp16], ctc, dropout[fp16], " - "tensorop[fp16], reduce[fp16|fp64], layernorm[bfp16|fp16], sum[bfp16|fp16], " + "tensorop, reduce[fp16|fp64], layernorm[bfp16|fp16], sum[bfp16|fp16], " "groupnorm[bfp16|fp16], cat[bfp16|fp16], addlayernorm[bfp16|fp16], " "t5layernorm[bfp16|fp16], adam[fp16], ampadam, reduceextreme[bfp16|fp16], " - "adamw[fp16], ampadamw, transformersadamw[fp16], transformersampadamw\n"); + "adamw[fp16], ampadamw, transformersadamw[fp16], transformersampadamw, " + "getitem[bfp16|fp16], reducecalculation[bfp16|fp16], rope[bfp16|fp16], " + "prelu[bfp16|fp16], kthvalue[bfp16|fp16], glu[bfp16|fp16], softmarginloss[bfp16|fp16], " + "multimarginloss[bfp16|fp16]\n"); exit(0); // NOLINT (concurrency-mt-unsafe) } @@ -190,22 +329,30 @@ inline std::string ParseBaseArg(int argc, char* argv[]) std::string arg = argv[1]; if(arg != "conv" && arg != "convfp16" && arg != "convint8" && arg != "convbfp16" && - arg != "convfp8" && arg != "convbfp8" && arg != "CBAInfer" && arg != "CBAInferfp16" && arg != "pool" && arg != "poolfp16" && arg != "lrn" && arg != "lrnfp16" && arg != "activ" && arg != "activfp16" && arg != "softmax" && arg != "softmaxfp16" && arg != "bnorm" && - arg != "bnormfp16" && arg != "rnn" && arg != "rnnfp16" && arg != "rnn_seq" && + arg != "bnormfp16" && arg != "bnormbfp16" && arg != "bnormfp16fp32" && + arg != "bnormbfp16fp32" && arg != "rnn" && arg != "rnnfp16" && arg != "rnn_seq" && arg != "rnn_seqfp16" && arg != "gemm" && arg != "gemmfp16" && arg != "ctc" && - arg != "dropout" && arg != "dropoutfp16" && arg != "tensorop" && arg != "tensoropfp16" && - arg != "reduce" && arg != "reducefp16" && arg != "reducefp64" && arg != "layernorm" && - arg != "layernormfp16" && arg != "layernormbfp16" && arg != "sum" && arg != "sumfp16" && - arg != "sumbfp16" && arg != "groupnorm" && arg != "groupnormfp16" && - arg != "groupnormbfp16" && arg != "cat" && arg != "catfp16" && arg != "catbfp16" && - arg != "addlayernorm" && arg != "addlayernormfp16" && arg != "addlayernormbfp16" && - arg != "t5layernorm" && arg != "t5layernormfp16" && arg != "t5layernormbfp16" && - arg != "adam" && arg != "adamfp16" && arg != "ampadam" && arg != "reduceextreme" && + arg != "dropout" && arg != "dropoutfp16" && arg != "tensorop" && arg != "reduce" && + arg != "reducefp16" && arg != "reducefp64" && arg != "layernorm" && arg != "layernormfp16" && + arg != "layernormbfp16" && arg != "sum" && arg != "sumfp16" && arg != "sumbfp16" && + arg != "groupnorm" && arg != "groupnormfp16" && arg != "groupnormbfp16" && arg != "cat" && + arg != "catfp16" && arg != "catbfp16" && arg != "addlayernorm" && + arg != "addlayernormfp16" && arg != "addlayernormbfp16" && arg != "t5layernorm" && + arg != "t5layernormfp16" && arg != "t5layernormbfp16" && arg != "adam" && + arg != "adamfp16" && arg != "ampadam" && arg != "reduceextreme" && arg != "reduceextremefp16" && arg != "reduceextremebfp16" && arg != "adamw" && arg != "adamwfp16" && arg != "ampadamw" && arg != "transformersadamw" && - arg != "transformersadamwfp16" && arg != "transformersampadamw" && arg != "--version") + arg != "transformersadamwfp16" && arg != "transformersampadamw" && arg != "getitem" && + arg != "getitemfp16" && arg != "getitembfp16" && arg != "reducecalculation" && + arg != "reducecalculationfp16" && arg != "reducecalculationbfp16" && arg != "rope" && + arg != "ropefp16" && arg != "ropebfp16" && arg != "prelu" && arg != "prelufp16" && + arg != "prelubfp16" && arg != "kthvalue" && arg != "kthvaluefp16" && + arg != "kthvaluebfp16" && arg != "glu" && arg != "glufp16" && arg != "glubfp16" && + arg != "softmarginloss" && arg != "softmarginlossfp16" && arg != "softmarginlossbfp16" && + arg != "multimarginloss" && arg != "multimarginlossfp16" && arg != "multimarginlossbfp16" && + arg != "--version") { printf("FAILED: Invalid Base Input Argument\n"); Usage(); diff --git a/driver/dropout_driver.hpp b/driver/dropout_driver.hpp index 80be8fa785..0215c8c64c 100644 --- a/driver/dropout_driver.hpp +++ b/driver/dropout_driver.hpp @@ -109,7 +109,7 @@ class DropoutDriver : public Driver tensor outhost; tensor din_host; - std::vector states_host; + std::vector states_host; std::vector reservespace; std::vector reservespace_host; @@ -222,17 +222,18 @@ int DropoutDriver::AllocateBuffersAndCopy() size_t statesSizeInBytes = 0; miopenDropoutGetStatesSize(GetHandle(), &statesSizeInBytes); - size_t states_size = statesSizeInBytes / sizeof(prngStates); + size_t states_size = statesSizeInBytes / sizeof(rocrand_state_xorwow); DEFINE_CONTEXT(ctx); #if MIOPEN_BACKEND_OPENCL clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr); #endif - states_dev = std::unique_ptr(new GPUMem(ctx, states_size, sizeof(prngStates))); + states_dev = + std::unique_ptr(new GPUMem(ctx, states_size, sizeof(rocrand_state_xorwow))); - if(inflags.GetValueInt("gen_file")) - generate_skipahead_file(); + // if(inflags.GetValueInt("gen_file")) + // generate_skipahead_file(); miopenSetDropoutDescriptor(DropoutDesc, GetHandle(), @@ -269,7 +270,7 @@ int DropoutDriver::AllocateBuffersAndCopy() reservespace = std::vector(reserveSpaceSize, static_cast(1)); reservespace_host = std::vector(reserveSpaceSize, static_cast(1)); - states_host = std::vector(states_size); + states_host = std::vector(states_size); Tgpu Data_scale = static_cast(0.01); diff --git a/driver/dropout_gpu_emulator.hpp b/driver/dropout_gpu_emulator.hpp index 435d3cba55..d6ee776e56 100644 --- a/driver/dropout_gpu_emulator.hpp +++ b/driver/dropout_gpu_emulator.hpp @@ -40,101 +40,18 @@ #include #include #include -#include -#include -#include "xorwow_skipahead_generator.hpp" -#define ROCRAND_2POW32_INV (2.3283064e-10f) - -static float uniform_distribution_emu(size_t v) -{ - return ROCRAND_2POW32_INV + (v * ROCRAND_2POW32_INV); -} - -static void xorwow_skipahead_emu( - unsigned long long skp, - prngStates* state, - const unsigned int skipahead_mat[XORWOW_PRECALC_MATRICES_NUM][XORWOW_PRECALC_MATRICES_SZ]) -{ - unsigned int xor_vec[XORWOW_DIM]; - unsigned int* p = &(state->x); - std::copy(p, p + XORWOW_DIM, std::begin(xor_vec)); - - unsigned int mat_idx = 0; - while(bool(skp) -#if(XORWOW_PRECALC_MATRICES_NUM * XORWOW_JUMP_LOG2) < 64 - && mat_idx < XORWOW_PRECALC_MATRICES_NUM +// disable __device__ qualifiers +#ifdef FQUALIFIERS +#error rocrand FQUALIFIERS defined externally, probably one of rocrand device header included prior to this #endif - ) - { - for(unsigned int i = 0; i < static_cast(skp & XORWOW_JUMP_LOG2_MASK); i++) - { - mat_vec(skipahead_mat[mat_idx], xor_vec); - } - skp >>= XORWOW_JUMP_LOG2; - mat_idx++; - } - -#if(XORWOW_PRECALC_MATRICES_NUM * XORWOW_JUMP_LOG2) < 64 - if(skp) - { - unsigned int matrixA[XORWOW_PRECALC_MATRICES_SZ], matrixB[XORWOW_PRECALC_MATRICES_SZ]; - std::copy(&(skipahead_mat[XORWOW_PRECALC_MATRICES_NUM - 1][0]), - &(skipahead_mat[XORWOW_PRECALC_MATRICES_NUM - 1][0]) + XORWOW_PRECALC_MATRICES_SZ, - std::begin(matrixA)); - - while(skp) - { - mat_pow(matrixB, matrixA, 1ULL << XORWOW_JUMP_LOG2); - std::copy(std::begin(matrixB), std::end(matrixB), std::begin(matrixA)); - - for(unsigned int i = 0; i < static_cast(skp & XORWOW_JUMP_LOG2_MASK); i++) - { - mat_vec(matrixA, xor_vec); - } - skp >>= XORWOW_JUMP_LOG2; - } - } -#endif - - std::copy(std::begin(xor_vec), std::end(xor_vec), p); -} - -static void xorwow_lite_init_emu(prngStates* cur_state, - const unsigned long long seed, - const unsigned long long subsequence, - const unsigned long long offset) -{ - cur_state->x = 123456789; - cur_state->y = 362436069; - cur_state->z = 521288629; - cur_state->w = 88675123; - cur_state->v = 5783321; - - cur_state->d = 6615241; - - // Adopt constants choice of rocRAND (https://github.com/ROCm/rocRAND) - const unsigned int s0 = static_cast(seed) ^ 0x2c7f967fU; - const unsigned int s1 = static_cast(seed >> 32) ^ 0xa03697cbU; - const unsigned int t0 = 1228688033 * s0; - const unsigned int t1 = 2073658381 * s1; - cur_state->x += t0; - cur_state->y ^= t0; - cur_state->z += t1; - cur_state->w ^= t1; - cur_state->v += t0; - cur_state->d += t1 + t0; - - xorwow_skipahead_emu(subsequence, cur_state, precalc_xorwow_skipahead_sequence_matrices); - - xorwow_skipahead_emu(offset, cur_state, precalc_xorwow_skipahead_matrices); - cur_state->d += static_cast(offset) * 362437; -} +#define FQUALIFIERS inline +#include "../src/kernels/miopen_rocrand.hpp" -static void InitKernelStateEmulator(std::vector& states, +static void InitKernelStateEmulator(std::vector& states, const miopenDropoutDescriptor_t dropoutDesc) { - size_t states_num = miopen::deref(dropoutDesc).stateSizeInBytes / sizeof(prngStates); + size_t states_num = miopen::deref(dropoutDesc).stateSizeInBytes / sizeof(rocrand_state_xorwow); size_t wk_grp_num = std::min(size_t(MAX_PRNG_STATE / 256), (states_num + 255) / 256); size_t glb_sz = wk_grp_num * 256; @@ -142,10 +59,10 @@ static void InitKernelStateEmulator(std::vector& states, { for(size_t i = 0; i < glb_sz; i++) { - size_t gid = i + j * glb_sz; - unsigned long long seq = gid; - unsigned long long offset = 0; - xorwow_lite_init_emu(&states[gid], miopen::deref(dropoutDesc).seed, seq, offset); + size_t gid = i + j * glb_sz; + rocrand_state_xorwow state_gid; + rocrand_init(miopen::deref(dropoutDesc).seed, gid, 0ULL, &state_gid); + states[gid] = state_gid; } } } @@ -202,7 +119,7 @@ void RunDropoutForwardEmulator(miopenHandle_t handle, const miopenTensorDescriptor_t outputTensor, std::vector& out, std::vector& reservespace, - std::vector& states, + std::vector& states, size_t in_offset = 0, size_t out_offset = 0, size_t rsvsp_offset = 0) @@ -272,8 +189,7 @@ void RunDropoutForwardEmulator(miopenHandle_t handle, if(!use_mask) reservespace[ri] = - uniform_distribution_emu(xorwow_next(&states[si % glb_sz])) > - dropout_rate; + prng::xorwow_uniform(&states[si % glb_sz]) > dropout_rate; out[oi] = bool(reservespace[ri]) && !miopen::float_equal(dropout_rate, 1.0) ? static_cast(in[ii] / (1 - dropout_rate)) diff --git a/driver/getitem_driver.hpp b/driver/getitem_driver.hpp new file mode 100644 index 0000000000..c48c9a0520 --- /dev/null +++ b/driver/getitem_driver.hpp @@ -0,0 +1,545 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#ifndef GUARD_MIOPEN_GETITEM_DRIVER_HPP +#define GUARD_MIOPEN_GETITEM_DRIVER_HPP + +#include "InputFlags.hpp" +#include "driver.hpp" +#include "tensor_driver.hpp" +#include "timer.hpp" +#include "random.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include <../test/tensor_holder.hpp> +#include <../test/verify.hpp> + +template +int32_t mloGetitemBackwardRunHost(miopenTensorDescriptor_t dyDesc, + uint32_t indexCount, + miopenTensorDescriptor_t* indexDescs, + miopenTensorDescriptor_t dxDesc, + miopenTensorDescriptor_t errorDesc, + Tgpu* dy, + int32_t** indexs, + Tcheck* dxhost, + int32_t* errorhost, + uint32_t dimCount, + int32_t* dims, + uint32_t sliceCount, + int32_t* slices, + uint32_t offset) +{ + auto dy_dims = miopen::deref(dyDesc).GetLengths(); + auto dy_numel = std::accumulate(dy_dims.begin(), dy_dims.end(), 1L, std::multiplies()); + auto dx_dims = miopen::deref(dxDesc).GetLengths(); + auto index_dims = miopen::deref(indexDescs[0]).GetLengths(); + auto index_numel = + std::accumulate(index_dims.begin(), index_dims.end(), 1L, std::multiplies()); + auto element_index = std::vector(indexCount * index_numel + indexCount); + + std::vector output_dims; + for(int32_t i = 0; i < dimCount; i++) + { + output_dims.push_back(dx_dims[dims[i]]); + } + + auto dim_info_offset = indexCount > 0 ? indexCount * index_dims[0] : 0; + auto start_dim = dims[0]; + + auto dy_tv = miopen::get_inner_expanded_tv<5>(miopen::deref(dyDesc)); + auto dxhost_tv = miopen::get_inner_expanded_tv<5>(miopen::deref(dxDesc)); + miopen::slice_tv<5>(dxhost_tv, sliceCount, slices); + + int32_t ret = 0; + + // Get element index form indexs + for(size_t j = 0; j < indexCount; j++) + { + const auto& index_dim = dims[j]; + const auto& dim_size = output_dims[j]; + + for(size_t o = 0; o < index_numel; o++) + { + int32_t getitem_index = indexs[j][o]; + + if(getitem_index >= 0 && getitem_index < dim_size) + { + element_index[(o * indexCount) + j] = getitem_index; + } + else if(getitem_index >= -dim_size && getitem_index < 0) + { + element_index[(o * indexCount) + j] = getitem_index + dim_size; + } + else + { + errorhost[j] = -1; + } + + if(o == 0) + { + element_index[dim_info_offset + j] = index_dim; + } + } + } + + // GetItem + for(size_t o = 0; o < dy_numel; o++) + { + tensor_layout_t<5> ncdhw(dy_tv, o); + tensor_layout_t<5> idx(ncdhw); + + if(indexCount > 0) + { + size_t dim_cursor = ncdhw.layout[start_dim]; + size_t i = start_dim; + size_t j = 0; + + for(; i < start_dim + indexCount; ++i, ++j) + { + size_t dim_idx = element_index[dim_info_offset + j]; + idx.layout[dim_idx] = element_index[(dim_cursor * indexCount) + j]; + } + + i = element_index[dim_info_offset + indexCount - 1] + 1; + dim_cursor = start_dim + 1; + for(; i < 5; ++i, ++dim_cursor) + { + idx.layout[i] = ncdhw.layout[dim_cursor]; + } + } + + dxhost[dxhost_tv.get_tensor_view_idx(idx)] += dy[dy_tv.get_tensor_view_idx(ncdhw)]; + } + + return ret; +} + +template +class GetitemDriver : public Driver +{ +public: + GetitemDriver() : Driver() + { + miopenCreateTensorDescriptor(&dyDesc); + miopenCreateTensorDescriptor(&dxDesc); + miopenCreateTensorDescriptor(&errorDesc); + + data_type = miopen_type{}; + } + + int AddCmdLineArgs() override; + int ParseCmdLineArgs(int argc, char* argv[]) override; + InputFlags& GetInputFlags() override { return inflags; } + + int GetandSetData() override; + + int AllocateBuffersAndCopy() override; + + int RunForwardGPU() override; + + int RunBackwardGPU() override; + int RunBackwardCPU(); + + Tref GetTolerance(); + + int VerifyBackward() override; + int VerifyForward() override; + ~GetitemDriver() override + { + miopenDestroyTensorDescriptor(dyDesc); + for(auto indexDesc : indexDescs) + { + miopenDestroyTensorDescriptor(indexDesc); + } + miopenDestroyTensorDescriptor(dxDesc); + miopenDestroyTensorDescriptor(errorDesc); + } + +private: + InputFlags inflags; + + miopenTensorDescriptor_t dyDesc; + std::vector indexDescs; + miopenTensorDescriptor_t dxDesc; + miopenTensorDescriptor_t errorDesc; + + std::unique_ptr dy_dev; + std::vector> index_devs; + std::unique_ptr dx_dev; + std::unique_ptr error_dev; + std::unique_ptr workspace_dev; + + std::vector dy; + std::vector> indexs; + std::vector dx; + std::vector error; + std::vector workspace; + std::vector dxhost; + std::vector errorhost; + + size_t ws_sizeInBytes; + + std::vector dims; + std::vector> slices; + std::vector slices_flat; + uint32_t offset; + + std::vector output_dims; + std::vector index_devs_ptr; + std::vector indexs_ptr; +}; + +template +int GetitemDriver::ParseCmdLineArgs(int argc, char* argv[]) +{ + inflags.Parse(argc, argv); + + if(inflags.GetValueInt("time") == 1) + { + miopenEnableProfiling(GetHandle(), true); + } + + if(inflags.GetValueInt("indexcount") < 0) + MIOPEN_THROW("Index count is negative: " + inflags.GetValueStr("indexcount") + "."); + + if(inflags.GetValueInt("dimcount") < 0) + MIOPEN_THROW("Dim count is negative: " + inflags.GetValueStr("dimcount") + "."); + + if(inflags.GetValueInt("slicecount") < 0) + MIOPEN_THROW("Slice count is negative: " + inflags.GetValueStr("slicecount") + "."); + + return miopenStatusSuccess; +} + +template +int GetitemDriver::GetandSetData() +{ + auto dyTensorParam = inflags.GetValueTensorUint64("doutput"); + auto dxTensorParam = inflags.GetValueTensorUint64("dinput"); + auto indexCountParam = inflags.GetValueInt("indexcount"); + auto dimCountParam = inflags.GetValueInt("dimcount"); + auto sliceCountParam = inflags.GetValueInt("slicecount"); + offset = inflags.GetValueInt("offset"); + + auto indexTensorLengths = inflags.GetValue2dVectorInt("indexs"); + if(indexTensorLengths.size() != indexCountParam) + MIOPEN_THROW("Error parsing indexs tensor: " + inflags.GetValueStr("indexs") + "."); + + dims = inflags.GetValueVectorInt("dims"); + if(dims.size() != dimCountParam) + MIOPEN_THROW("Error parsing dims tensor: " + inflags.GetValueStr("dims") + "."); + + for(auto dim : dims) + { + output_dims.push_back(dxTensorParam.lengths[dim]); + } + + slices = inflags.GetValue2dVectorInt("slices"); + if(slices.size() != sliceCountParam) + MIOPEN_THROW("Error parsing slices: " + inflags.GetValueStr("slices") + "."); + + for(auto slice : slices) + { + for(int32_t i = 0; i < 4; i++) + { + slices_flat.push_back(slice[i]); + } + } + + if(SetTensorNd(dyDesc, dyTensorParam.lengths, data_type) != miopenStatusSuccess) + MIOPEN_THROW("Error parsing doutput tensor: " + inflags.GetValueStr("doutput") + "."); + + for(auto indexTensorLength : indexTensorLengths) + { + miopenTensorDescriptor_t indexDesc; + miopenCreateTensorDescriptor(&indexDesc); + if(SetTensorNd(indexDesc, indexTensorLength, miopenInt32) != miopenStatusSuccess) + MIOPEN_THROW("Error parsing indexs tensor: " + inflags.GetValueStr("indexs") + "."); + indexDescs.push_back(indexDesc); + } + + if(SetTensorNd(dxDesc, dxTensorParam.lengths, data_type) != miopenStatusSuccess) + MIOPEN_THROW("Error parsing dinput tensor: " + inflags.GetValueStr("dinput") + "."); + + std::vector error_length; + error_length.push_back(indexCountParam); + if(SetTensorNd(errorDesc, error_length, miopen_type{}) != miopenStatusSuccess) + MIOPEN_THROW("Error making error tensor: " + inflags.GetValueStr("indexcount") + "."); + + return 0; +} + +template +int GetitemDriver::AddCmdLineArgs() +{ + inflags.AddInputFlag("forw", 'F', "0", "Run only Forward Getitem (Default=0)", "int"); + inflags.AddTensorFlag("doutput", 'O', "128x128", "doutput tensor descriptor"); + inflags.AddTensorFlag("indexs", 'D', "128", "indexs tensor descriptor"); + inflags.AddTensorFlag("dinput", 'N', "128x128", "dinput tensor descriptor"); + + inflags.AddInputFlag("indexcount", '1', "1", "the number of indexs tensor(Default=1)", "int"); + inflags.AddInputFlag("dimcount", '2', "1", "The dimensions(Default=1)", "int"); + inflags.AddInputFlag("dims", '3', "0", "The dimensions(Default=0)", "vector"); + inflags.AddInputFlag("slicecount", '4', "0", "The number of slices(Default=0)", "int"); + inflags.AddInputFlag("slices", + '5', + "", + "The slices(Default=\'\'" + ")", + "vector>"); + inflags.AddInputFlag("offset", '6', "0", "The offset of output(Default=0)", "int"); + + inflags.AddInputFlag("iter", 'i', "10", "Number of Iterations (Default=10)", "int"); + inflags.AddInputFlag("verify", 'V', "1", "Verify Each Layer (Default=1)", "int"); + inflags.AddInputFlag("time", 't', "0", "Time Each Layer (Default=0)", "int"); + inflags.AddInputFlag( + "wall", 'w', "0", "Wall-clock Time Each Layer, Requires time == 1 (Default=0)", "int"); + + return miopenStatusSuccess; +} + +template +int GetitemDriver::AllocateBuffersAndCopy() +{ + size_t dy_sz = GetTensorSize(dyDesc); + size_t dx_sz = GetTensorSize(dxDesc); + size_t error_sz = GetTensorSize(errorDesc); + + miopenGetGetitemWorkspaceSize( + GetHandle(), indexDescs.size(), indexDescs.data(), &ws_sizeInBytes); + + uint32_t ctx = 0; + + dy_dev = std::unique_ptr(new GPUMem(ctx, dy_sz, sizeof(Tgpu))); + dx_dev = std::unique_ptr(new GPUMem(ctx, dx_sz, sizeof(Tgpu))); + error_dev = std::unique_ptr(new GPUMem(ctx, error_sz, sizeof(int32_t))); + workspace_dev = std::unique_ptr(new GPUMem(ctx, ws_sizeInBytes, sizeof(std::byte))); + + dy = std::vector(dy_sz, static_cast(0)); + dx = std::vector(dx_sz, static_cast(0)); + error = std::vector(error_sz, static_cast(0)); + workspace = std::vector(ws_sizeInBytes / sizeof(int32_t), static_cast(0)); + dxhost = std::vector(dx_sz, static_cast(0)); + errorhost = std::vector(error_sz, static_cast(0)); + + for(int32_t i = 0; i < dy_sz; i++) + { + dy[i] = prng::gen_A_to_B(static_cast(-1), static_cast(1)); + } + + for(int32_t i = 0; i < indexDescs.size(); i++) + { + size_t index_sz = GetTensorSize(indexDescs[i]); + index_devs.push_back(std::unique_ptr(new GPUMem(ctx, index_sz, sizeof(int32_t)))); + indexs.push_back(std::vector(index_sz, static_cast(0))); + auto& index = indexs.back(); + auto index_dev = index_devs.back().get(); + + for(int j = 0; j < index_sz; j++) + { + index[j] = prng::gen_A_to_B(static_cast(0), + static_cast(output_dims[i])); + } + if(index_dev->ToGPU(GetStream(), index.data()) != 0) + std::cerr << "Error copying (index) to GPU, size: " << index_dev->GetSize() + << std::endl; + index_devs_ptr.push_back(index_dev->GetMem()); + indexs_ptr.push_back(index.data()); + } + + if(dy_dev->ToGPU(GetStream(), dy.data()) != 0) + std::cerr << "Error copying (dy) to GPU, size: " << dy_dev->GetSize() << std::endl; + + if(workspace_dev->ToGPU(GetStream(), workspace.data()) != 0) + std::cerr << "Error copying (workspace) to GPU, size: " << workspace_dev->GetSize() + << std::endl; + + if(dx_dev->ToGPU(GetStream(), dx.data()) != 0) + std::cerr << "Error copying (dx) to GPU, size: " << dx_dev->GetSize() << std::endl; + + if(error_dev->ToGPU(GetStream(), error.data()) != 0) + std::cerr << "Error copying (error) to GPU, size: " << error_dev->GetSize() << std::endl; + + return miopenStatusSuccess; +} + +template +int GetitemDriver::RunForwardGPU() +{ + return miopenStatusSuccess; +} + +template +int GetitemDriver::RunBackwardGPU() +{ + float kernel_total_time = 0; + float kernel_first_time = 0; + + Timer t; + START_TIME + + for(int32_t i = 0; i < inflags.GetValueInt("iter"); i++) + { + + if(dx_dev->ToGPU(GetStream(), dx.data()) != 0) + std::cerr << "Error copying (dx) to GPU, size: " << dx_dev->GetSize() << std::endl; + + miopenGetitemBackward(GetHandle(), + workspace_dev->GetMem(), + ws_sizeInBytes, + dyDesc, + dy_dev->GetMem(), + indexDescs.size(), + indexDescs.data(), + index_devs_ptr.data(), + dxDesc, + dx_dev->GetMem(), + errorDesc, + error_dev->GetMem(), + dims.size(), + dims.data(), + slices.size(), + slices_flat.data(), + offset); + + float time = 0; + miopenGetKernelTime(GetHandle(), &time); + kernel_total_time += time; + if(i == 0) + kernel_first_time = time; + } + + if(inflags.GetValueInt("time") == 1) + { + STOP_TIME + int32_t iter = inflags.GetValueInt("iter"); + if(WALL_CLOCK) + std::cout << "Wall-clock Time Backward Getitem Elapsed: " << t.gettime_ms() / iter + << " ms" << std::endl; + + float kernel_average_time = + iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; + std::cout << "GPU Kernel Time Backward Getitem Elapsed: " << kernel_average_time << " ms" + << std::endl; + } + + if(dx_dev->FromGPU(GetStream(), dx.data()) != 0) + std::cerr << "Error copying (dx_dev) from GPU, size: " << dx_dev->GetSize() << std::endl; + + if(error_dev->FromGPU(GetStream(), error.data()) != 0) + std::cerr << "Error copying (error_dev) from GPU, size: " << error_dev->GetSize() + << std::endl; + + return miopenStatusSuccess; +} + +template +int GetitemDriver::RunBackwardCPU() +{ + mloGetitemBackwardRunHost(dyDesc, + indexDescs.size(), + indexDescs.data(), + dxDesc, + errorDesc, + dy.data(), + indexs_ptr.data(), + dxhost.data(), + errorhost.data(), + dims.size(), + dims.data(), + slices.size(), + slices_flat.data(), + offset); + + return miopenStatusSuccess; +} + +template +Tref GetitemDriver::GetTolerance() +{ + // Computation error of fp16 is ~2^13 (=8192) bigger than + // the one of fp32 because mantissa is shorter by 13 bits. + // In the case of layernorm, there is a cumulative sum operation, and in the case of + // floating point operation, the result value can change if the order of the summed values + // is changed. So apply a threshold that is 10 times larger than other operations. + auto tolerance = std::is_same::value ? 1.5e-4 : 8.2e-1; + + // bf16 mantissa has 7 bits, by 3 bits shorter than fp16. + // If there is an atomic operation on the GPU kernel, a large error occurs depending on the + // calculation order, so it is multiplied by 10 times. + if(std::is_same::value) + tolerance *= 8000.0; + return tolerance; +} + +template +int GetitemDriver::VerifyForward() +{ + return miopenStatusSuccess; +} + +template +int GetitemDriver::VerifyBackward() +{ + RunBackwardCPU(); + const Tref tolerance = GetTolerance(); + + auto error_dx = miopen::rms_range(dxhost, dx); + + if(!std::isfinite(error_dx) || error_dx > tolerance) + { + std::cout << "Backward Getitem FAILED: " << error_dx << " > " << tolerance << std::endl; + return EC_VerifyBwd; + } + else + { + std::cout << "Backward Getitem Verifies OK on CPU reference (" << error_dx << " < " + << tolerance << ')' << std::endl; + } + + auto error_error = miopen::rms_range(errorhost, error); + + if(!std::isfinite(error_error) || std::abs(static_cast(error_error)) != 0.0f) + { + std::cout << "Backward Getitem FAILED: Result does not equal" << std::endl; + return EC_VerifyBwd; + } + else + { + std::cout << "Backward Getitem Verifies OK on CPU and GPU" << std::endl; + } + + return miopenStatusSuccess; +} + +#endif // GUARD_MIOPEN_GETITEM_DRIVER_HPP diff --git a/driver/glu_driver.hpp b/driver/glu_driver.hpp new file mode 100644 index 0000000000..38deb2d69e --- /dev/null +++ b/driver/glu_driver.hpp @@ -0,0 +1,482 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include "InputFlags.hpp" +#include "driver.hpp" +#include "tensor_driver.hpp" +#include "timer.hpp" +#include "random.hpp" + +#include +#include +#include +#include +#include + +#include <../test/verify.hpp> + +#include +#include + +template +T sigmoid(T x) +{ + return 1.0f / (1.0f + exp(-x)); +} + +template +int mloGLUForwardContiguousDim0RunHost(const Tgpu* input, + miopenTensorDescriptor_t outputDesc, + Tcheck* outputHost) +{ + auto output_numel = miopen::deref(outputDesc).GetElementSize(); + auto inputFirstHalf = input; + auto inputSecondHalf = input + output_numel; + + int ret = 0; + + for(size_t o = 0; o < output_numel; o++) + { + Tcheck valA = static_cast(inputFirstHalf[o]); + Tcheck valB = static_cast(inputSecondHalf[o]); + Tcheck val = valA * sigmoid(valB); + outputHost[o] = val; + } + + return ret; +} + +template +int mloGLUBackwardCongiguousDim0RunHost(const Tgpu* input, + miopenTensorDescriptor_t outputGradDesc, + const Tgpu* outputGrad, + Tcheck* inputGradHost) +{ + int ret = 0; + + auto outputGrad_numel = miopen::deref(outputGradDesc).GetElementSize(); + auto inputFirstHalf = input; + auto inputSecondHalf = input + outputGrad_numel; + auto inputFistHalf_grad = inputGradHost; + auto inputSecondHalf_grad = inputGradHost + outputGrad_numel; + + for(size_t o = 0; o < outputGrad_numel; o++) + { + Tcheck inputFirstHalf_v = static_cast(inputFirstHalf[o]); + Tcheck sigmoid_v = sigmoid(static_cast(inputSecondHalf[o])); + Tcheck grad_v = static_cast(outputGrad[o]); + + inputFistHalf_grad[o] = sigmoid_v * grad_v; + inputSecondHalf_grad[o] = (1 - sigmoid_v) * sigmoid_v * grad_v * inputFirstHalf_v; + } + + return ret; +} + +template +class GLUDriver : public Driver +{ +public: + GLUDriver() : Driver() + { + miopenCreateTensorDescriptor(&inputTensor); + miopenCreateTensorDescriptor(&outputTensor); + miopenCreateTensorDescriptor(&inputTensorGrad); + miopenCreateTensorDescriptor(&outputTensorGrad); + + data_type = miopen_type{}; + } + + int AddCmdLineArgs() override; + int ParseCmdLineArgs(int argc, char* argv[]) override; + InputFlags& GetInputFlags() override { return inflags; } + + int GetandSetData() override; + std::vector GetInputTensorLengthsFromCmdLine(); + + int AllocateBuffersAndCopy() override; + + int RunForwardGPU() override; + int RunForwardCPU(); // Verify implements it + + int RunBackwardGPU() override; + int RunBackwardCPU(); // Verify implements it + + Tref GetTolerance(); + + int VerifyBackward() override; + int VerifyForward() override; + ~GLUDriver() override + { + miopenDestroyTensorDescriptor(outputTensor); + miopenDestroyTensorDescriptor(inputTensor); + miopenDestroyTensorDescriptor(inputTensorGrad); + miopenDestroyTensorDescriptor(outputTensorGrad); + } + +private: + InputFlags inflags; + + int forw; + + miopenTensorDescriptor_t inputTensor; + miopenTensorDescriptor_t outputTensor; + + // Backwards + miopenTensorDescriptor_t inputTensorGrad; + miopenTensorDescriptor_t outputTensorGrad; + + std::unique_ptr in_dev; + std::unique_ptr out_dev; + + std::unique_ptr inGrad_dev; + std::unique_ptr outGrad_dev; + + std::vector in; + std::vector out; + std::vector outhost; + + std::vector inGrad; + std::vector outGrad; + std::vector inGradhost; + + uint32_t dim; +}; + +template +int GLUDriver::ParseCmdLineArgs(int argc, char* argv[]) +{ + inflags.Parse(argc, argv); + + if(inflags.GetValueInt("time") == 1) + { + miopenEnableProfiling(GetHandle(), true); + } + + forw = inflags.GetValueInt("forw"); + + if(forw != 0 && forw != 1) + { + MIOPEN_THROW("Invalid Forward Mode"); + } + + return miopenStatusSuccess; +} + +template +int GLUDriver::GetandSetData() +{ + std::vector in_len = inflags.GetValueTensor("dim_lengths").lengths; + dim = inflags.GetValueInt("dim_to_split"); + + SetTensorNd(inputTensor, in_len, data_type); + + std::vector out_len; + + for(int i = 0; i < in_len.size(); i++) + { + if(i != dim) + { + out_len.push_back(in_len[i]); + } + else + { + out_len.push_back(in_len[i] / 2); + } + } + + if(out_len.empty()) + out_len.push_back(1); + + SetTensorNd(outputTensor, out_len, data_type); + + // Backwards + SetTensorNd(inputTensorGrad, in_len, data_type); + SetTensorNd(outputTensorGrad, out_len, data_type); + + return miopenStatusSuccess; +} + +template +int GLUDriver::AddCmdLineArgs() +{ + inflags.AddInputFlag("forw", + 'F', + "1", + "Run only Forward (1) or Run both Forward and Backward (0) (Default=1)", + "int"); + inflags.AddTensorFlag( + "dim_lengths", 'D', "256x512", "The dimensional lengths of the input tensor"); + inflags.AddInputFlag("dim_to_split", + 'R', + "0", + "The indice of the dimensions to be split half(Default=0)", + "int"); + inflags.AddInputFlag("iter", 'i', "10", "Number of Iterations (Default=10)", "int"); + inflags.AddInputFlag("verify", 'V', "1", "Verify Each Layer (Default=1)", "int"); + inflags.AddInputFlag("time", 't', "0", "Time Each Layer (Default=0)", "int"); + inflags.AddInputFlag( + "wall", 'w', "0", "Wall-clock Time Each Layer, Requires time == 1 (Default=0)", "int"); + + return miopenStatusSuccess; +} + +template +int GLUDriver::AllocateBuffersAndCopy() +{ + uint32_t ctx = 0; + + size_t in_sz = GetTensorSpace(inputTensor); + + if(forw == 1) + { + size_t out_sz = GetTensorSpace(outputTensor); + + // GPU allocation + in_dev = std::unique_ptr(new GPUMem(ctx, in_sz, sizeof(Tgpu))); + out_dev = std::unique_ptr(new GPUMem(ctx, out_sz, sizeof(Tgpu))); + + // GPU host allocation + in = std::vector(in_sz, static_cast(0)); + out = std::vector(out_sz, static_cast(0)); + + // CPU allocation + outhost = std::vector(out_sz, static_cast(0)); + + for(int i = 0; i < in_sz; i++) + { + in[i] = prng::gen_A_to_B(static_cast(0.0), static_cast(1.0)); + } + + if(in_dev->ToGPU(GetStream(), in.data()) != 0) + std::cerr << "Error copying (input) to GPU, size: " << in_dev->GetSize() << std::endl; + + if(out_dev->ToGPU(GetStream(), out.data()) != 0) + std::cerr << "Error copying (out) to GPU, size: " << out_dev->GetSize() << std::endl; + } + + if(forw == 0) + { + size_t out_sz = GetTensorSpace(outputTensor); + size_t inGrad_sz = GetTensorSpace(inputTensorGrad); + size_t outGrad_sz = GetTensorSpace(outputTensorGrad); + + // GPU allocation + in_dev = std::unique_ptr(new GPUMem(ctx, in_sz, sizeof(Tgpu))); + out_dev = std::unique_ptr(new GPUMem(ctx, out_sz, sizeof(Tgpu))); + inGrad_dev = std::unique_ptr(new GPUMem(ctx, inGrad_sz, sizeof(Tgpu))); + outGrad_dev = std::unique_ptr(new GPUMem(ctx, outGrad_sz, sizeof(Tgpu))); + + // GPU host allocation + in = std::vector(in_sz, static_cast(0)); + out = std::vector(out_sz, static_cast(0)); + inGrad = std::vector(inGrad_sz, static_cast(0)); + outGrad = std::vector(outGrad_sz, static_cast(0)); + + // CPU allocation + outhost = std::vector(out_sz, static_cast(0)); + inGradhost = std::vector(inGrad_sz, static_cast(0)); + + for(int i = 0; i < in_sz; i++) + { + in[i] = prng::gen_A_to_B(static_cast(0.0), static_cast(1.0)); + } + for(int i = 0; i < outGrad_sz; i++) + { + outGrad[i] = prng::gen_A_to_B(static_cast(0.0), static_cast(1.0)); + } + + if(in_dev->ToGPU(GetStream(), in.data()) != 0) + std::cerr << "Error copying (input) to GPU, size: " << in_dev->GetSize() << std::endl; + if(out_dev->ToGPU(GetStream(), out.data()) != 0) + std::cerr << "Error copying (out) to GPU, size: " << out_dev->GetSize() << std::endl; + if(outGrad_dev->ToGPU(GetStream(), outGrad.data()) != 0) + std::cerr << "Error copying (output gradient) to GPU, size: " << outGrad_dev->GetSize() + << std::endl; + if(inGrad_dev->ToGPU(GetStream(), inGrad.data()) != 0) + std::cerr << "Error copying (input gradient) to GPU, size: " << inGrad_dev->GetSize() + << std::endl; + } + + return miopenStatusSuccess; +} + +template +int GLUDriver::RunForwardGPU() +{ + float kernel_total_time = 0; + float kernel_first_time = 0; + + Timer t; + START_TIME + + for(int i = 0; i < inflags.GetValueInt("iter"); i++) + { + miopenStatus_t status = miopenGLUForward( + GetHandle(), inputTensor, in_dev->GetMem(), outputTensor, out_dev->GetMem(), dim); + + MIOPEN_THROW_IF(status != miopenStatusSuccess, "Error in miopenGLUForward"); + + float time = 0.0; + miopenGetKernelTime(GetHandle(), &time); + kernel_total_time += time; + if(i == 0) + kernel_first_time = time; + } + + if(inflags.GetValueInt("time") == 1) + { + STOP_TIME + int iter = inflags.GetValueInt("iter"); + if(WALL_CLOCK) + std::cout << "Wall-clock Time Forward GLU Elapsed: " << t.gettime_ms() / iter + << " ms\n"; + + float kernel_average_time = + iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; + std::cout << "GPU Kernel Time Forward GLU Elapsed: " << kernel_average_time << " ms\n"; + } + + if(out_dev->FromGPU(GetStream(), out.data()) != 0) + std::cerr << "Error copying (out_dev) from GPU, size: " << out_dev->GetSize() << std::endl; + + return miopenStatusSuccess; +} + +template +int GLUDriver::RunForwardCPU() +{ + MIOPEN_THROW_IF(dim != 0, "This driver only supports dim = 0"); + mloGLUForwardContiguousDim0RunHost(in.data(), outputTensor, outhost.data()); + + return miopenStatusSuccess; +} + +template +int GLUDriver::RunBackwardGPU() +{ + float kernel_total_time = 0; + float kernel_first_time = 0; + Timer t; + START_TIME; + for(int i = 0; i < inflags.GetValueInt("iter"); i++) + { + miopenStatus_t status = miopenGLUBackward(GetHandle(), + inputTensor, + in_dev->GetMem(), + outputTensorGrad, + outGrad_dev->GetMem(), + inputTensorGrad, + inGrad_dev->GetMem(), + dim); + + MIOPEN_THROW_IF(status != miopenStatusSuccess, "Error in miopenGLUBackward"); + + float time = 0.0; + miopenGetKernelTime(GetHandle(), &time); + kernel_total_time += time; + if(i == 0) + kernel_first_time = time; + } + + if(inflags.GetValueInt("time") == 1) + { + STOP_TIME + int iter = inflags.GetValueInt("iter"); + if(WALL_CLOCK) + std::cout << "Wall-clock Time Backward GLU Elapsed: " << t.gettime_ms() / iter + << " ms\n"; + float kernel_average_time = + iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; + std::cout << "GPU Kernel Time Backward GLU Elapsed: " << kernel_average_time << " ms\n"; + } + + if(inGrad_dev->FromGPU(GetStream(), inGrad.data()) != 0) + std::cerr << "Error copying (out_dev) from GPU, size: " << inGrad_dev->GetSize() + << std::endl; + + return miopenStatusSuccess; +} + +template +Tref GLUDriver::GetTolerance() +{ + Tref tolerance = std::numeric_limits::epsilon() * 10; + return tolerance; +} + +template +int GLUDriver::VerifyForward() +{ + RunForwardCPU(); + const Tref tolerance = GetTolerance(); + auto error = miopen::rms_range(outhost, out); + + if(!std::isfinite(error) || error > tolerance) + { + std::cout << "Forward GLU FAILED: " << error << " > " << tolerance << std::endl; + return EC_VerifyFwd; + } + else + { + std::cout << "Forward GLU Verifies OK on CPU reference (" << error << " < " << tolerance + << ')' << std::endl; + } + + return miopenStatusSuccess; +} + +template +int GLUDriver::RunBackwardCPU() +{ + MIOPEN_THROW_IF(dim != 0, "This driver only supports dim = 0"); + mloGLUBackwardCongiguousDim0RunHost( + in.data(), outputTensorGrad, outGrad.data(), inGradhost.data()); + + return miopenStatusSuccess; +} + +template +int GLUDriver::VerifyBackward() +{ + RunBackwardCPU(); + const Tref tolerance = GetTolerance(); + auto error = miopen::rms_range(inGradhost, inGrad); + if(!std::isfinite(error) || error > tolerance) + { + std::cout << "Backward GLU FAILED: " << error << " > " << tolerance << std::endl; + return EC_VerifyBwd; + } + else + { + std::cout << "Backward GLU Verifies OK on CPU reference (" << error << " < " << tolerance + << ')' << std::endl; + } + + return miopenStatusSuccess; +} diff --git a/driver/groupnorm_driver.hpp b/driver/groupnorm_driver.hpp index c143496cdd..15717aa776 100644 --- a/driver/groupnorm_driver.hpp +++ b/driver/groupnorm_driver.hpp @@ -89,7 +89,6 @@ class GroupNormDriver : public Driver private: InputFlags inflags; - int forw; int dim_size; miopenTensorDescriptor_t inputDesc; @@ -110,8 +109,8 @@ class GroupNormDriver : public Driver std::vector weight; std::vector bias; std::vector out; - std::vector mean; - std::vector rstd; + std::vector mean; + std::vector rstd; std::vector outhost; std::vector meanhost; std::vector rstdhost; @@ -158,14 +157,14 @@ template int GroupNormDriver::AddCmdLineArgs() { inflags.AddInputFlag("forw", 'F', "1", "Run only Forward GroupNorm (Default=1)", "int"); - inflags.AddInputFlag("batchsize", 'n', "100", "Mini-batch size (Default=100)", "int"); - inflags.AddInputFlag("in_channels", 'c', "6", "Number of Input Channels (Default=6)", "int"); - inflags.AddInputFlag("in_d", 'D', "0", "Input Depth (Default=0)", "int"); - inflags.AddInputFlag("in_h", 'H', "32", "Input Height (Default=32)", "int"); - inflags.AddInputFlag("in_w", 'W', "32", "Input Width (Default=32)", "int"); + inflags.AddInputFlag("batchsize", 'n', "32", "Mini-batch size (Default=100)", "int"); + inflags.AddInputFlag("in_channels", 'c', "32", "Number of Input Channels (Default=6)", "int"); + inflags.AddInputFlag("in_d", 'D', "14", "Input Depth (Default=0)", "int"); + inflags.AddInputFlag("in_h", 'H', "14", "Input Height (Default=32)", "int"); + inflags.AddInputFlag("in_w", 'W', "14", "Input Width (Default=32)", "int"); inflags.AddInputFlag("eps", 'e', "0.00001", "Alpha (Default=0.00001)", "double"); - inflags.AddInputFlag("num_groups", 'g', "3", "num_groups", "int"); + inflags.AddInputFlag("num_groups", 'g', "4", "num_groups", "int"); inflags.AddInputFlag( "mode", 'm', "0", "elemwise affine mode (0), weight and bias mode (1) (Default=0)", "int"); @@ -224,15 +223,15 @@ int GroupNormDriver::AllocateBuffersAndCopy() weight_dev = std::unique_ptr(new GPUMem(ctx, weight_sz, sizeof(Tgpu))); bias_dev = std::unique_ptr(new GPUMem(ctx, bias_sz, sizeof(Tgpu))); out_dev = std::unique_ptr(new GPUMem(ctx, out_sz, sizeof(Tgpu))); - mean_dev = std::unique_ptr(new GPUMem(ctx, mean_sz, sizeof(Tref))); - rstd_dev = std::unique_ptr(new GPUMem(ctx, rstd_sz, sizeof(Tref))); + mean_dev = std::unique_ptr(new GPUMem(ctx, mean_sz, sizeof(Tgpu))); + rstd_dev = std::unique_ptr(new GPUMem(ctx, rstd_sz, sizeof(Tgpu))); in = std::vector(in_sz, static_cast(0)); weight = std::vector(weight_sz, static_cast(0)); bias = std::vector(bias_sz, static_cast(0)); out = std::vector(out_sz, static_cast(0)); - mean = std::vector(mean_sz, static_cast(0)); - rstd = std::vector(rstd_sz, static_cast(0)); + mean = std::vector(mean_sz, static_cast(0)); + rstd = std::vector(rstd_sz, static_cast(0)); outhost = std::vector(out_sz, static_cast(0)); meanhost = std::vector(mean_sz, static_cast(0)); rstdhost = std::vector(rstd_sz, static_cast(0)); @@ -347,23 +346,14 @@ int GroupNormDriver::RunBackwardGPU() template Tref GroupNormDriver::GetTolerance() { - if(data_type == miopenHalf) - { - return 1e-3; - } - else if(data_type == miopenFloat) - { - return 5e-5; - } - else if(data_type == miopenDouble) - { - return 1e-10; - } - else if(data_type == miopenBFloat16) - { - return 5e-3; - } - return 0; + // Computation error of fp16 is ~2^13 (=8192) bigger than + // the one of fp32 because mantissa is shorter by 13 bits. + auto tolerance = std::is_same::value ? 1.5e-6 : 8.2e-3; + + // bf16 mantissa has 7 bits, by 3 bits shorter than fp16. + if(std::is_same::value) + tolerance *= 8.0; + return tolerance; } template diff --git a/driver/gru_verify_gemm.hpp b/driver/gru_verify_gemm.hpp index 3b6573f566..6266070be8 100644 --- a/driver/gru_verify_gemm.hpp +++ b/driver/gru_verify_gemm.hpp @@ -104,7 +104,7 @@ void RunGRUForwardGEMMCPUVerify(miopenHandle_t handle, } // initial dropoput - std::vector dropout_states_host; + std::vector dropout_states_host; std::vector dropout_reservespace_host; std::vector dropout_hid_state; miopenTensorDescriptor_t dropout_inputTensor{}, dropout_outputTensor{}; @@ -112,8 +112,8 @@ void RunGRUForwardGEMMCPUVerify(miopenHandle_t handle, { size_t statesSizeInBytes = 0; miopenDropoutGetStatesSize(handle, &statesSizeInBytes); - size_t states_size = statesSizeInBytes / sizeof(prngStates); - dropout_states_host = std::vector(states_size); + size_t states_size = statesSizeInBytes / sizeof(rocrand_state_xorwow); + dropout_states_host = std::vector(states_size); InitKernelStateEmulator(dropout_states_host, dropoutDesc); std::array drop_in_len = {{batch_n, hy_h * bi}}; diff --git a/driver/kthvalue_driver.hpp b/driver/kthvalue_driver.hpp new file mode 100644 index 0000000000..75f7e5b535 --- /dev/null +++ b/driver/kthvalue_driver.hpp @@ -0,0 +1,387 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once +#include "InputFlags.hpp" +#include "driver.hpp" +#include "tensor_driver.hpp" +#include "timer.hpp" +#include "random.hpp" + +#include <../test/tensor_holder.hpp> +#include <../test/verify.hpp> + +#include +#include +#include + +#include + +template +void mloKthvalueFwdRunHost(TIO* input, + miopenTensorDescriptor_t pInputDesc, + TIO* outputHost, + miopenTensorDescriptor_t outputDesc, + size_t* indices, + miopenTensorDescriptor_t indicesDesc, + size_t k, + int dim) +{ + auto inputDesc = miopen::deref(pInputDesc); + size_t inputSize = inputDesc.GetElementSize(); + size_t dimSize = inputDesc.GetLengths()[dim]; + size_t dimStride = inputDesc.GetStrides()[dim]; + auto inputTv = miopen::get_inner_expanded_tv<5>(miopen::deref(pInputDesc)); + auto inputTvWithoutDim = miopen::get_tv_without_dim<5>(inputTv, dim); + auto outputTv = miopen::get_inner_expanded_tv<5>(miopen::deref(outputDesc)); + auto indicesTv = miopen::get_inner_expanded_tv<5>(miopen::deref(indicesDesc)); + + size_t numSlice = inputSize / dimSize; + + std::vector elements; + std::vector ids(dimSize); + for(int i = 0; i < dimSize; ++i) + { + ids[i] = i; + } + + for(int slideID = 0; slideID < numSlice; ++slideID) + { + elements.clear(); + tensor_layout_t<4> layout(inputTvWithoutDim, slideID); + auto idx = inputTvWithoutDim.get_tensor_view_idx(layout); + + for(int j = 0; j < dimSize; ++j) + { + elements.push_back(static_cast(input[idx + j * dimStride])); + } + + std::sort(ids.begin(), ids.end(), [=](size_t x, size_t y) -> bool { + return elements[x] < elements[y]; + }); + auto output_layout = tensor_layout_t<5>(outputTv, slideID); + auto indices_layout = tensor_layout_t<5>(indicesTv, slideID); + outputHost[outputTv.get_tensor_view_idx(output_layout)] = + static_cast(elements[ids[k - 1]]); + indices[indicesTv.get_tensor_view_idx(indices_layout)] = ids[k - 1]; + } +} + +template +class KthvalueDriver : public Driver +{ +public: + KthvalueDriver() : Driver() + { + miopenCreateTensorDescriptor(&inputDesc); + miopenCreateTensorDescriptor(&indicesDesc); + miopenCreateTensorDescriptor(&outputDesc); + + data_type = miopen_type{}; + } + + std::vector ComputeStrides(std::vector input); + int AddCmdLineArgs() override; + int ParseCmdLineArgs(int argc, char* argv[]) override; + InputFlags& GetInputFlags() override { return inflags; } + + int GetandSetData() override; + + int AllocateBuffersAndCopy() override; + + int RunForwardGPU() override; + int RunForwardCPU(); + + int RunBackwardGPU() override; + int RunBackwardCPU(); + + int VerifyBackward() override; + int VerifyForward() override; + ~KthvalueDriver() override + { + miopenDestroyTensorDescriptor(inputDesc); + miopenDestroyTensorDescriptor(indicesDesc); + miopenDestroyTensorDescriptor(outputDesc); + } + +private: + InputFlags inflags; + + miopenTensorDescriptor_t inputDesc; + miopenTensorDescriptor_t indicesDesc; + miopenTensorDescriptor_t outputDesc; + + std::unique_ptr input_dev; + std::unique_ptr indices_dev; + std::unique_ptr output_dev; + + std::vector input; + std::vector indices; + std::vector indicesHost; + std::vector output; + std::vector outputHost; + + bool isContiguous; + int dim; + size_t k; + bool keepDim; +}; + +template +int KthvalueDriver::ParseCmdLineArgs(int argc, char* argv[]) +{ + inflags.Parse(argc, argv); + isContiguous = inflags.GetValueInt("is-contiguous") == 1 ? true : false; + k = inflags.GetValueInt("k"); + dim = inflags.GetValueInt("dim"); + keepDim = inflags.GetValueInt("keep-dim") == 1 ? true : false; + auto inDims = inflags.GetValueTensor("dim-lengths").lengths; + int num_dim = inDims.size(); + if(dim < -num_dim || dim >= num_dim) + { + MIOPEN_THROW(miopenStatusBadParm, "Kthvalue: dim doesn't not exist"); + } + + if(inflags.GetValueInt("time") == 1) + { + miopenEnableProfiling(GetHandle(), true); + } + return miopenStatusSuccess; +} + +template +int KthvalueDriver::GetandSetData() +{ + auto inDims = inflags.GetValueTensor("dim-lengths").lengths; + std::vector inStride = ComputeStrides(inDims); + auto outDims = inflags.GetValueTensor("dim-lengths").lengths; + + if(dim < 0) + { + dim += inDims.size(); + } + if(!keepDim) + { + outDims.erase(outDims.begin() + dim); + if(outDims.empty()) + outDims.push_back(1); + } + else + { + outDims[dim] = 1; + } + + SetTensorNd(inputDesc, inDims, inStride, data_type); + SetTensorNd(outputDesc, outDims, data_type); + SetTensorNd(indicesDesc, outDims, miopenInt64); + + return 0; +} + +// Equivalent to: tensor.tranpose(0, -1).contiguous().tranpose(0, -1) incase contiguous = False +template +std::vector KthvalueDriver::ComputeStrides(std::vector inputDim) +{ + if(!isContiguous) + std::swap(inputDim.front(), inputDim.back()); + std::vector strides(inputDim.size()); + strides.back() = 1; + for(int i = inputDim.size() - 2; i >= 0; --i) + strides[i] = strides[i + 1] * inputDim[i + 1]; + if(!isContiguous) + std::swap(strides.front(), strides.back()); + return strides; +} + +template +int KthvalueDriver::AddCmdLineArgs() +{ + inflags.AddInputFlag("forw", 'F', "1", "Run only Forward (Default=1)", "int"); + inflags.AddTensorFlag( + "dim-lengths", 'D', "256x4x2", "The dimensional lengths of the input tensor"); + inflags.AddInputFlag("k", 'k', "1", "k (Default=1)", "int"); + inflags.AddInputFlag("dim", 'd', "-1", "dim (Default=-1)", "int"); + inflags.AddInputFlag("keep-dim", + 'K', + "0", + "Whether the output tensor has dim retained or not (Default=0)", + "int"); + inflags.AddInputFlag("is-contiguous", 'c', "1", "is-contiguous (Default=1)", "int"); + inflags.AddInputFlag("iter", 'i', "10", "Number of Iterations (Default=10)", "int"); + inflags.AddInputFlag("verify", 'V', "1", "Verify Each Layer (Default=1)", "int"); + inflags.AddInputFlag("time", 't', "0", "Time Each Layer (Default=0)", "int"); + inflags.AddInputFlag( + "wall", 'w', "0", "Wall-clock Time Each Layer, Requires time == 1 (Default=0)", "int"); + + return miopenStatusSuccess; +} + +template +int KthvalueDriver::AllocateBuffersAndCopy() +{ + size_t in_sz = miopen::deref(inputDesc).GetElementSize(); + size_t idx_sz = miopen::deref(indicesDesc).GetElementSize(); + size_t out_sz = miopen::deref(outputDesc).GetElementSize(); + + uint32_t ctx = 0; + + input_dev = std::unique_ptr(new GPUMem(ctx, in_sz, sizeof(TIO))); + indices_dev = std::unique_ptr(new GPUMem(ctx, idx_sz, sizeof(size_t))); + output_dev = std::unique_ptr(new GPUMem(ctx, out_sz, sizeof(TIO))); + + input = std::vector(in_sz, static_cast(0)); + indices = std::vector(idx_sz, 0); + indicesHost = std::vector(idx_sz, 0); + output = std::vector(out_sz, static_cast(0)); + outputHost = std::vector(out_sz, static_cast(0)); + + for(int i = 0; i < in_sz; i++) + { + input[i] = prng::gen_A_to_B(static_cast(-10), static_cast(10)); + } + + fill(output.begin(), output.end(), static_cast(0)); + fill(indices.begin(), indices.end(), static_cast(0)); + + if(input_dev->ToGPU(GetStream(), input.data()) != 0) + std::cerr << "Error copying (in) to GPU, size: " << input_dev->GetSize() << std::endl; + + if(indices_dev->ToGPU(GetStream(), indices.data()) != 0) + std::cerr << "Error copying (idx) to GPU, size: " << indices_dev->GetSize() << std::endl; + + if(output_dev->ToGPU(GetStream(), output.data()) != 0) + std::cerr << "Error copying (out) to GPU, size: " << output_dev->GetSize() << std::endl; + + return miopenStatusSuccess; +} + +template +int KthvalueDriver::RunForwardGPU() +{ + float kernel_total_time = 0; + float kernel_first_time = 0; + + Timer t; + START_TIME + + for(int i = 0; i < inflags.GetValueInt("iter"); i++) + { + miopenKthvalueForward(GetHandle(), + inputDesc, + input_dev->GetMem(), + outputDesc, + output_dev->GetMem(), + indicesDesc, + (size_t*)indices_dev->GetMem(), + k, + dim, + keepDim); + float time = 0.0; + miopenGetKernelTime(GetHandle(), &time); + kernel_total_time += time; + if(i == 0) + kernel_first_time = time; + } + + if(inflags.GetValueInt("time") == 1) + { + STOP_TIME + int iter = inflags.GetValueInt("iter"); + if(WALL_CLOCK) + std::cout << "Wall-clock Time Kthvalue Fwd Elapsed: " << t.gettime_ms() / iter << " ms" + << std::endl; + + float kernel_average_time = + iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; + std::cout << "GPU Kernel Time Kthvalue Fwd Elapsed: " << kernel_average_time << " ms" + << std::endl; + } + + if(output_dev->FromGPU(GetStream(), output.data()) != 0) + std::cerr << "Error copying (out_dev) from GPU, size: " << output_dev->GetSize() + << std::endl; + + if(indices_dev->FromGPU(GetStream(), indices.data()) != 0) + std::cerr << "Error copying (indices_dev) from GPU, size: " << indices_dev->GetSize() + << std::endl; + + return miopenStatusSuccess; +} + +template +int KthvalueDriver::RunForwardCPU() +{ + mloKthvalueFwdRunHost(input.data(), + inputDesc, + outputHost.data(), + outputDesc, + indicesHost.data(), + indicesDesc, + k, + dim); + + return miopenStatusSuccess; +} + +template +int KthvalueDriver::RunBackwardGPU() +{ + return miopenStatusSuccess; +} + +template +int KthvalueDriver::RunBackwardCPU() +{ + return miopenStatusSuccess; +} + +template +int KthvalueDriver::VerifyForward() +{ + RunForwardCPU(); + + double tolerance = std::numeric_limits::epsilon() * 10; + auto errorOutput = miopen::rms_range(outputHost, output); + + if(!std::isfinite(errorOutput) || errorOutput > tolerance) + { + std::cout << "Forward Kthvalue output FAILED: " << errorOutput << " > " << tolerance + << std::endl; + return EC_VerifyFwd; + } + else + { + std::cout << "Forward Kthvalue Verifies OK on CPU reference (" << errorOutput << "< " + << tolerance << ')' << std::endl; + } + + return miopenStatusSuccess; +} + +template +int KthvalueDriver::VerifyBackward() +{ + return miopenStatusSuccess; +} diff --git a/driver/layernorm_driver.hpp b/driver/layernorm_driver.hpp index ea5b841c08..cb5c787dbb 100644 --- a/driver/layernorm_driver.hpp +++ b/driver/layernorm_driver.hpp @@ -145,7 +145,6 @@ class LayerNormDriver : public Driver private: InputFlags inflags; - int forw; int dim_size; miopenTensorDescriptor_t inputDesc; @@ -166,8 +165,8 @@ class LayerNormDriver : public Driver std::vector weight; std::vector bias; std::vector out; - std::vector mean; - std::vector rstd; + std::vector mean; + std::vector rstd; std::vector outhost; std::vector meanhost; std::vector rstdhost; @@ -276,15 +275,15 @@ int LayerNormDriver::AllocateBuffersAndCopy() weight_dev = std::unique_ptr(new GPUMem(ctx, weight_sz, sizeof(Tgpu))); bias_dev = std::unique_ptr(new GPUMem(ctx, bias_sz, sizeof(Tgpu))); out_dev = std::unique_ptr(new GPUMem(ctx, out_sz, sizeof(Tgpu))); - mean_dev = std::unique_ptr(new GPUMem(ctx, mean_sz, sizeof(Tref))); - rstd_dev = std::unique_ptr(new GPUMem(ctx, rstd_sz, sizeof(Tref))); + mean_dev = std::unique_ptr(new GPUMem(ctx, mean_sz, sizeof(Tgpu))); + rstd_dev = std::unique_ptr(new GPUMem(ctx, rstd_sz, sizeof(Tgpu))); in = std::vector(in_sz, Tgpu0val); weight = std::vector(weight_sz, Tgpu0val); bias = std::vector(bias_sz, Tgpu0val); out = std::vector(out_sz, Tgpu0val); - mean = std::vector(mean_sz, Tref0ref); - rstd = std::vector(rstd_sz, Tref0ref); + mean = std::vector(mean_sz, Tgpu0val); + rstd = std::vector(rstd_sz, Tgpu0val); outhost = std::vector(out_sz, Tref0ref); meanhost = std::vector(mean_sz, Tref0ref); rstdhost = std::vector(rstd_sz, Tref0ref); diff --git a/driver/lstm_verify_gemm.hpp b/driver/lstm_verify_gemm.hpp index bd052c6142..9077c25af9 100644 --- a/driver/lstm_verify_gemm.hpp +++ b/driver/lstm_verify_gemm.hpp @@ -110,7 +110,7 @@ void RunLSTMForwardGEMMCPUVerify(miopenHandle_t handle, } // initial dropoput - std::vector dropout_states_host; + std::vector dropout_states_host; std::vector dropout_reservespace_host; std::vector dropout_hid_state; miopenTensorDescriptor_t dropout_inputTensor{}, dropout_outputTensor{}; @@ -118,8 +118,8 @@ void RunLSTMForwardGEMMCPUVerify(miopenHandle_t handle, { size_t statesSizeInBytes = 0; miopenDropoutGetStatesSize(handle, &statesSizeInBytes); - size_t states_size = statesSizeInBytes / sizeof(prngStates); - dropout_states_host = std::vector(states_size); + size_t states_size = statesSizeInBytes / sizeof(rocrand_state_xorwow); + dropout_states_host = std::vector(states_size); InitKernelStateEmulator(dropout_states_host, dropoutDesc); std::array drop_in_len = {{batch_n, hy_h * bi}}; diff --git a/driver/main.cpp b/driver/main.cpp index 6389cb2a84..14a98e1ff0 100644 --- a/driver/main.cpp +++ b/driver/main.cpp @@ -52,10 +52,10 @@ int main(int argc, char* argv[]) std::cout << " " << argv[i]; std::cout << std::endl; - Driver* drv = nullptr; + std::shared_ptr drv; for(auto f : rdm::GetRegistry()) { - drv = f(base_arg); + drv.reset(f(base_arg)); if(drv != nullptr) break; } diff --git a/driver/mloConvHost.hpp b/driver/mloConvHost.hpp index 537f2fb171..c0f6272801 100644 --- a/driver/mloConvHost.hpp +++ b/driver/mloConvHost.hpp @@ -30,7 +30,6 @@ #include #include -#include #include #include "calcerr.hpp" @@ -76,7 +75,7 @@ void ADNN_mm_cpu(const Dtype* a_ptr, (!(a_flags & ADNN_MM_TRANSPOSE) && (b_flags & ADNN_MM_TRANSPOSE) && ((a_cols != b_cols) || (a_rows != c_rows) || (b_rows != c_cols)))) { - printf("MM_CPU ERROR; %zd %zd %zd %zd %zd %zd\n", + printf("MM_CPU ERROR; %zu %zu %zu %zu %zu %zu\n", a_cols, a_rows, b_cols, diff --git a/driver/mloPReLUHost.hpp b/driver/mloPReLUHost.hpp new file mode 100644 index 0000000000..c3460b6425 --- /dev/null +++ b/driver/mloPReLUHost.hpp @@ -0,0 +1,104 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include <../test/ford.hpp> + +#include +#include +#include + +template +int32_t mloPReLUBackwardRunHost(const miopenTensorDescriptor_t inputDesc, + const miopenTensorDescriptor_t weightDesc, + const miopenTensorDescriptor_t doutputDesc, + const miopenTensorDescriptor_t dinputDesc, + const Tgpu* input, + const Tgpu* weight, + const Tgpu* doutput, + Tcheck* dinput_host, + Tcheck* dweight_host) +{ + auto input_tv = miopen::get_inner_expanded_tv<5>(miopen::deref(inputDesc)); + auto doutput_tv = miopen::get_inner_expanded_tv<5>(miopen::deref(doutputDesc)); + auto dinput_tv = miopen::get_inner_expanded_tv<5>(miopen::deref(dinputDesc)); + + auto input_sz = miopen::deref(inputDesc).GetElementSize(); + auto weight_sz = miopen::deref(weightDesc).GetElementSize(); + auto weight_grad_collector = std::vector(input_sz); + + par_ford(input_sz)([&](int gid) { + auto tensor_layout = tensor_layout_t<5>(input_tv, gid); + float input_v = static_cast(input[input_tv.get_tensor_view_idx(tensor_layout)]); + float grad_v = static_cast(doutput[doutput_tv.get_tensor_view_idx(tensor_layout)]); + + if(dinput_host) + { + float weight_v; + if(weight_sz == 1) + weight_v = static_cast(weight[0]); + else + weight_v = static_cast(weight[tensor_layout.layout[1]]); + float input_grad_v = input_v > 0 ? grad_v : weight_v * grad_v; + dinput_host[dinput_tv.get_tensor_view_idx(tensor_layout)] = + static_cast(input_grad_v); + } + if(dweight_host) + { + weight_grad_collector[gid] = input_v > 0 ? 0 : input_v * grad_v; + } + }); + + if(dweight_host) + { + if(weight_sz == 1) + { + double sum = 0; + for(int i = 0; i < input_sz; ++i) + sum += static_cast(weight_grad_collector[i]); + dweight_host[0] = static_cast(sum); + } + else + { + size_t inner_size = std::accumulate( + &input_tv.size[2], &input_tv.size[4], 1ul, std::multiplies()); + size_t outer_size = inner_size * input_tv.size[1]; + par_ford(weight_sz)([&](int i) { + double sum = 0; + ford(input_tv.size[0])([&](int j) { + ford(inner_size)([&](int k) { + sum += static_cast( + weight_grad_collector[j * outer_size + i * inner_size + k]); + }); + }); + dweight_host[i] = static_cast(sum); + }); + } + } + + return miopenStatusSuccess; +} diff --git a/driver/multimarginloss_driver.hpp b/driver/multimarginloss_driver.hpp new file mode 100644 index 0000000000..dab040ef3e --- /dev/null +++ b/driver/multimarginloss_driver.hpp @@ -0,0 +1,468 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include "InputFlags.hpp" +#include "driver.hpp" +#include "tensor_driver.hpp" +#include "timer.hpp" +#include "random.hpp" +#include +#include +#include +#include +#include +#include +#include <../test/tensor_holder.hpp> +#include <../test/verify.hpp> +#include + +template +int32_t mloMultiMarginLossForwardRunHost(const miopenTensorDescriptor_t iDesc, + const miopenTensorDescriptor_t tDesc, + const miopenTensorDescriptor_t wDesc, + const miopenTensorDescriptor_t oDesc, + const long p, + const float margin, + const miopenLossReductionMode_t reduction_mode, + const Tgpu* input, + const uint64_t* target, + const Tgpu* weight, + Tcheck* ref_output) +{ + auto I_tv = miopen::get_inner_expanded_tv<2>(miopen::deref(iDesc)); + auto T_tv = miopen::get_inner_expanded_tv<1>(miopen::deref(tDesc)); + auto W_tv = miopen::get_inner_expanded_tv<1>(miopen::deref(wDesc)); + auto O_tv = miopen::get_inner_expanded_tv<1>(miopen::deref(oDesc)); + auto N = I_tv.size[0], C = I_tv.size[1]; + + int32_t ret = miopenStatusSuccess; + double sum_loss = 0; + + for(size_t n = 0; n < N; n++) + { + double loss = 0; + uint64_t y = target[T_tv.get_tensor_view_idx({n})]; + if(y >= C) + continue; + for(size_t c = 0; c < C; c++) + { + if(y == c) + continue; + double t = margin - static_cast(input[I_tv.get_tensor_view_idx({n, y})]) + + static_cast(input[I_tv.get_tensor_view_idx({n, c})]); + + if(t < 0) + continue; + if(p == 2) + t = t * t; + t = weight[W_tv.get_tensor_view_idx({y})] * t; + loss += t / C; + } + if(reduction_mode != MIOPEN_LOSS_REDUCTION_NONE) + sum_loss += loss; + else + ref_output[O_tv.get_tensor_view_idx({n})] = static_cast(loss); + } + if(reduction_mode == MIOPEN_LOSS_REDUCTION_MEAN) + ref_output[0] = static_cast(sum_loss / N); + else if(reduction_mode == MIOPEN_LOSS_REDUCTION_SUM) + ref_output[0] = static_cast(sum_loss); + return ret; +} + +template +class MultiMarginLossDriver : public Driver +{ +public: + MultiMarginLossDriver() : Driver() + { + miopenCreateTensorDescriptor(&iDesc); + miopenCreateTensorDescriptor(&tDesc); + miopenCreateTensorDescriptor(&wDesc); + miopenCreateTensorDescriptor(&oDesc); + + data_type = miopen_type{}; + } + + int AddCmdLineArgs() override; + int ParseCmdLineArgs(int argc, char* argv[]) override; + InputFlags& GetInputFlags() override { return inflags; } + + int GetandSetData() override; + + int AllocateBuffersAndCopy() override; + + int RunForwardGPU() override; + int RunForwardCPU(); + + int RunBackwardGPU() override; + + Tref GetTolerance(); + int VerifyBackward() override; + int VerifyForward() override; + ~MultiMarginLossDriver() override + { + miopenDestroyTensorDescriptor(iDesc); + miopenDestroyTensorDescriptor(tDesc); + miopenDestroyTensorDescriptor(wDesc); + miopenDestroyTensorDescriptor(oDesc); + } + +private: + InputFlags inflags; + + // forw = 0 -> run both fw, bw, = 1 -> run only fw, = 2 -> run only bw + int forw; + + miopenTensorDescriptor_t iDesc; + miopenTensorDescriptor_t tDesc; + miopenTensorDescriptor_t wDesc; + miopenTensorDescriptor_t oDesc; + + std::unique_ptr i_dev; + std::unique_ptr t_dev; + std::unique_ptr w_dev; + std::unique_ptr o_dev; + std::unique_ptr workspace_dev; + + std::vector I; + std::vector T; + std::vector W; + std::vector O; + std::vector Ohost; + + long p; + float margin; + miopenLossReductionMode_t reduction_mode; + size_t ws_sizeInBytes; +}; + +template +int MultiMarginLossDriver::AddCmdLineArgs() +{ + inflags.AddInputFlag("forw", + 'F', + "1", + "Run Forward or Backward. 0 to run both Fw and Bw, 1 to run only Fw, 2 to " + "run only Bw (Default=1)", + "int"); + inflags.AddInputFlag("dim", 'D', "41x4", "Dim of input tensor (Default=41x4)", "tensor"); + inflags.AddInputFlag("contiguous", 'C', "1", "Tensor is contiguous or not (Default=1)", "int"); + inflags.AddInputFlag("iter", 'i', "10", "Number of Iterations (Default=10)", "int"); + inflags.AddInputFlag("verify", 'V', "1", "Verify Each Layer (Default=1)", "int"); + inflags.AddInputFlag("time", 't', "0", "Time Each Layer (Default=0)", "int"); + inflags.AddInputFlag( + "wall", 'w', "0", "Wall-clock Time Each Layer, Requires time == 1 (Default=0)", "int"); + inflags.AddInputFlag("reduce", + 'R', + "none", + "Specifies the reduction to apply to the output ('none'|'mean'|'sum') " + "(Default=none to indicate no reduction)", + "str"); + + return miopenStatusSuccess; +} + +template +int MultiMarginLossDriver::ParseCmdLineArgs(int argc, char* argv[]) +{ + inflags.Parse(argc, argv); + + auto reduction = inflags.GetValueStr("reduce"); + if(reduction != "none" && reduction != "mean" && reduction != "sum") + return miopenStatusInvalidValue; + if(inflags.GetValueInt("time") == 1) + { + miopenEnableProfiling(GetHandle(), true); + } + forw = inflags.GetValueInt("forw"); + if(forw == 2) + { + std::cerr << "MultiMarginLoss backward is not implemented." << std::endl; + return miopenStatusNotImplemented; + } + return miopenStatusSuccess; +} + +template +int MultiMarginLossDriver::GetandSetData() +{ + // Set tensor description + std::vector in_len = inflags.GetValueTensor("dim").lengths; + size_t N = in_len[0], C = in_len[1]; + if(inflags.GetValueInt("contiguous") == 1) + { + SetTensorNd(iDesc, in_len, data_type); + + std::vector t_len = {N}; + SetTensorNd(tDesc, t_len, miopenInt64); + + std::vector w_len = {C}; + SetTensorNd(wDesc, w_len, data_type); + } + else + { + std::vector in_strides(in_len.size()); + in_strides.back() = 1; + for(int i = in_len.size() - 2; i >= 0; --i) + in_strides[i] = in_strides[i + 1] * in_len[i + 1]; + in_strides[0] *= 2; + SetTensorNd(iDesc, in_len, in_strides, data_type); + + std::vector t_len = {N}; + std::vector t_strides = {2}; + SetTensorNd(tDesc, t_len, t_strides, miopenInt64); + + std::vector w_lens = {C}; + std::vector w_strides = {2}; + SetTensorNd(wDesc, w_lens, w_strides, data_type); + } + + // Set p and margin + // p = 1 or 2 + p = prng::gen_A_to_B(static_cast(1), static_cast(3)); + margin = prng::gen_A_to_B(static_cast(0.5), static_cast(1.5)); + + // Set reduction_mode + auto reduction = inflags.GetValueStr("reduce"); + if(reduction == "none") + reduction_mode = MIOPEN_LOSS_REDUCTION_NONE; + else if(reduction == "mean") + reduction_mode = MIOPEN_LOSS_REDUCTION_MEAN; + else if(reduction == "sum") + reduction_mode = MIOPEN_LOSS_REDUCTION_SUM; + + // Set output tensor description (forw = 1 or = 0) + if(forw == 0 || forw == 1) + { + if(reduction == "none") + { + std::vector o_lens = {N}; + SetTensorNd(oDesc, o_lens, data_type); + } + else + { + std::vector o_lens = {1}; + SetTensorNd(oDesc, o_lens, data_type); + } + } + + return miopenStatusSuccess; +} + +template +int MultiMarginLossDriver::AllocateBuffersAndCopy() +{ + uint32_t ctx = 0; + + // for unpacked tensor, we need to use GetTensorSpace instead of GetTensorSize + size_t i_sz = GetTensorSpace(iDesc); + size_t t_sz = GetTensorSpace(tDesc); + size_t w_sz = GetTensorSpace(wDesc); + i_dev = std::make_unique(ctx, i_sz, sizeof(Tgpu)); + t_dev = std::make_unique(ctx, t_sz, sizeof(uint64_t)); + w_dev = std::make_unique(ctx, w_sz, sizeof(Tgpu)); + I = std::vector(i_sz); + T = std::vector(t_sz); + W = std::vector(w_sz); + for(int i = 0; i < i_sz; i++) + { + I[i] = prng::gen_A_to_B(static_cast(-1), static_cast(1)); + } + int C = miopen::deref(iDesc).GetLengths()[1]; + // 0 to C - 1 + for(int i = 0; i < t_sz; i++) + { + T[i] = prng::gen_A_to_B(static_cast(0), static_cast(C)); + } + for(int i = 0; i < w_sz; i++) + { + W[i] = prng::gen_A_to_B(static_cast(-1), static_cast(1)); + } + + if(i_dev->ToGPU(GetStream(), I.data()) != 0) + { + std::cerr << "Error copying (I) to GPU, size: " << i_dev->GetSize() << std::endl; + return miopenStatusAllocFailed; + } + + if(t_dev->ToGPU(GetStream(), T.data()) != 0) + { + std::cerr << "Error copying (T) to GPU, size: " << t_dev->GetSize() << std::endl; + return miopenStatusAllocFailed; + } + + if(w_dev->ToGPU(GetStream(), W.data()) != 0) + { + std::cerr << "Error copying (W) to GPU, size: " << w_dev->GetSize() << std::endl; + return miopenStatusAllocFailed; + } + + if(forw == 0 || forw == 1) + { + size_t o_sz = GetTensorSpace(oDesc); + + miopenGetMultiMarginLossForwardWorkspaceSize( + GetHandle(), iDesc, tDesc, wDesc, oDesc, p, margin, reduction_mode, &ws_sizeInBytes); + if(ws_sizeInBytes == static_cast(-1)) + { + return miopenStatusAllocFailed; + } + + o_dev = std::make_unique(ctx, o_sz, sizeof(Tgpu)); + O = std::vector(o_sz, static_cast(0)); + Ohost = std::vector(o_sz, static_cast(0)); + if(o_dev->ToGPU(GetStream(), O.data()) != 0) + { + std::cerr << "Error copying (out) to GPU, size: " << o_dev->GetSize() << std::endl; + return miopenStatusAllocFailed; + } + + if(ws_sizeInBytes == 0) + workspace_dev = nullptr; + else + workspace_dev = std::make_unique(ctx, ws_sizeInBytes, sizeof(std::byte)); + } + + return miopenStatusSuccess; +} + +template +int MultiMarginLossDriver::RunForwardGPU() +{ + float kernel_total_time = 0; + float kernel_first_time = 0; + + Timer t; + START_TIME + + for(int i = 0; i < inflags.GetValueInt("iter"); i++) + { + miopenStatus_t status = miopenMultiMarginLossForward( + GetHandle(), + iDesc, + i_dev->GetMem(), + tDesc, + t_dev->GetMem(), + wDesc, + w_dev->GetMem(), + oDesc, + o_dev->GetMem(), + p, + margin, + reduction_mode, + workspace_dev == nullptr ? nullptr : workspace_dev->GetMem(), + ws_sizeInBytes); + + MIOPEN_THROW_IF(status != miopenStatusSuccess, "Error in miopenMultiMarginLossForward"); + + float time = 0.0; + miopenGetKernelTime(GetHandle(), &time); + kernel_total_time += time; + if(i == 0) + kernel_first_time = time; + } + + if(inflags.GetValueInt("time") == 1) + { + STOP_TIME + int iter = inflags.GetValueInt("iter"); + if(WALL_CLOCK) + std::cout << "Wall-clock Time Forward MultiMarginLoss Elapsed: " + << t.gettime_ms() / iter << " ms" << std::endl; + + float kernel_average_time = + iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; + std::cout << "GPU Kernel Time Forward MultiMarginLoss Elapsed: " << kernel_average_time + << " ms" << std::endl; + } + + if(o_dev->FromGPU(GetStream(), O.data()) != 0) + { + std::cerr << "Error copying (o_dev) from GPU, size: " << o_dev->GetSize() << std::endl; + return miopenStatusInternalError; + } + + return miopenStatusSuccess; +} + +template +int MultiMarginLossDriver::RunForwardCPU() +{ + return mloMultiMarginLossForwardRunHost(iDesc, + tDesc, + wDesc, + oDesc, + p, + margin, + reduction_mode, + I.data(), + T.data(), + W.data(), + Ohost.data()); +} + +template +int MultiMarginLossDriver::RunBackwardGPU() +{ + std::cerr << "MultiMarginLoss backward is not implemented." << std::endl; + return miopenStatusNotImplemented; +} + +template +Tref MultiMarginLossDriver::GetTolerance() +{ + Tref tolerance = std::numeric_limits::epsilon() * 10; + return tolerance; +} + +template +int MultiMarginLossDriver::VerifyForward() +{ + RunForwardCPU(); + + const Tref tolerance = GetTolerance(); + auto error = miopen::rms_range(Ohost, O); + if(!std::isfinite(error) || error > tolerance) + { + std::cout << "Forward MultiMarginLoss FAILED: " << error << " > " << tolerance << std::endl; + return EC_VerifyFwd; + } + else + { + std::cout << "Forward MultiMarginLoss Verifies OK on CPU reference (" << error << " < " + << tolerance << ')' << std::endl; + } + + return miopenStatusSuccess; +} + +template +int MultiMarginLossDriver::VerifyBackward() +{ + return miopenStatusNotImplemented; +} diff --git a/driver/prelu_driver.hpp b/driver/prelu_driver.hpp new file mode 100644 index 0000000000..e304f27465 --- /dev/null +++ b/driver/prelu_driver.hpp @@ -0,0 +1,394 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include "driver.hpp" +#include "mloPReLUHost.hpp" +#include "tensor_driver.hpp" +#include "timer.hpp" + +#include <../test/ford.hpp> +#include <../test/verify.hpp> + +#include + +template +class PReLUDriver : public Driver +{ +public: + PReLUDriver() : Driver() + { + miopenCreateTensorDescriptor(&inputDesc); + miopenCreateTensorDescriptor(&weightDesc); + miopenCreateTensorDescriptor(&doutputDesc); + miopenCreateTensorDescriptor(&dinputDesc); + miopenCreateTensorDescriptor(&dweightDesc); + + data_type = miopen_type{}; + } + + int AddCmdLineArgs() override; + int ParseCmdLineArgs(int argc, char* argv[]) override; + InputFlags& GetInputFlags() override { return inflags; } + + int GetandSetData() override; + std::vector GetTensorLengthsFromCmdLine(); + + int AllocateBuffersAndCopy() override; + + int RunForwardGPU() override; + int RunForwardCPU(); + + int RunBackwardGPU() override; + int RunBackwardCPU(); + + Tref GetTolerance(); + int VerifyBackward() override; + int VerifyForward() override; + ~PReLUDriver() override + { + miopenDestroyTensorDescriptor(inputDesc); + miopenDestroyTensorDescriptor(weightDesc); + miopenDestroyTensorDescriptor(doutputDesc); + miopenDestroyTensorDescriptor(dinputDesc); + miopenDestroyTensorDescriptor(dweightDesc); + } + +private: + InputFlags inflags; + + int forw; + + miopenTensorDescriptor_t inputDesc; + miopenTensorDescriptor_t weightDesc; + miopenTensorDescriptor_t doutputDesc; + miopenTensorDescriptor_t dinputDesc; + miopenTensorDescriptor_t dweightDesc; + + std::unique_ptr input_dev; + std::unique_ptr weight_dev; + std::unique_ptr doutput_dev; + std::unique_ptr dinput_dev; + std::unique_ptr dweight_dev; + std::unique_ptr workspace_dev; + + std::vector input; + std::vector weight; + std::vector doutput; + std::vector dinput; + std::vector dweight; + + std::vector dinput_host; + std::vector dweight_host; + + size_t ws_sizeInBytes; +}; + +template +int PReLUDriver::ParseCmdLineArgs(int argc, char* argv[]) +{ + inflags.Parse(argc, argv); + + if(inflags.GetValueInt("time") == 1) + { + miopenEnableProfiling(GetHandle(), true); + } + + auto inTensorParam = inflags.GetValueTensor("input"); + auto input_length = inTensorParam.lengths; + if(input_length.empty()) + { + std::cout << "Tensor must not be empty"; + return miopenStatusBadParm; + } + + std::vector weight_length = {inflags.GetValueInt("NumParameters")}; + if(weight_length[0] != 1 && (input_length.size() == 1 || weight_length[0] != input_length[1])) + { + std::cout << "NumParameters must be 1 or the second dim of DimLengths"; + return miopenStatusBadParm; + } + + return miopenStatusSuccess; +} + +template +int PReLUDriver::GetandSetData() +{ + auto inTensorParam = inflags.GetValueTensor("input"); + auto input_length = inTensorParam.lengths; + std::vector weight_length = {inflags.GetValueInt("NumParameters")}; + + if(SetTensorNd(inputDesc, input_length, data_type) != miopenStatusSuccess) + MIOPEN_THROW("Error parsing input tensor: " + inflags.GetValueStr("input") + "."); + if(SetTensorNd(dinputDesc, input_length, data_type) != miopenStatusSuccess) + MIOPEN_THROW("Error parsing input gradient tensor"); + + if(SetTensorNd(weightDesc, weight_length, data_type) != miopenStatusSuccess) + MIOPEN_THROW("Error parsing weight tensor"); + if(SetTensorNd(dweightDesc, weight_length, data_type) != miopenStatusSuccess) + MIOPEN_THROW("Error parsing weight gradient tensor"); + + if(SetTensorNd(doutputDesc, input_length, data_type) != miopenStatusSuccess) + MIOPEN_THROW("Error parsing output gradient tensor"); + + return miopenStatusSuccess; +} + +template +int PReLUDriver::AddCmdLineArgs() +{ + inflags.AddInputFlag("forw", 'F', "1", "Run only Forward PReLU (Default=1)", "int"); + inflags.AddTensorFlag("input", 'D', "256x4x1x1x8723", "input tensor descriptor"); + inflags.AddInputFlag( + "NumParameters", + 'P', + "1", + "Number of weight to learn. Although it takes an int as input, there is only two values " + "are legitimate: 1, or the number of channels (the second dim) at input (Default=1)", + "int"); + inflags.AddInputFlag("iter", 'i', "10", "Number of Iterations (Default=10)", "int"); + inflags.AddInputFlag("verify", 'V', "0", "Verify Each Layer (Default=0)", "int"); + inflags.AddInputFlag("time", 't', "0", "Time Each Layer (Default=0)", "int"); + inflags.AddInputFlag( + "wall", 'w', "0", "Wall-clock Time Each Layer, Requires time == 1 (Default=0)", "int"); + + return miopenStatusSuccess; +} + +template +int PReLUDriver::AllocateBuffersAndCopy() +{ + size_t input_sz = GetTensorSize(inputDesc); + size_t weight_sz = GetTensorSize(weightDesc); + + miopenGetPReLUBackwardWorkspaceSize(GetHandle(), inputDesc, weightDesc, &ws_sizeInBytes); + + if(ws_sizeInBytes == static_cast(-1)) + return miopenStatusAllocFailed; + + uint32_t ctx = 0; + + input_dev = std::unique_ptr(new GPUMem(ctx, input_sz, sizeof(Tgpu))); + weight_dev = std::unique_ptr(new GPUMem(ctx, weight_sz, sizeof(Tgpu))); + doutput_dev = std::unique_ptr(new GPUMem(ctx, input_sz, sizeof(Tgpu))); + dinput_dev = std::unique_ptr(new GPUMem(ctx, input_sz, sizeof(Tgpu))); + dweight_dev = std::unique_ptr(new GPUMem(ctx, weight_sz, sizeof(Tgpu))); + workspace_dev = std::unique_ptr(new GPUMem(ctx, ws_sizeInBytes, sizeof(std::byte))); + + input = std::vector(input_sz); + weight = std::vector(weight_sz); + doutput = std::vector(input_sz, static_cast(1.0f)); + dinput = std::vector(input_sz, std::numeric_limits::quiet_NaN()); + dweight = std::vector(weight_sz, std::numeric_limits::quiet_NaN()); + + dinput_host = std::vector(input_sz, std::numeric_limits::quiet_NaN()); + dweight_host = std::vector(weight_sz, std::numeric_limits::quiet_NaN()); + + for(int i = 0; i < input_sz; i++) + input[i] = prng::gen_A_to_B(static_cast(-1e-5), static_cast(1e-6)); + + for(int i = 0; i < weight_sz; i++) + weight[i] = prng::gen_A_to_B(static_cast(-1e-5), static_cast(1e-6)); + + if(input_dev->ToGPU(GetStream(), input.data()) != 0) + { + std::cerr << "Error copying (input) to GPU, size: " << input_dev->GetSize() << std::endl; + return miopenStatusAllocFailed; + } + + if(weight_dev->ToGPU(GetStream(), weight.data()) != 0) + { + std::cerr << "Error copying (weight) to GPU, size: " << weight_dev->GetSize() << std::endl; + return miopenStatusAllocFailed; + } + + if(doutput_dev->ToGPU(GetStream(), doutput.data()) != 0) + { + std::cerr << "Error copying (out grad) to GPU, size: " << doutput_dev->GetSize() + << std::endl; + return miopenStatusAllocFailed; + } + + if(dinput_dev->ToGPU(GetStream(), dinput.data()) != 0) + { + std::cerr << "Error copying (input grad) to GPU, size: " << dinput_dev->GetSize() + << std::endl; + return miopenStatusAllocFailed; + } + + if(dweight_dev->ToGPU(GetStream(), dweight.data()) != 0) + { + std::cerr << "Error copying (weight grad) to GPU, size: " << dweight_dev->GetSize() + << std::endl; + return miopenStatusAllocFailed; + } + + return miopenStatusSuccess; +} + +template +int PReLUDriver::RunForwardGPU() +{ + return miopenStatusNotImplemented; +} + +template +int PReLUDriver::RunForwardCPU() +{ + return miopenStatusNotImplemented; +} + +template +int PReLUDriver::RunBackwardGPU() +{ + float kernel_total_time = 0; + float kernel_first_time = 0; + + Timer t; + START_TIME + + for(int i = 0; i < inflags.GetValueInt("iter"); i++) + { + miopenPReLUBackward(GetHandle(), + workspace_dev->GetMem(), + ws_sizeInBytes, + inputDesc, + input_dev->GetMem(), + weightDesc, + weight_dev->GetMem(), + doutputDesc, + doutput_dev->GetMem(), + dinputDesc, + dinput_dev->GetMem(), + dweightDesc, + dweight_dev->GetMem()); + + float time = 0.0; + miopenGetKernelTime(GetHandle(), &time); + kernel_total_time += time; + if(i == 0) + kernel_first_time = time; + } + + if(inflags.GetValueInt("time") == 1) + { + STOP_TIME + int iter = inflags.GetValueInt("iter"); + if(WALL_CLOCK) + std::cout << "Wall-clock Time Backward PReLU Elapsed: " << t.gettime_ms() / iter + << " ms" << std::endl; + + float kernel_average_time = + iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; + std::cout << "GPU Kernel Time Backward PReLU Elapsed: " << kernel_average_time << " ms" + << std::endl; + } + + if(dinput_dev->FromGPU(GetStream(), dinput.data()) != 0) + { + std::cerr << "Error copying (dinput_dev) from GPU, size: " << dinput_dev->GetSize() + << std::endl; + return miopenStatusInternalError; + } + if(dweight_dev->FromGPU(GetStream(), dweight.data()) != 0) + { + std::cerr << "Error copying (dweight_dev) from GPU, size: " << dweight_dev->GetSize() + << std::endl; + return miopenStatusInternalError; + } + + return miopenStatusSuccess; +} + +template +int PReLUDriver::RunBackwardCPU() +{ + return mloPReLUBackwardRunHost(inputDesc, + weightDesc, + doutputDesc, + dinputDesc, + input.data(), + weight.data(), + doutput.data(), + dinput_host.data(), + dweight_host.data()); +} + +template +Tref PReLUDriver::GetTolerance() +{ + // Computation error of fp16 is ~2^13 (=8192) bigger than + // the one of fp32 because mantissa is shorter by 13 bits. + auto tolerance = std::is_same::value ? 1.5e-6 : 8.2e-3; + + // bf16 mantissa has 7 bits, by 3 bits shorter than fp16. + if(std::is_same::value) + tolerance *= 8.0; + return tolerance; +} + +template +int PReLUDriver::VerifyForward() +{ + return miopenStatusNotImplemented; +} + +template +int PReLUDriver::VerifyBackward() +{ + RunBackwardCPU(); + const Tref tolerance = GetTolerance(); + auto error_dinput = miopen::rms_range(dinput_host, dinput); + auto error_dweight = miopen::rms_range(dweight_host, dweight); + + if(!std::isfinite(error_dinput) || error_dinput > tolerance) + { + std::cout << "Backward PReLU Input Gradient FAILED: " << error_dinput << " > " << tolerance + << std::endl; + return EC_VerifyBwd; + } + else + { + std::cout << "Backward PReLU Input Gradient Verifies OK on CPU reference (" << error_dinput + << " < " << tolerance << ')' << std::endl; + } + + if(!std::isfinite(error_dweight) || error_dweight > tolerance) + { + std::cout << "Backward PReLU Weight Gradient FAILED: " << error_dweight << " > " + << tolerance << std::endl; + return EC_VerifyBwd; + } + else + { + std::cout << "Backward PReLU Weight Gradient Verifies OK on CPU reference (" + << error_dweight << " < " << tolerance << ')' << std::endl; + } + + return miopenStatusSuccess; +} diff --git a/driver/sum_driver.hpp b/driver/reducecalculation_driver.hpp similarity index 55% rename from driver/sum_driver.hpp rename to driver/reducecalculation_driver.hpp index 830b89c1dd..2001969509 100644 --- a/driver/sum_driver.hpp +++ b/driver/reducecalculation_driver.hpp @@ -23,8 +23,8 @@ * SOFTWARE. * *******************************************************************************/ -#ifndef GUARD_MIOPEN_SUM_DRIVER_HPP -#define GUARD_MIOPEN_SUM_DRIVER_HPP +#ifndef GUARD_MIOPEN_REDUCE_CALCULATION_DRIVER_HPP +#define GUARD_MIOPEN_REDUCE_CALCULATION_DRIVER_HPP #include "InputFlags.hpp" #include "driver.hpp" @@ -33,6 +33,7 @@ #include "random.hpp" #include #include +#include #include #include #include @@ -41,62 +42,63 @@ #include #include <../test/tensor_holder.hpp> #include <../test/verify.hpp> - -#ifndef MLO_SUMMHOST_H_ -#define MLO_SUMMHOST_H_ - -template -int32_t mloSumForwardRunHost(miopenTensorDescriptor_t inputDesc, - miopenTensorDescriptor_t yDesc, - Tgpu* input, - Tcheck* outputhost, - int32_t dim, - miopenSumNanPropagation_t nanPropagation) +#include "../src/kernels/MIOpenReduceCalculation.hpp" + +#ifndef MLO_REDUCE_CALCULATIONMHOST_H_ +#define MLO_REDUCE_CALCULATIONMHOST_H_ + +template +int32_t mloReduceCalculationForwardRunHost(miopenTensorDescriptor_t inputDesc, + miopenTensorDescriptor_t outputDesc, + Tgpu* input, + Tcheck* outputhost, + int32_t dim, + miopenReduceCalculationNanPropagation_t nanPropagation) { auto input_dims = miopen::deref(inputDesc).GetLengths(); - auto output_dims = miopen::deref(yDesc).GetLengths(); + auto output_dims = miopen::deref(outputDesc).GetLengths(); auto reduce_size = input_dims[dim]; auto output_numel = std::accumulate(output_dims.begin(), output_dims.end(), 1LL, std::multiplies()); auto inner_size = 1ULL; - for(int32_t i = dim + 1; i < input_dims.size(); i++) + for(int32_t i = dim + 1; i < input_dims.size(); ++i) { inner_size *= input_dims[i]; } int32_t ret = 0; - for(size_t o = 0; o < output_numel; o++) + for(size_t o = 0; o < output_numel; ++o) { size_t input_idx = (o / inner_size) * inner_size * reduce_size + o % inner_size; - Tcheck sum = 0.0f; - for(size_t i = 0; i < reduce_size; i++) + Tcheck calculation = 0.0f; + for(size_t i = 0; i < reduce_size; ++i) { Tcheck val = static_cast(input[input_idx]); - if(nanPropagation && isnan(val)) + if(nanPropagation && std::isnan(val)) { val = 0.0f; } - sum += val; + reduce_func{}.calculate(calculation, val); input_idx += inner_size; } - outputhost[o] = sum; + outputhost[o] = calculation; } return ret; } #endif template -class SumDriver : public Driver +class ReduceCalculationDriver : public Driver { public: - SumDriver() : Driver() + ReduceCalculationDriver() : Driver() { miopenCreateTensorDescriptor(&inputDesc); - miopenCreateTensorDescriptor(&yDesc); + miopenCreateTensorDescriptor(&outputDesc); data_type = miopen_type{}; } @@ -118,19 +120,17 @@ class SumDriver : public Driver Tref GetTolerance(); int VerifyBackward() override; int VerifyForward() override; - ~SumDriver() override + ~ReduceCalculationDriver() override { miopenDestroyTensorDescriptor(inputDesc); - miopenDestroyTensorDescriptor(yDesc); + miopenDestroyTensorDescriptor(outputDesc); } private: InputFlags inflags; - int forw; - miopenTensorDescriptor_t inputDesc; - miopenTensorDescriptor_t yDesc; + miopenTensorDescriptor_t outputDesc; std::unique_ptr in_dev; std::unique_ptr out_dev; @@ -143,11 +143,12 @@ class SumDriver : public Driver size_t ws_sizeInBytes; int dim; - miopenSumNanPropagation_t nanPropagation; + miopenReduceCalculationNanPropagation_t nanPropagation; + miopenReduceCalculationOp_t reduceCalculationOp; }; template -int SumDriver::ParseCmdLineArgs(int argc, char* argv[]) +int ReduceCalculationDriver::ParseCmdLineArgs(int argc, char* argv[]) { inflags.Parse(argc, argv); @@ -159,16 +160,18 @@ int SumDriver::ParseCmdLineArgs(int argc, char* argv[]) } template -int SumDriver::GetandSetData() +int ReduceCalculationDriver::GetandSetData() { - std::vector in_len = GetInputTensorLengthsFromCmdLine(); - dim = inflags.GetValueInt("DimToReduce"); + auto inTensorParam = inflags.GetValueTensor("input"); + dim = inflags.GetValueInt("DimToReduce"); + auto in_len = inTensorParam.lengths; - SetTensorNd(inputDesc, in_len, data_type); + if(SetTensorNd(inputDesc, in_len, data_type) != miopenStatusSuccess) + MIOPEN_THROW("Error parsing input tensor: " + inflags.GetValueStr("input") + "."); std::vector out_len; - for(int i = 0; i < in_len.size(); i++) + for(int i = 0; i < in_len.size(); ++i) { if(i != dim) { @@ -179,31 +182,39 @@ int SumDriver::GetandSetData() if(out_len.empty()) out_len.push_back(1); - SetTensorNd(yDesc, out_len, data_type); + if(SetTensorNd(outputDesc, out_len, data_type) != miopenStatusSuccess) + MIOPEN_THROW("Error setting output tensor."); - nanPropagation = static_cast(inflags.GetValueInt("NanPropagation")); + nanPropagation = + static_cast(inflags.GetValueInt("NanPropagation")); + reduceCalculationOp = + static_cast(inflags.GetValueInt("ReduceCalculationOp")); return 0; } template -int SumDriver::AddCmdLineArgs() +int ReduceCalculationDriver::AddCmdLineArgs() { - inflags.AddInputFlag("forw", 'F', "1", "Run only Forward Sum (Default=1)", "int"); - inflags.AddInputFlag("batchsize", 'n', "256", "Mini-batch size (Default=100)", "int"); - inflags.AddInputFlag("in_channels", 'c', "4", "Number of Input Channels (Default=3)", "int"); - inflags.AddInputFlag("in_d", 'D', "0", "Input Depth (Default=0)", "int"); - inflags.AddInputFlag("in_h", 'H', "0", "Input Height (Default=32)", "int"); - inflags.AddInputFlag("in_w", 'W', "8732", "Input Width (Default=32)", "int"); - + inflags.AddInputFlag( + "forw", 'F', "1", "Run only Forward Reduce Calculation (Default=1)", "int"); + inflags.AddTensorFlag("input", 'X', "256x4x8732", "input tensor descriptor"); inflags.AddInputFlag( "DimToReduce", 'R', "1", "The indice of the dimensions to be reduced(Default=1)", "int"); - inflags.AddInputFlag("NanPropagation", - 'N', - "0", - "Nan number propagation mode (check the miopenSumNanPropagation_t in " - "miopen.h) (Default=0 to indicate no Nan propagation)", - "int"); + inflags.AddInputFlag( + "NanPropagation", + 'N', + "0", + "Nan number propagation mode (check the miopenReduceCalculationNanPropagation_t in " + "miopen.h) (Default=0 to indicate no Nan propagation)", + "int"); + inflags.AddInputFlag( + "ReduceCalculationOp", + 'O', + "2", + "Reduce Calculation Operation Type (check the miopenReduceCalculationOp_t in " + "miopen.h) (Default=2 to add the values of the reduced elements)", + "int"); inflags.AddInputFlag("iter", 'i', "10", "Number of Iterations (Default=10)", "int"); inflags.AddInputFlag("verify", 'V', "1", "Verify Each Layer (Default=1)", "int"); inflags.AddInputFlag("time", 't', "0", "Time Each Layer (Default=0)", "int"); @@ -214,48 +225,13 @@ int SumDriver::AddCmdLineArgs() } template -std::vector SumDriver::GetInputTensorLengthsFromCmdLine() -{ - int in_n = inflags.GetValueInt("batchsize"); - int in_c = inflags.GetValueInt("in_channels"); - int in_w = inflags.GetValueInt("in_w"); - int in_h = inflags.GetValueInt("in_h"); - int in_d = inflags.GetValueInt("in_d"); - - if((in_n != 0) && (in_c != 0) && (in_d != 0) && (in_h != 0) && (in_w != 0)) - { - return std::vector({in_n, in_c, in_d, in_h, in_w}); - } - else if((in_n != 0) && (in_c != 0) && (in_h != 0) && (in_w != 0)) - { - return std::vector({in_n, in_c, in_h, in_w}); - } - else if((in_n != 0) && (in_c != 0) && (in_w != 0)) - { - return std::vector({in_n, in_c, in_w}); - } - else if((in_n != 0) && (in_w != 0)) - { - return std::vector({in_n, in_w}); - } - else if(in_n != 0) - { - return std::vector({in_n}); - } - else - { - std::cerr << "Error Input Tensor Lengths\n" << std::endl; - return std::vector({0}); - } -} - -template -int SumDriver::AllocateBuffersAndCopy() +int ReduceCalculationDriver::AllocateBuffersAndCopy() { size_t in_sz = GetTensorSize(inputDesc); - size_t out_sz = GetTensorSize(yDesc); + size_t out_sz = GetTensorSize(outputDesc); - miopenGetSumWorkspaceSize(GetHandle(), inputDesc, dim, yDesc, &ws_sizeInBytes); + miopenGetReduceCalculationWorkspaceSize( + GetHandle(), inputDesc, dim, reduceCalculationOp, outputDesc, &ws_sizeInBytes); if(ws_sizeInBytes == static_cast(-1)) return miopenStatusAllocFailed; @@ -269,22 +245,27 @@ int SumDriver::AllocateBuffersAndCopy() out = std::vector(out_sz, static_cast(0)); outhost = std::vector(out_sz, static_cast(0)); - for(int i = 0; i < in_sz; i++) + for(int i = 0; i < in_sz; ++i) { in[i] = prng::gen_A_to_B(static_cast(0.0), static_cast(1.0)); } if(in_dev->ToGPU(GetStream(), in.data()) != 0) + { std::cerr << "Error copying (in) to GPU, size: " << in_dev->GetSize() << std::endl; - + return miopenStatusAllocFailed; + } if(out_dev->ToGPU(GetStream(), out.data()) != 0) + { std::cerr << "Error copying (out) to GPU, size: " << out_dev->GetSize() << std::endl; + return miopenStatusAllocFailed; + } return miopenStatusSuccess; } template -int SumDriver::RunForwardGPU() +int ReduceCalculationDriver::RunForwardGPU() { float kernel_total_time = 0; float kernel_first_time = 0; @@ -292,17 +273,18 @@ int SumDriver::RunForwardGPU() Timer t; START_TIME - for(int i = 0; i < inflags.GetValueInt("iter"); i++) + for(int i = 0; i < inflags.GetValueInt("iter"); ++i) { - miopenSumForward(GetHandle(), - nanPropagation, - workspace_dev->GetMem(), - ws_sizeInBytes, - inputDesc, - in_dev->GetMem(), - dim, - yDesc, - out_dev->GetMem()); + miopenReduceCalculationForward(GetHandle(), + nanPropagation, + workspace_dev->GetMem(), + ws_sizeInBytes, + inputDesc, + in_dev->GetMem(), + dim, + reduceCalculationOp, + outputDesc, + out_dev->GetMem()); float time = 0.0; miopenGetKernelTime(GetHandle(), &time); @@ -316,37 +298,48 @@ int SumDriver::RunForwardGPU() STOP_TIME int iter = inflags.GetValueInt("iter"); if(WALL_CLOCK) - std::cout << "Wall-clock Time Forward Sum Elapsed: " << t.gettime_ms() / iter - << " ms\n"; + std::cout << "Wall-clock Time Forward Reduce Calculation Elapsed: " + << t.gettime_ms() / iter << " ms" << std::endl; float kernel_average_time = iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; - std::cout << "GPU Kernel Time Forward Sum Elapsed: " << kernel_average_time << " ms\n"; + std::cout << "GPU Kernel Time Forward Reduce Calculation Elapsed: " << kernel_average_time + << " ms" << std::endl; } if(out_dev->FromGPU(GetStream(), out.data()) != 0) + { std::cerr << "Error copying (out_dev) from GPU, size: " << out_dev->GetSize() << std::endl; - + return miopenStatusInternalError; + } return miopenStatusSuccess; } template -int SumDriver::RunForwardCPU() +int ReduceCalculationDriver::RunForwardCPU() { - mloSumForwardRunHost( - inputDesc, yDesc, in.data(), outhost.data(), dim, nanPropagation); + if(reduceCalculationOp == MIOPEN_REDUCE_CALCULATION_SUM) + { + return mloReduceCalculationForwardRunHost( + inputDesc, outputDesc, in.data(), outhost.data(), dim, nanPropagation); + } + else if(reduceCalculationOp == MIOPEN_REDUCE_CALCULATION_PROD) + { + return mloReduceCalculationForwardRunHost( + inputDesc, outputDesc, in.data(), outhost.data(), dim, nanPropagation); + } - return miopenStatusSuccess; + return miopenStatusInternalError; } template -int SumDriver::RunBackwardGPU() +int ReduceCalculationDriver::RunBackwardGPU() { return miopenStatusSuccess; } template -Tref SumDriver::GetTolerance() +Tref ReduceCalculationDriver::GetTolerance() { // Computation error of fp16 is ~2^13 (=8192) bigger than // the one of fp32 because mantissa is shorter by 13 bits. @@ -359,7 +352,7 @@ Tref SumDriver::GetTolerance() } template -int SumDriver::VerifyForward() +int ReduceCalculationDriver::VerifyForward() { RunForwardCPU(); const Tref tolerance = GetTolerance(); @@ -367,22 +360,23 @@ int SumDriver::VerifyForward() if(!std::isfinite(error) || error > tolerance) { - std::cout << "Forward Sum FAILED: " << error << " > " << tolerance << std::endl; + std::cout << "Forward Reduce Calculation FAILED: " << error << " > " << tolerance + << std::endl; return EC_VerifyFwd; } else { - std::cout << "Forward Sum Verifies OK on CPU reference (" << error << " < " << tolerance - << ')' << std::endl; + std::cout << "Forward Reduce Calculation Verifies OK on CPU reference (" << error << " < " + << tolerance << ')' << std::endl; } return miopenStatusSuccess; } template -int SumDriver::VerifyBackward() +int ReduceCalculationDriver::VerifyBackward() { return miopenStatusSuccess; } -#endif // GUARD_MIOPEN_SUM_DRIVER_HPP +#endif // GUARD_MIOPEN_REDUCE_CALCULATION_DRIVER_HPP diff --git a/driver/reduceextreme_driver.hpp b/driver/reduceextreme_driver.hpp index 7f5fbbc301..a06f5288a1 100644 --- a/driver/reduceextreme_driver.hpp +++ b/driver/reduceextreme_driver.hpp @@ -134,8 +134,6 @@ class ReduceExtremeDriver : public Driver private: InputFlags inflags; - int forw; - miopenTensorDescriptor_t xDesc; miopenTensorDescriptor_t yDesc; miopenTensorDescriptor_t indiceDesc; diff --git a/driver/rnn_driver.hpp b/driver/rnn_driver.hpp index 03da917e0a..22f3d63131 100644 --- a/driver/rnn_driver.hpp +++ b/driver/rnn_driver.hpp @@ -158,7 +158,7 @@ class RNNDriver : public Driver std::vector cy_host; std::vector dhx_host; std::vector dcx_host; - std::vector dropout_states_host; + std::vector dropout_states_host; miopenRNNDescriptor_t rnnDesc; @@ -195,10 +195,6 @@ int RNNDriver::ParseCmdLineArgs(int argc, char* argv[]) { inflags.Parse(argc, argv); - if(inflags.GetValueInt("time") == 1) - { - miopenEnableProfiling(GetHandle(), true); - } return miopenStatusSuccess; } @@ -259,9 +255,16 @@ int RNNDriver::AddCmdLineArgs() "Use specified directory to cache verification data. Off by default.", "string"); */ - inflags.AddInputFlag("time", 't', "0", "Time Each Layer (Default=0)", "int"); inflags.AddInputFlag( - "wall", 'w', "0", "Wall-clock Time Each Layer, Requires time == 1 (Default=0)", "int"); + "wall", + 'w', + "0", + "Wall-clock, for host and gpu, Time Each Layer, Disabled = 0,\ + OldWallClock = 1,\ + SeparateClocksSynced = 2,\ + SeparateClocksNotSynced = 3 \ + (Default = 0) ", + "int"); inflags.AddInputFlag("dump_output", 'o', "0", "Dumps the output buffers (Default=0)", "int"); /* // DL: These have not been implemented. Removing them for now. inflags.AddInputFlag("in_data", 'd', "", "Input data filename (Default=)", "string"); @@ -484,7 +487,7 @@ int RNNDriver::SetRNNDescriptorFromCmdLineArgs() size_t statesSizeInBytes = 0; miopenDropoutGetStatesSize(GetHandle(), &statesSizeInBytes); - size_t states_size = statesSizeInBytes / sizeof(prngStates); + size_t states_size = statesSizeInBytes / sizeof(rocrand_state_xorwow); DEFINE_CONTEXT(ctx); #if MIOPEN_BACKEND_OPENCL @@ -492,7 +495,7 @@ int RNNDriver::SetRNNDescriptorFromCmdLineArgs() #endif dropout_states_dev = - std::unique_ptr(new GPUMem(ctx, states_size, sizeof(prngStates))); + std::unique_ptr(new GPUMem(ctx, states_size, sizeof(rocrand_state_xorwow))); miopenSetDropoutDescriptor(DropoutDesc, GetHandle(), @@ -784,9 +787,7 @@ int RNNDriver::RunForwardGPU() if(inflags.GetValueInt("forw") != 0 && !(inflags.GetValueInt("forw") & 1)) return miopenStatusSuccess; - Timer t; - float wl_time_forward = 0.0; - float kl_time_forward = 0.0; + RNNCombTimeLoger t(GetStream(), inflags.GetValueInt("iter"), inflags.GetValueInt("wall")); for(int i = 0; i < inflags.GetValueInt("iter"); i++) { @@ -801,8 +802,7 @@ int RNNDriver::RunForwardGPU() workspace_dev->ToGPU(q, workspace.data()); reservespace_dev->ToGPU(q, reservespace.data()); } - - START_TIME + t.Start(); if(inflags.GetValueInt("fwdtype") == 0) { miopenRNNForwardTraining(GetHandle(), @@ -854,31 +854,14 @@ int RNNDriver::RunForwardGPU() workspace_dev->GetMem(), workspace_dev->GetSize()); } - miopen::deref(GetHandle()).Finish(); - STOP_TIME - if(i > 0 || inflags.GetValueInt("iter") == 1) - { - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - // printf("wall time: %f\n", t.gettime_ms()); - wl_time_forward += t.gettime_ms(); - kl_time_forward += time; - } + t.StopAndPush(); } - - if(inflags.GetValueInt("time") == 1) - { - int n_iter = inflags.GetValueInt("iter") > 1 ? inflags.GetValueInt("iter") - 1 - : inflags.GetValueInt("iter"); - printf("GPU Kernel Time Forward RNN Elapsed: %f ms\n", kl_time_forward / n_iter); - } - + miopen::deref(GetHandle()).Finish(); if(WALL_CLOCK) { - int n_iter = inflags.GetValueInt("iter") > 1 ? inflags.GetValueInt("iter") - 1 - : inflags.GetValueInt("iter"); - printf("Wall-clock Time Forward RNN Elapsed: %f ms\n", wl_time_forward / n_iter); + printf("Forward RNN time results:\n"); + t.Print(); } out_dev->FromGPU(GetStream(), out.data()); @@ -1111,15 +1094,12 @@ int RNNDriver::RunBackwardGPU() if((inflags.GetValueInt("forw") & 2) || (inflags.GetValueInt("forw") == 0)) { - Timer t; - float wl_time_backward_data = 0.0; - float kl_time_backward_data = 0.0; - + RNNCombTimeLoger t(GetStream(), inflags.GetValueInt("iter"), inflags.GetValueInt("wall")); workspace_dev->ToGPU(q, workspace.data()); for(int i = 0; i < inflags.GetValueInt("iter"); i++) { - START_TIME + t.Start(); ret = miopenRNNBackwardData(GetHandle(), rnnDesc, adjustedSeqLen, @@ -1147,31 +1127,13 @@ int RNNDriver::RunBackwardGPU() workspace_dev->GetSize(), reservespace_dev->GetMem(), reservespace_dev->GetSize()); - miopen::deref(GetHandle()).Finish(); - STOP_TIME - if(i > 0 || inflags.GetValueInt("iter") == 1) - { - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - wl_time_backward_data += t.gettime_ms(); - kl_time_backward_data += time; - } + t.StopAndPush(); } - - if(inflags.GetValueInt("time") == 1) - { - int n_iter = inflags.GetValueInt("iter") > 1 ? inflags.GetValueInt("iter") - 1 - : inflags.GetValueInt("iter"); - printf("GPU Kernel Time Backward Data RNN Elapsed: %f ms\n", - kl_time_backward_data / n_iter); - } - + miopen::deref(GetHandle()).Finish(); if(WALL_CLOCK) { - int n_iter = inflags.GetValueInt("iter") > 1 ? inflags.GetValueInt("iter") - 1 - : inflags.GetValueInt("iter"); - printf("Wall-clock Time Backward Data RNN Elapsed: %f ms\n", - wl_time_backward_data / n_iter); + printf("Backward Data RNN time results:\n"); + t.Print(); } din_dev->FromGPU(GetStream(), din.data()); @@ -1182,13 +1144,11 @@ int RNNDriver::RunBackwardGPU() if((inflags.GetValueInt("forw") & 4) || (inflags.GetValueInt("forw") == 0)) { - Timer t; - float wl_time_backward_weight = 0.0; - float kl_time_backward_weight = 0.0; + RNNCombTimeLoger t(GetStream(), inflags.GetValueInt("iter"), inflags.GetValueInt("wall")); for(int i = 0; i < inflags.GetValueInt("iter"); i++) { - START_TIME + t.Start(); ret = miopenRNNBackwardWeights(GetHandle(), rnnDesc, adjustedSeqLen, @@ -1204,42 +1164,18 @@ int RNNDriver::RunBackwardGPU() workspace_dev->GetSize(), reservespace_dev->GetMem(), reservespace_dev->GetSize()); - miopen::deref(GetHandle()).Finish(); - STOP_TIME - if(i > 0 || inflags.GetValueInt("iter") == 1) - { - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - wl_time_backward_weight += t.gettime_ms(); - kl_time_backward_weight += time; - } - } - - if(inflags.GetValueInt("time") == 1) - { - int n_iter = inflags.GetValueInt("iter") > 1 ? inflags.GetValueInt("iter") - 1 - : inflags.GetValueInt("iter"); - printf("GPU Kernel Time Backward Weights RNN Elapsed: %f ms\n", - kl_time_backward_weight / n_iter); + t.StopAndPush(); } + miopen::deref(GetHandle()).Finish(); if(WALL_CLOCK) { - int n_iter = inflags.GetValueInt("iter") > 1 ? inflags.GetValueInt("iter") - 1 - : inflags.GetValueInt("iter"); - printf("Wall-clock Time Backward Weights RNN Elapsed: %f ms\n", - wl_time_backward_weight / n_iter); + printf("Backward Weights RNN time results:\n"); + t.Print(); } dwei_dev->FromGPU(GetStream(), dwei.data()); } - /* - if(inflags.GetValueInt("dump_output")) - { - dumpBufferToFile("dump_bwd_din_gpu.bin", din.data(), din.size()); - dumpBufferToFile("dump_bwd_dwei_gpu.bin", dwei.data(), dwei.size()); - } - */ return ret; } diff --git a/driver/rnn_seq_driver.hpp b/driver/rnn_seq_driver.hpp index 8d5b720960..2a05741b2f 100644 --- a/driver/rnn_seq_driver.hpp +++ b/driver/rnn_seq_driver.hpp @@ -271,7 +271,7 @@ class RNNSeqDriver : public Driver std::vector cy_host; std::vector dhx_host; std::vector dcx_host; - std::vector dropout_states_host; + std::vector dropout_states_host; miopenRNNDescriptor_t rnnDesc; @@ -294,11 +294,6 @@ int RNNSeqDriver::ParseCmdLineArgs(int argc, char* argv[]) { inflags.Parse(argc, argv); - if(inflags.GetValueInt("time") == 1) - { - miopenEnableProfiling(GetHandle(), true); - } - int nn_dir = inflags.GetValueInt("forw"); if(inflags.GetValueInt("fwdtype") == 1 && !(nn_dir == 0 || nn_dir == 1)) { @@ -487,10 +482,17 @@ int RNNSeqDriver::AddCmdLineArgs() inflags.AddInputFlag("iter", 'i', "1", "Number of Iterations (Default=1)", "int"); inflags.AddInputFlag("verify", 'V', "1", "Verify Each Layer (Default=1)", "int"); - inflags.AddInputFlag("time", 't', "0", "Time Each Layer (Default=0)", "int"); inflags.AddInputFlag( - "wall", 'w', "0", "Wall-clock Time Each Layer, Requires time == 1 (Default=0)", "int"); + "wall", + 'w', + "0", + "Wall-clock, for host and gpu, Time Each Layer, Disabled = 0,\ + OldWallClock = 1,\ + SeparateClocksSynced = 2,\ + SeparateClocksNotSynced = 3 \ + (Default = 0) ", + "int"); inflags.AddInputFlag("dump_output", 'o', "0", "Dumps the output buffers (Default=0)", "int"); /* // DL: These have not been implemented. Removing them for now. inflags.AddInputFlag("in_data", 'd', "", "Input data filename (Default=)", "string"); @@ -715,7 +717,7 @@ int RNNSeqDriver::SetRNNDescriptorFromCmdLineArgs() size_t statesSizeInBytes = 0; miopenDropoutGetStatesSize(GetHandle(), &statesSizeInBytes); - size_t states_size = statesSizeInBytes / sizeof(prngStates); + size_t states_size = statesSizeInBytes / sizeof(rocrand_state_xorwow); DEFINE_CONTEXT(ctx); #if MIOPEN_BACKEND_OPENCL @@ -723,7 +725,7 @@ int RNNSeqDriver::SetRNNDescriptorFromCmdLineArgs() #endif dropout_states_dev = - std::unique_ptr(new GPUMem(ctx, states_size, sizeof(prngStates))); + std::unique_ptr(new GPUMem(ctx, states_size, sizeof(rocrand_state_xorwow))); miopenSetDropoutDescriptor(DropoutDesc, GetHandle(), @@ -1073,9 +1075,7 @@ int RNNSeqDriver::RunForwardGPU() if(inflags.GetValueInt("forw") != 0 && !(inflags.GetValueInt("forw") & 1)) return miopenStatusSuccess; - Timer t; - float wl_time_forward = 0.0; - float kl_time_forward = 0.0; + RNNCombTimeLoger t(GetStream(), inflags.GetValueInt("iter"), inflags.GetValueInt("wall")); from_gpu_out = std::vector(out_dev->GetSize() / sizeof(Tgpu), static_cast(0)); @@ -1085,7 +1085,7 @@ int RNNSeqDriver::RunForwardGPU() workspace_dev->ToGPU(q, workspace.data()); reservespace_dev->ToGPU(q, reservespace.data()); - START_TIME + t.Start(); miopenRNNForward(GetHandle(), rnnDesc, fwd_type, @@ -1106,32 +1106,14 @@ int RNNSeqDriver::RunForwardGPU() reservespace_dev->GetMem(), reservespace_dev->GetSize()); - miopen::deref(GetHandle()).Finish(); - STOP_TIME - - // TODO chage timing system - if(i > 0 || inflags.GetValueInt("iter") == 1) - { - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - // printf("wall time: %f\n", t.gettime_ms()); - wl_time_forward += t.gettime_ms(); - kl_time_forward += time; - } - } - - if(inflags.GetValueInt("time") == 1) - { - int n_iter = inflags.GetValueInt("iter") > 1 ? inflags.GetValueInt("iter") - 1 - : inflags.GetValueInt("iter"); - printf("GPU Kernel Time Forward RNN Elapsed: %f ms\n", kl_time_forward / n_iter); + t.StopAndPush(); } + miopen::deref(GetHandle()).Finish(); if(WALL_CLOCK) { - int n_iter = inflags.GetValueInt("iter") > 1 ? inflags.GetValueInt("iter") - 1 - : inflags.GetValueInt("iter"); - printf("Wall-clock Time Forward RNN Elapsed: %f ms\n", wl_time_forward / n_iter); + printf("Forward RNN time results:\n"); + t.Print(); } if(io_layout != miopenRNNDataSeqMajorNotPadded) @@ -1184,9 +1166,7 @@ int RNNSeqDriver::RunBackwardGPU() if((inflags.GetValueInt("forw") & 2) || (inflags.GetValueInt("forw") == 0)) { - Timer t; - float wl_time_backward_data = 0.0; - float kl_time_backward_data = 0.0; + RNNCombTimeLoger t(GetStream(), inflags.GetValueInt("iter"), inflags.GetValueInt("wall")); workspace_dev->ToGPU(q, workspace.data()); if(inflags.GetValueInt("inputmode") == 1) @@ -1199,7 +1179,7 @@ int RNNSeqDriver::RunBackwardGPU() for(int i = 0; i < inflags.GetValueInt("iter"); i++) { - START_TIME + t.Start(); ret = miopenRNNBackwardSeqData(GetHandle(), rnnDesc, outputSeqTensor, @@ -1221,33 +1201,16 @@ int RNNSeqDriver::RunBackwardGPU() workspace_dev->GetSize(), reservespace_dev->GetMem(), reservespace_dev->GetSize()); - - miopen::deref(GetHandle()).Finish(); - STOP_TIME - if(i > 0 || inflags.GetValueInt("iter") == 1) - { - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - wl_time_backward_data += t.gettime_ms(); - kl_time_backward_data += time; - } - } - - if(inflags.GetValueInt("time") == 1) - { - int n_iter = inflags.GetValueInt("iter") > 1 ? inflags.GetValueInt("iter") - 1 - : inflags.GetValueInt("iter"); - printf("GPU Kernel Time Backward Data RNN Elapsed: %f ms\n", - kl_time_backward_data / n_iter); + t.StopAndPush(); } + miopen::deref(GetHandle()).Finish(); if(WALL_CLOCK) { - int n_iter = inflags.GetValueInt("iter") > 1 ? inflags.GetValueInt("iter") - 1 - : inflags.GetValueInt("iter"); - printf("Wall-clock Time Backward Data RNN Elapsed: %f ms\n", - wl_time_backward_data / n_iter); + printf("Backward Data RNN time results:\n"); + t.Print(); } + if(io_layout != miopenRNNDataSeqMajorNotPadded) { auto from_gpu_din = std::vector(din_dev->GetSize() / sizeof(Tgpu)); @@ -1288,13 +1251,11 @@ int RNNSeqDriver::RunBackwardGPU() if((inflags.GetValueInt("forw") & 4) || (inflags.GetValueInt("forw") == 0)) { - Timer t; - float wl_time_backward_weight = 0.0; - float kl_time_backward_weight = 0.0; + RNNCombTimeLoger t(GetStream(), inflags.GetValueInt("iter"), inflags.GetValueInt("wall")); for(int i = 0; i < inflags.GetValueInt("iter"); i++) { - START_TIME + t.Start(); ret = miopenRNNBackwardWeightsSeqTensor(GetHandle(), rnnDesc, inputSeqTensor, @@ -1309,31 +1270,15 @@ int RNNSeqDriver::RunBackwardGPU() workspace_dev->GetSize(), reservespace_dev->GetMem(), reservespace_dev->GetSize()); - miopen::deref(GetHandle()).Finish(); - STOP_TIME - if(i > 0 || inflags.GetValueInt("iter") == 1) - { - float time = 0.0; - miopenGetKernelTime(GetHandle(), &time); - wl_time_backward_weight += t.gettime_ms(); - kl_time_backward_weight += time; - } + t.StopAndPush(); } - if(inflags.GetValueInt("time") == 1) - { - int n_iter = inflags.GetValueInt("iter") > 1 ? inflags.GetValueInt("iter") - 1 - : inflags.GetValueInt("iter"); - printf("GPU Kernel Time Backward Weights RNN Elapsed: %f ms\n", - kl_time_backward_weight / n_iter); - } + miopen::deref(GetHandle()).Finish(); if(WALL_CLOCK) { - int n_iter = inflags.GetValueInt("iter") > 1 ? inflags.GetValueInt("iter") - 1 - : inflags.GetValueInt("iter"); - printf("Wall-clock Time Backward Weights RNN Elapsed: %f ms\n", - wl_time_backward_weight / n_iter); + printf("Backward Weights RNN time results:\n"); + t.Print(); } dwei_dev->FromGPU(GetStream(), dwei.data()); } diff --git a/driver/rnn_verify_gemm.hpp b/driver/rnn_verify_gemm.hpp index c31f51e1c7..94f90dc579 100644 --- a/driver/rnn_verify_gemm.hpp +++ b/driver/rnn_verify_gemm.hpp @@ -107,7 +107,7 @@ void RunRNNForwardGEMMCPUVerify(miopenHandle_t handle, int wei_shift_bias = ((in_h + hy_h) * bi + (bi * hy_h + hy_h) * bi * (numlayer - 1)) * hy_h; // initial dropoput - std::vector dropout_states_host; + std::vector dropout_states_host; std::vector dropout_reservespace_host; std::vector dropout_hid_state; miopenTensorDescriptor_t dropout_inputTensor{}, dropout_outputTensor{}; @@ -115,8 +115,8 @@ void RunRNNForwardGEMMCPUVerify(miopenHandle_t handle, { size_t statesSizeInBytes = 0; miopenDropoutGetStatesSize(handle, &statesSizeInBytes); - size_t states_size = statesSizeInBytes / sizeof(prngStates); - dropout_states_host = std::vector(states_size); + size_t states_size = statesSizeInBytes / sizeof(rocrand_state_xorwow); + dropout_states_host = std::vector(states_size); InitKernelStateEmulator(dropout_states_host, dropoutDesc); std::array drop_in_len = {{batch_n, hy_h * bi}}; diff --git a/driver/rope_driver.hpp b/driver/rope_driver.hpp new file mode 100644 index 0000000000..ecaf55ac64 --- /dev/null +++ b/driver/rope_driver.hpp @@ -0,0 +1,452 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#ifndef GUARD_MIOPEN_ROPE_DRIVER_HPP +#define GUARD_MIOPEN_ROPE_DRIVER_HPP + +#include "InputFlags.hpp" +#include "driver.hpp" +#include "tensor_driver.hpp" +#include "timer.hpp" +#include "random.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include <../test/tensor_holder.hpp> +#include <../test/verify.hpp> + +template +int32_t mloRoPEForwardRunHost(miopenTensorDescriptor_t xDesc, + miopenTensorDescriptor_t cosDesc, + miopenTensorDescriptor_t sinDesc, + miopenTensorDescriptor_t yDesc, + Tgpu* x, + Tgpu* cos, + Tgpu* sin, + Tcheck* yhost) +{ + auto x_dims = miopen::deref(xDesc).GetLengths(); + auto cos_dims = miopen::deref(cosDesc).GetLengths(); + auto input_numel = + std::accumulate(x_dims.begin(), x_dims.end(), 1LL, std::multiplies()); + auto rotary_numel = + std::accumulate(cos_dims.begin(), cos_dims.end(), 1LL, std::multiplies()); + + int32_t ret = 0; + + for(size_t o = 0; o < input_numel; o++) + { + size_t freq_idx = o % rotary_numel; + Tcheck input = static_cast(x[o]); + Tcheck input_rotate_half = + (o % 2 == 0) ? static_cast(-x[o + 1]) : static_cast(x[o - 1]); + + Tcheck cos_val = static_cast(cos[freq_idx]); + Tcheck sin_val = static_cast(sin[freq_idx]); + + Tcheck val = (input * cos_val) + (input_rotate_half * sin_val); + + yhost[o] = val; + } + + return ret; +} + +template +int32_t mloRoPEBackwardRunHost(miopenTensorDescriptor_t dyDesc, + miopenTensorDescriptor_t cosDesc, + miopenTensorDescriptor_t sinDesc, + miopenTensorDescriptor_t dxDesc, + Tgpu* dy, + Tgpu* cos, + Tgpu* sin, + Tcheck* dxhost) +{ + auto dy_dims = miopen::deref(dyDesc).GetLengths(); + auto input_numel = + std::accumulate(dy_dims.begin(), dy_dims.end(), 1LL, std::multiplies()); + auto cos_dims = miopen::deref(cosDesc).GetLengths(); + auto rotary_numel = + std::accumulate(cos_dims.begin(), cos_dims.end(), 1LL, std::multiplies()); + + int32_t ret = 0; + + for(size_t o = 0; o < input_numel; o++) + { + size_t freq_idx = o % rotary_numel; + + Tcheck output_grad = static_cast(dy[o]); + Tcheck output_grad_rotate_half = + (o % 2 == 0) ? static_cast(dy[o + 1]) : static_cast(-dy[o - 1]); + + Tcheck cos_val = static_cast(cos[freq_idx]); + Tcheck sin_val = (freq_idx % 2 == 0) ? static_cast(sin[freq_idx + 1]) + : static_cast(sin[freq_idx - 1]); + + Tcheck val = (output_grad * cos_val) + (output_grad_rotate_half * sin_val); + + dxhost[o] = val; + } + + return ret; +} + +template +class RoPEDriver : public Driver +{ +public: + RoPEDriver() : Driver() + { + miopenCreateTensorDescriptor(&x_dyDesc); + miopenCreateTensorDescriptor(&cosDesc); + miopenCreateTensorDescriptor(&sinDesc); + miopenCreateTensorDescriptor(&y_dxDesc); + + data_type = miopen_type{}; + } + + int AddCmdLineArgs() override; + int ParseCmdLineArgs(int argc, char* argv[]) override; + InputFlags& GetInputFlags() override { return inflags; } + + int GetandSetData() override; + std::vector GetInputTensorLengthsFromCmdLine(); + + int AllocateBuffersAndCopy() override; + + int RunForwardGPU() override; + int RunForwardCPU(); + + int RunBackwardGPU() override; + int RunBackwardCPU(); + + Tref GetTolerance(); + int VerifyBackward() override; + int VerifyForward() override; + ~RoPEDriver() override + { + + miopenDestroyTensorDescriptor(x_dyDesc); + miopenDestroyTensorDescriptor(cosDesc); + miopenDestroyTensorDescriptor(sinDesc); + miopenDestroyTensorDescriptor(y_dxDesc); + } + +private: + InputFlags inflags; + + miopenTensorDescriptor_t x_dyDesc; + miopenTensorDescriptor_t cosDesc; + miopenTensorDescriptor_t sinDesc; + miopenTensorDescriptor_t y_dxDesc; + + std::unique_ptr x_dy_dev; + std::unique_ptr cos_dev; + std::unique_ptr sin_dev; + std::unique_ptr y_dx_dev; + + std::vector x_dy; + std::vector cos; + std::vector sin; + std::vector y_dx; + std::vector y_dxhost; +}; + +template +int RoPEDriver::ParseCmdLineArgs(int argc, char* argv[]) +{ + inflags.Parse(argc, argv); + + if(inflags.GetValueInt("time") == 1) + { + miopenEnableProfiling(GetHandle(), true); + } + return miopenStatusSuccess; +} + +template +int RoPEDriver::GetandSetData() +{ + auto inTensorParam = inflags.GetValueTensorUint64("input"); + + auto in_len = inTensorParam.lengths; + std::vector rotary_dim = {in_len[1], in_len[2], in_len[3]}; + + if(SetTensorNd(x_dyDesc, in_len, data_type) != miopenStatusSuccess) + MIOPEN_THROW("Error parsing input tensor: " + inflags.GetValueStr("input") + "."); + + if(SetTensorNd(cosDesc, rotary_dim, data_type) != miopenStatusSuccess) + MIOPEN_THROW("Error setting cos tensor."); + + if(SetTensorNd(sinDesc, rotary_dim, data_type) != miopenStatusSuccess) + MIOPEN_THROW("Error setting sin tensor."); + + if(SetTensorNd(y_dxDesc, in_len, data_type) != miopenStatusSuccess) + MIOPEN_THROW("Error setting output tensor."); + + return 0; +} + +template +int RoPEDriver::AddCmdLineArgs() +{ + inflags.AddInputFlag("forw", 'F', "1", "Run only Forward RoPE (Default=1)", "int"); + inflags.AddTensorFlag("input", 'X', "100x3x32x32", "input tensor descriptor"); + + inflags.AddInputFlag("iter", 'i', "10", "Number of Iterations (Default=10)", "int"); + inflags.AddInputFlag("verify", 'V', "1", "Verify Each Layer (Default=1)", "int"); + inflags.AddInputFlag("time", 't', "0", "Time Each Layer (Default=0)", "int"); + inflags.AddInputFlag( + "wall", 'w', "0", "Wall-clock Time Each Layer, Requires time == 1 (Default=0)", "int"); + + return miopenStatusSuccess; +} + +template +int RoPEDriver::AllocateBuffersAndCopy() +{ + const Tgpu Tgpu0val = static_cast(0.0); + const Tgpu Tgpu1val = static_cast(1.0); + const Tgpu Tgpuminus1val = static_cast(-1.0); + const Tref Tref0ref = static_cast(0.0); + size_t x_dy_sz = GetTensorSize(x_dyDesc); + size_t cos_sz = GetTensorSize(cosDesc); + size_t sin_sz = GetTensorSize(sinDesc); + size_t y_dx_sz = GetTensorSize(y_dxDesc); + + uint32_t ctx = 0; + + x_dy_dev = std::unique_ptr(new GPUMem(ctx, x_dy_sz, sizeof(Tgpu))); + cos_dev = std::unique_ptr(new GPUMem(ctx, cos_sz, sizeof(Tgpu))); + sin_dev = std::unique_ptr(new GPUMem(ctx, sin_sz, sizeof(Tgpu))); + y_dx_dev = std::unique_ptr(new GPUMem(ctx, y_dx_sz, sizeof(Tgpu))); + + x_dy = std::vector(x_dy_sz, Tgpu0val); + cos = std::vector(cos_sz, Tgpu0val); + sin = std::vector(sin_sz, Tgpu0val); + y_dx = std::vector(y_dx_sz, Tgpu0val); + y_dxhost = std::vector(y_dx_sz, Tref0ref); + + for(int i = 0; i < x_dy_sz; ++i) + { + x_dy[i] = prng::gen_A_to_B(Tgpuminus1val, Tgpu1val); + } + + if(x_dy_dev->ToGPU(GetStream(), x_dy.data()) != 0) + std::cerr << "Error copying (x) to GPU, size: " << x_dy_dev->GetSize() << std::endl; + + for(int i = 0; i < cos_sz; ++i) + { + cos[i] = prng::gen_A_to_B(Tgpuminus1val, Tgpu1val); + sin[i] = prng::gen_A_to_B(Tgpuminus1val, Tgpu1val); + } + + if(cos_dev->ToGPU(GetStream(), cos.data()) != 0) + std::cerr << "Error copying (cos) to GPU, size: " << cos_dev->GetSize() << std::endl; + + if(sin_dev->ToGPU(GetStream(), sin.data()) != 0) + std::cerr << "Error copying (sin) to GPU, size: " << sin_dev->GetSize() << std::endl; + + return miopenStatusSuccess; +} + +template +int RoPEDriver::RunForwardGPU() +{ + float kernel_total_time = 0.0; + float kernel_first_time = 0.0; + + Timer t; + START_TIME + + for(int i = 0; i < inflags.GetValueInt("iter"); ++i) + { + miopenRoPEForward(GetHandle(), + x_dyDesc, + x_dy_dev->GetMem(), + cosDesc, + cos_dev->GetMem(), + sinDesc, + sin_dev->GetMem(), + y_dxDesc, + y_dx_dev->GetMem()); + + float time = 0.0; + miopenGetKernelTime(GetHandle(), &time); + kernel_total_time += time; + if(i == 0) + kernel_first_time = time; + } + + if(inflags.GetValueInt("time") == 1) + { + STOP_TIME + int iter = inflags.GetValueInt("iter"); + if(WALL_CLOCK) + std::cout << "Wall-clock Time Forward RoPE Elapsed: " << t.gettime_ms() / iter << " ms" + << std::endl; + + float kernel_average_time = + iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; + std::cout << "GPU Kernel Time Forward RoPE Elapsed: " << kernel_average_time << " ms" + << std::endl; + } + + if(y_dx_dev->FromGPU(GetStream(), y_dx.data()) != 0) + std::cerr << "Error copying (y_dev) from GPU, size: " << y_dx_dev->GetSize() << std::endl; + + return miopenStatusSuccess; +} + +template +int RoPEDriver::RunForwardCPU() +{ + mloRoPEForwardRunHost( + x_dyDesc, cosDesc, sinDesc, y_dxDesc, x_dy.data(), cos.data(), sin.data(), y_dxhost.data()); + + return miopenStatusSuccess; +} + +template +int RoPEDriver::RunBackwardGPU() +{ + float kernel_total_time = 0.0; + float kernel_first_time = 0.0; + + Timer t; + START_TIME + + for(int i = 0; i < inflags.GetValueInt("iter"); ++i) + { + miopenRoPEBackward(GetHandle(), + x_dyDesc, + x_dy_dev->GetMem(), + cosDesc, + cos_dev->GetMem(), + sinDesc, + sin_dev->GetMem(), + y_dxDesc, + y_dx_dev->GetMem()); + + float time = 0.0; + miopenGetKernelTime(GetHandle(), &time); + kernel_total_time += time; + if(i == 0) + kernel_first_time = time; + } + + if(inflags.GetValueInt("time") == 1) + { + STOP_TIME + int iter = inflags.GetValueInt("iter"); + if(WALL_CLOCK) + std::cout << "Wall-clock Time Backward RoPE Elapsed: " << t.gettime_ms() / iter << " ms" + << std::endl; + + float kernel_average_time = + iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; + std::cout << "GPU Kernel Time Backward RoPE Elapsed: " << kernel_average_time << " ms" + << std::endl; + } + + if(y_dx_dev->FromGPU(GetStream(), y_dx.data()) != 0) + std::cerr << "Error copying (y_dx_dev) from GPU, size: " << y_dx_dev->GetSize() + << std::endl; + + return miopenStatusSuccess; +} + +template +int RoPEDriver::RunBackwardCPU() +{ + mloRoPEBackwardRunHost( + x_dyDesc, cosDesc, sinDesc, y_dxDesc, x_dy.data(), cos.data(), sin.data(), y_dxhost.data()); + + return miopenStatusSuccess; +} + +template +Tref RoPEDriver::GetTolerance() +{ + // Computation error of fp16 is ~2^13 (=8192) bigger than + // the one of fp32 because mantissa is shorter by 13 bits. + auto tolerance = std::is_same::value ? 1.5e-6 : 8.2e-3; + + // bf16 mantissa has 7 bits, by 3 bits shorter than fp16. + if(std::is_same::value) + tolerance *= 8.0; + return tolerance; +} + +template +int RoPEDriver::VerifyForward() +{ + RunForwardCPU(); + const Tref tolerance = GetTolerance(); + + auto error = miopen::rms_range(y_dxhost, y_dx); + + if(!std::isfinite(error) || error > tolerance) + { + std::cout << "Forward RoPE FAILED: " << error << " > " << tolerance << std::endl; + return EC_VerifyFwd; + } + else + { + std::cout << "Forward RoPE Verifies OK on CPU reference (" << error << " < " << tolerance + << ')' << std::endl; + } + + return miopenStatusSuccess; +} + +template +int RoPEDriver::VerifyBackward() +{ + RunBackwardCPU(); + const Tref tolerance = GetTolerance(); + + auto error = miopen::rms_range(y_dxhost, y_dx); + + if(!std::isfinite(error) || error > tolerance) + { + std::cout << "Backward RoPE FAILED: " << error << " > " << tolerance << std::endl; + return EC_VerifyBwd; + } + else + { + std::cout << "Backward RoPE Verifies OK on CPU reference (" << error << " < " << tolerance + << ')' << std::endl; + } + + return miopenStatusSuccess; +} + +#endif // GUARD_MIOPEN_ROPE_DRIVER_HPP diff --git a/driver/softmarginloss_driver.hpp b/driver/softmarginloss_driver.hpp new file mode 100644 index 0000000000..2b79df80a3 --- /dev/null +++ b/driver/softmarginloss_driver.hpp @@ -0,0 +1,581 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include "InputFlags.hpp" +#include "driver.hpp" +#include "tensor_driver.hpp" +#include "timer.hpp" +#include "random.hpp" +#include +#include +#include +#include +#include +#include <../test/tensor_holder.hpp> +#include <../test/verify.hpp> +#include + +template +int32_t mloSoftMarginLossForwardRunHost(miopenTensorDescriptor_t inputDesc, + miopenTensorDescriptor_t targetDesc, + miopenTensorDescriptor_t outputDesc, + Tgpu* input, + Tgpu* target, + Tcheck* outputhost, + miopenLossReductionMode_t reduction_mode) +{ + auto input_numel = miopen::deref(inputDesc).GetElementSize(); + auto i_tv = miopen::get_inner_expanded_tv<5>(miopen::deref(inputDesc)); + auto t_tv = miopen::get_inner_expanded_tv<5>(miopen::deref(targetDesc)); + auto o_tv = miopen::get_inner_expanded_tv<5>(miopen::deref(outputDesc)); + + int32_t ret = miopenStatusSuccess; + + double sum_loss = 0; + for(size_t gid = 0; gid < input_numel; gid++) + { + tensor_layout_t<5> idx(i_tv, gid); + double i = input[i_tv.get_tensor_view_idx(idx)]; + double t = target[t_tv.get_tensor_view_idx(idx)]; + double loss = log1p(exp(-i * t)); + if(reduction_mode != MIOPEN_LOSS_REDUCTION_NONE) + sum_loss += loss; + else + outputhost[o_tv.get_tensor_view_idx(idx)] = static_cast(loss); + } + if(reduction_mode == MIOPEN_LOSS_REDUCTION_MEAN) + outputhost[0] = static_cast(sum_loss / input_numel); + else if(reduction_mode == MIOPEN_LOSS_REDUCTION_SUM) + outputhost[0] = static_cast(sum_loss); + + return ret; +} + +template +int32_t mloSoftMarginLossBackwardRunHost(miopenTensorDescriptor_t inputDesc, + miopenTensorDescriptor_t targetDesc, + miopenTensorDescriptor_t dODesc, + miopenTensorDescriptor_t dIDesc, + Tgpu* input, + Tgpu* target, + Tgpu* dO, + Tcheck* dIhost, + miopenLossReductionMode_t reduction_mode) +{ + auto input_numel = miopen::deref(inputDesc).GetElementSize(); + auto i_tv = miopen::get_inner_expanded_tv<5>(miopen::deref(inputDesc)); + auto t_tv = miopen::get_inner_expanded_tv<5>(miopen::deref(targetDesc)); + auto dO_tv = miopen::get_inner_expanded_tv<5>(miopen::deref(dODesc)); + auto dI_tv = miopen::get_inner_expanded_tv<5>(miopen::deref(dIDesc)); + + int32_t ret = miopenStatusSuccess; + + par_ford(input_numel)([&](size_t gid) { + tensor_layout_t<5> idx(i_tv, gid); + double i = input[i_tv.get_tensor_view_idx(idx)]; + double t = target[t_tv.get_tensor_view_idx(idx)]; + double _dO = dO[dO_tv.get_tensor_view_idx(idx)]; + if(reduction_mode == MIOPEN_LOSS_REDUCTION_MEAN) + dIhost[dI_tv.get_tensor_view_idx(idx)] = + static_cast(-t / (exp(i * t) + 1) * _dO / input_numel); + else + dIhost[dI_tv.get_tensor_view_idx(idx)] = + static_cast(-t / (exp(i * t) + 1) * _dO); + }); + return ret; +} + +template +class SoftMarginLossDriver : public Driver +{ +public: + SoftMarginLossDriver() : Driver() + { + miopenCreateTensorDescriptor(&inputDesc); + miopenCreateTensorDescriptor(&targetDesc); + miopenCreateTensorDescriptor(&outputDesc); + miopenCreateTensorDescriptor(&dODesc); + miopenCreateTensorDescriptor(&dIDesc); + + data_type = miopen_type{}; + } + + int AddCmdLineArgs() override; + int ParseCmdLineArgs(int argc, char* argv[]) override; + InputFlags& GetInputFlags() override { return inflags; } + + int GetandSetData() override; + + int AllocateBuffersAndCopy() override; + + int RunForwardGPU() override; + int RunForwardCPU(); + + int RunBackwardGPU() override; + int RunBackwardCPU(); + + Tref GetTolerance(); + int VerifyBackward() override; + int VerifyForward() override; + ~SoftMarginLossDriver() override + { + miopenDestroyTensorDescriptor(inputDesc); + miopenDestroyTensorDescriptor(targetDesc); + miopenDestroyTensorDescriptor(outputDesc); + miopenDestroyTensorDescriptor(dODesc); + miopenDestroyTensorDescriptor(dIDesc); + } + +private: + InputFlags inflags; + + // forw = 0 -> run both fw, bw, = 1 -> run only fw, = 2 -> run only bw + int forw; + + miopenTensorDescriptor_t inputDesc; + miopenTensorDescriptor_t targetDesc; + miopenTensorDescriptor_t outputDesc; + miopenTensorDescriptor_t dODesc; + miopenTensorDescriptor_t dIDesc; + + std::unique_ptr in_dev; + std::unique_ptr target_dev; + std::unique_ptr out_dev; + std::unique_ptr dO_dev; + std::unique_ptr dI_dev; + std::unique_ptr workspace_dev; + + std::vector in; + std::vector target; + std::vector out; + std::vector outhost; + std::vector dO; + std::vector dI; + std::vector dIhost; + + miopenLossReductionMode_t reduction_mode; + size_t ws_sizeInBytes; +}; + +template +int SoftMarginLossDriver::AddCmdLineArgs() +{ + inflags.AddInputFlag("forw", + 'F', + "1", + "Run Forward or Backward. 0 to run both Fw and Bw, 1 to run only Fw, 2 to " + "run only Bw (Default=1)", + "int"); + inflags.AddInputFlag( + "dim", 'D', "4x25x4x25", "Dim of input tensor (Default=4x25x4x25)", "tensor"); + inflags.AddInputFlag("stride", + 'S', + "-1", + "Stride of input tensor. Tensor is contiguous or not depend on this " + "flag. Example uncont tensor: -D 32x80x870 -S 69600x1x80. If not specify " + "this flag (-S -1), " + "stride will be auto calculated based on flag C. (Default=-1) ", + "tensor"); + inflags.AddInputFlag( + "contiguous", + 'C', + "1", + "Tensor is contiguous or not. This flag will be ignored if flag S != -1 (Default=1)", + "int"); + inflags.AddInputFlag("iter", 'i', "10", "Number of Iterations (Default=10)", "int"); + inflags.AddInputFlag("verify", 'V', "1", "Verify Each Layer (Default=1)", "int"); + inflags.AddInputFlag("time", 't', "0", "Time Each Layer (Default=0)", "int"); + inflags.AddInputFlag( + "wall", 'w', "0", "Wall-clock Time Each Layer, Requires time == 1 (Default=0)", "int"); + inflags.AddInputFlag("reduce", + 'R', + "none", + "Specifies the reduction to apply to the output ('none'|'mean'|'sum') " + "(Default=none to indicate no reduction)", + "string"); + + return miopenStatusSuccess; +} + +template +int SoftMarginLossDriver::ParseCmdLineArgs(int argc, char* argv[]) +{ + inflags.Parse(argc, argv); + + auto reduction = inflags.GetValueStr("reduce"); + if(reduction != "none" && reduction != "mean" && reduction != "sum") + return miopenStatusInvalidValue; + if(inflags.GetValueInt("time") == 1) + { + miopenEnableProfiling(GetHandle(), true); + } + forw = inflags.GetValueInt("forw"); + return miopenStatusSuccess; +} + +template +int SoftMarginLossDriver::GetandSetData() +{ + // Set input tensor description + std::vector in_len = inflags.GetValueTensor("dim").lengths; + if(inflags.GetValueStr("stride") != "-1") + { + std::vector in_stride = inflags.GetValueTensor("stride").lengths; + SetTensorNd(inputDesc, in_len, in_stride, data_type); + } + else + { + if(inflags.GetValueInt("contiguous") == 1) + { + SetTensorNd(inputDesc, in_len, data_type); + } + else + { + std::vector in_strides(in_len.size()); + in_strides.back() = 1; + for(int i = in_len.size() - 2; i >= 0; --i) + in_strides[i] = in_strides[i + 1] * in_len[i + 1]; + in_strides[0] *= 2; + SetTensorNd(inputDesc, in_len, in_strides, data_type); + } + } + + // Driver will only run contiguous target and output tensor. Please do that too with ROCm + // benchmark + + // Set target tensor description + SetTensorNd(targetDesc, in_len, data_type); + + // Set reduction_mode + auto reduction = inflags.GetValueStr("reduce"); + if(reduction == "none") + reduction_mode = MIOPEN_LOSS_REDUCTION_NONE; + else if(reduction == "mean") + reduction_mode = MIOPEN_LOSS_REDUCTION_MEAN; + else if(reduction == "sum") + reduction_mode = MIOPEN_LOSS_REDUCTION_SUM; + + // Set output tensor description (forw = 1 or = 0) + if(forw == 0 || forw == 1) + { + if(reduction == "none") + SetTensorNd(outputDesc, in_len, data_type); + else + { + std::vector out_lens = {1}; + SetTensorNd(outputDesc, out_lens, data_type); + } + } + + // Set dO, dI tensor description (forw = 2 or 0) + if(forw == 0 || forw == 2) + { + SetTensorNd(dODesc, in_len, data_type); + SetTensorNd(dIDesc, in_len, data_type); + } + + return miopenStatusSuccess; +} + +template +int SoftMarginLossDriver::AllocateBuffersAndCopy() +{ + // Only input tensor can be unpacked in driver for this op. Please do that too with ROCm + // benchmark + + uint32_t ctx = 0; + + // for unpacked tensor, we need to use GetTensorSpace instead of GetTensorSize + size_t in_sz = GetTensorSpace(inputDesc); + size_t target_sz = GetTensorSpace(targetDesc); + in_dev = std::make_unique(ctx, in_sz, sizeof(Tgpu)); + target_dev = std::make_unique(ctx, target_sz, sizeof(Tgpu)); + in = std::vector(in_sz); + target = std::vector(target_sz); + for(int i = 0; i < in_sz; i++) + { + in[i] = prng::gen_A_to_B(static_cast(-1), static_cast(1)); + } + // -1 or 1 + for(int i = 0; i < target_sz; i++) + { + target[i] = + (prng::gen_A_to_B(static_cast(0), static_cast(2)) == 0) ? -1 + : 1; + } + if(in_dev->ToGPU(GetStream(), in.data()) != 0) + { + std::cerr << "Error copying (in) to GPU, size: " << in_dev->GetSize() << std::endl; + return miopenStatusAllocFailed; + } + + if(target_dev->ToGPU(GetStream(), target.data()) != 0) + { + std::cerr << "Error copying (target) to GPU, size: " << target_dev->GetSize() << std::endl; + return miopenStatusAllocFailed; + } + + if(forw == 0 || forw == 1) + { + size_t out_sz = GetTensorSpace(outputDesc); + + miopenGetSoftMarginLossForwardWorkspaceSize( + GetHandle(), inputDesc, targetDesc, outputDesc, reduction_mode, &ws_sizeInBytes); + if(ws_sizeInBytes == static_cast(-1)) + { + return miopenStatusAllocFailed; + } + + out_dev = std::make_unique(ctx, out_sz, sizeof(Tgpu)); + if(ws_sizeInBytes == 0) + workspace_dev = nullptr; + else + workspace_dev = std::make_unique(ctx, ws_sizeInBytes, sizeof(std::byte)); + out = std::vector(out_sz, static_cast(0)); + outhost = std::vector(out_sz, static_cast(0)); + if(out_dev->ToGPU(GetStream(), out.data()) != 0) + { + std::cerr << "Error copying (out) to GPU, size: " << out_dev->GetSize() << std::endl; + return miopenStatusAllocFailed; + } + } + if(forw == 0 || forw == 2) + { + size_t dO_sz = GetTensorSpace(dODesc); + size_t dI_sz = GetTensorSpace(dIDesc); + dO_dev = std::make_unique(ctx, dO_sz, sizeof(Tgpu)); + dI_dev = std::make_unique(ctx, dI_sz, sizeof(Tgpu)); + // similar to output_grad = torch.one_likes(output) + dO = std::vector(dO_sz, static_cast(1)); + dI = std::vector(dI_sz, static_cast(0)); + dIhost = std::vector(dI_sz, static_cast(0)); + + if(dO_dev->ToGPU(GetStream(), dO.data()) != 0) + { + std::cerr << "Error copying (dO) to GPU, size: " << dO_dev->GetSize() << std::endl; + return miopenStatusAllocFailed; + } + if(dI_dev->ToGPU(GetStream(), dI.data()) != 0) + { + std::cerr << "Error copying (dI) to GPU, size: " << dI_dev->GetSize() << std::endl; + return miopenStatusAllocFailed; + } + } + + return miopenStatusSuccess; +} + +template +int SoftMarginLossDriver::RunForwardGPU() +{ + float kernel_total_time = 0; + float kernel_first_time = 0; + + Timer t; + START_TIME + + for(int i = 0; i < inflags.GetValueInt("iter"); i++) + { + miopenStatus_t status = miopenSoftMarginLossForward( + GetHandle(), + inputDesc, + in_dev->GetMem(), + targetDesc, + target_dev->GetMem(), + outputDesc, + out_dev->GetMem(), + reduction_mode, + (workspace_dev == nullptr) ? nullptr : workspace_dev->GetMem(), + ws_sizeInBytes); + + MIOPEN_THROW_IF(status != miopenStatusSuccess, "Error in miopenSoftMarginLossForward"); + + float time = 0.0; + miopenGetKernelTime(GetHandle(), &time); + kernel_total_time += time; + if(i == 0) + kernel_first_time = time; + } + + if(inflags.GetValueInt("time") == 1) + { + STOP_TIME + int iter = inflags.GetValueInt("iter"); + if(WALL_CLOCK) + std::cout << "Wall-clock Time Forward SoftMarginLoss Elapsed: " << t.gettime_ms() / iter + << " ms" << std::endl; + + float kernel_average_time = + iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; + std::cout << "GPU Kernel Time Forward SoftMarginLoss Elapsed: " << kernel_average_time + << " ms" << std::endl; + } + + if(out_dev->FromGPU(GetStream(), out.data()) != 0) + { + std::cerr << "Error copying (out_dev) from GPU, size: " << out_dev->GetSize() << std::endl; + return miopenStatusInternalError; + } + + return miopenStatusSuccess; +} + +template +int SoftMarginLossDriver::RunForwardCPU() +{ + return mloSoftMarginLossForwardRunHost(inputDesc, + targetDesc, + outputDesc, + in.data(), + target.data(), + outhost.data(), + reduction_mode); +} + +template +int SoftMarginLossDriver::RunBackwardGPU() +{ + float kernel_total_time = 0; + float kernel_first_time = 0; + + Timer t; + START_TIME + + for(int i = 0; i < inflags.GetValueInt("iter"); i++) + { + miopenStatus_t status = miopenSoftMarginLossBackward(GetHandle(), + inputDesc, + in_dev->GetMem(), + targetDesc, + target_dev->GetMem(), + dODesc, + dO_dev->GetMem(), + dIDesc, + dI_dev->GetMem(), + reduction_mode); + + MIOPEN_THROW_IF(status != miopenStatusSuccess, "Error in miopenSoftMarginLossBackward"); + + float time = 0.0; + miopenGetKernelTime(GetHandle(), &time); + kernel_total_time += time; + if(i == 0) + kernel_first_time = time; + } + + if(inflags.GetValueInt("time") == 1) + { + STOP_TIME + int iter = inflags.GetValueInt("iter"); + if(WALL_CLOCK) + std::cout << "Wall-clock Time Backward SoftMarginLoss Elapsed: " + << t.gettime_ms() / iter << " ms" << std::endl; + + float kernel_average_time = + iter > 1 ? (kernel_total_time - kernel_first_time) / (iter - 1) : kernel_first_time; + std::cout << "GPU Kernel Time Backward SoftMarginLoss Elapsed: " << kernel_average_time + << " ms" << std::endl; + } + + if(dI_dev->FromGPU(GetStream(), dI.data()) != 0) + { + std::cerr << "Error copying (dI_dev) from GPU, size: " << dI_dev->GetSize() << std::endl; + return miopenStatusInternalError; + } + + return miopenStatusSuccess; +} + +template +int SoftMarginLossDriver::RunBackwardCPU() +{ + return mloSoftMarginLossBackwardRunHost(inputDesc, + targetDesc, + dODesc, + dIDesc, + in.data(), + target.data(), + dO.data(), + dIhost.data(), + reduction_mode); +} + +template +Tref SoftMarginLossDriver::GetTolerance() +{ + Tref tolerance = std::numeric_limits::epsilon() * 10; + return tolerance; +} + +template +int SoftMarginLossDriver::VerifyForward() +{ + // Please note that with fp16 reduction, if input tensor is too big the result will be wrong + // because of fp16 overflow / underflow. For sum reduction, try with input tensor >= 90k + // elements and output will be overflow. + // Example: ./MIOpenDriver softmarginlossfp16 -t 1 -R sum -F 1 -D 90000 + RunForwardCPU(); + const Tref tolerance = GetTolerance(); + auto error = miopen::rms_range(outhost, out); + if(!std::isfinite(error) || error > tolerance) + { + std::cout << "Forward SoftMarginLoss FAILED: " << error << " > " << tolerance << std::endl; + return EC_VerifyFwd; + } + else + { + std::cout << "Forward SoftMarginLoss Verifies OK on CPU reference (" << error << " < " + << tolerance << ')' << std::endl; + } + + return miopenStatusSuccess; +} + +template +int SoftMarginLossDriver::VerifyBackward() +{ + // Please note that with fp16 MEAN reduction backward, if input tensor is too big the result + // will be wrong because of fp16 underflow (divisor is too big). + // Example: ./MIOpenDriver softmarginlossfp16 -t 1 -R mean -F 2 -D 500000 + // SUM reduction backward still worked because this case divisor = 1, nothing special. + RunBackwardCPU(); + const Tref tolerance = GetTolerance(); + auto error = miopen::rms_range(dIhost, dI); + + if(!std::isfinite(error) || error > tolerance) + { + std::cout << "Backward SoftMarginLoss FAILED: " << error << " > " << tolerance << std::endl; + return EC_VerifyBwd; + } + else + { + std::cout << "Backward SoftMarginLoss Verifies OK on CPU reference (" << error << " < " + << tolerance << ')' << std::endl; + } + + return miopenStatusSuccess; +} diff --git a/driver/t5layernorm_driver.hpp b/driver/t5layernorm_driver.hpp index 94a4f6b934..192cd5d62b 100644 --- a/driver/t5layernorm_driver.hpp +++ b/driver/t5layernorm_driver.hpp @@ -221,7 +221,6 @@ class T5LayerNormDriver : public Driver private: InputFlags inflags; - int forw; int dim_size; miopenTensorDescriptor_t xDesc; @@ -295,7 +294,7 @@ int T5LayerNormDriver::GetandSetData() MIOPEN_THROW("Error setting weight tensor."); if(SetTensorNd(yDesc, in_len, data_type) != miopenStatusSuccess) - MIOPEN_THROW("Error setting doutput tensor."); + MIOPEN_THROW("Error setting output tensor."); if(SetTensorNd(rstdDesc, outer_len, data_type) != miopenStatusSuccess) MIOPEN_THROW("Error setting rstd tensor."); @@ -318,12 +317,12 @@ int T5LayerNormDriver::GetandSetData() template int T5LayerNormDriver::AddCmdLineArgs() { - inflags.AddInputFlag("forw", 'F', "1", "Run only Forward T5LayerNorm (Default=1)", "int"); + inflags.AddInputFlag("forw", 'F', "0", "Run only Forward T5LayerNorm (Default=1)", "int"); inflags.AddTensorFlag("input", 'X', "100x3x32x32", "input tensor descriptor"); inflags.AddInputFlag("eps", 'e', "0.00001", "Alpha (Default=0.00001)", "double"); inflags.AddInputFlag( - "mode", 'm', "0", "elemwise affine mode (0), weight mode (1) (Default=0)", "int"); + "mode", 'm', "5", "elemwise affine mode (5), weight mode (6) (Default=5)", "int"); inflags.AddInputFlag("iter", 'i', "10", "Number of Iterations (Default=10)", "int"); inflags.AddInputFlag("verify", 'V', "1", "Verify Each Layer (Default=1)", "int"); diff --git a/driver/tensor_driver.hpp b/driver/tensor_driver.hpp index f6868fab98..c353a6ee11 100644 --- a/driver/tensor_driver.hpp +++ b/driver/tensor_driver.hpp @@ -173,6 +173,13 @@ inline int SetTensorNd(miopenTensorDescriptor_t t, return miopenSetTensorDescriptor(t, data_type, len.size(), len.data(), nullptr); } +inline int SetTensorNd(miopenTensorDescriptor_t t, + std::vector& len, + miopenDataType_t data_type = miopenFloat) +{ + return miopenSetTensorDescriptorV2(t, data_type, len.size(), len.data(), nullptr); +} + inline int SetTensorNd(miopenTensorDescriptor_t t, std::vector& len, std::vector& strides, diff --git a/driver/timer.hpp b/driver/timer.hpp index ca5dfdc4e6..b0daff8a51 100644 --- a/driver/timer.hpp +++ b/driver/timer.hpp @@ -139,4 +139,142 @@ class Timer2 std::chrono::time_point et; }; +class RNNCombTimeLoger +{ +public: + RNNCombTimeLoger(hipStream_t main_stream, size_t size, int mode) + : stream(main_stream), clockMode(static_cast(mode)) + { + if(clockMode != ClockMode::Disabled) + { + hostTimePerLaunch.reserve(size); + + startEvent.reserve(size); + endEvent.reserve(size); + for(auto i = size; i > 0; --i) + { + startEvent.push_back(miopen::make_hip_event()); + endEvent.push_back(miopen::make_hip_event()); + } + } + } + + void Start() + { + if(clockMode == ClockMode::Disabled) + return; + + auto launchCount = hostTimePerLaunch.size(); + + if(launchCount >= startEvent.size()) + { + printf("Executed more iterations than planned\n"); + return; + } + + hipEventRecord(startEvent[launchCount].get(), stream); + st = std::chrono::steady_clock::now(); + } + void StopAndPush() + { + if(clockMode == ClockMode::Disabled) + return; + + auto end = std::chrono::steady_clock::now(); + auto launchCount = hostTimePerLaunch.size(); + + if(launchCount >= endEvent.size()) + { + printf("Executed more iterations than planned\n"); + return; + } + hipEventRecord(endEvent[launchCount].get(), stream); + + if(clockMode == ClockMode::OldWallClock) + { + std::chrono::time_point st2 = + std::chrono::steady_clock::now(); + + hipEventSynchronize(endEvent[launchCount].get()); + + std::chrono::time_point end2 = + std::chrono::steady_clock::now(); + + hostTimePerLaunch.push_back( + std::chrono::duration_cast>(end - st) + .count() + + std::chrono::duration_cast>(end2 - st2) + .count()); + } + else + { + if(clockMode == ClockMode::SeparateClocksSynced) + hipEventSynchronize(endEvent[launchCount].get()); + + hostTimePerLaunch.push_back( + std::chrono::duration_cast>(end - st) + .count()); + } + } + + void Print() const + { + auto n_iter = hostTimePerLaunch.size(); + if(clockMode == ClockMode::Disabled || n_iter == 0) + return; + + float gpu_avg = 0.0f; + float host_avg = 0.0f; + float gpu_time = 0.0f; + + if(clockMode == ClockMode::SeparateClocksNotSynced) + hipEventSynchronize(endEvent[n_iter - 1].get()); + + for(auto i = 0ull; i < n_iter; ++i) + { + hipEventElapsedTime(&gpu_time, startEvent[i].get(), endEvent[i].get()); + + if(clockMode != ClockMode::OldWallClock) + { + printf("launch#%llu, host_time= %f ms, gpu_time= %f ms\n", + i, + hostTimePerLaunch[i], + gpu_time); + } + + if(i > 0) + { + gpu_avg += gpu_time; + host_avg += hostTimePerLaunch[i]; + } + } + + if(n_iter == 1) + hipEventElapsedTime(&gpu_time, startEvent[0].get(), endEvent[0].get()); + + printf("GPU Kernel Time Elapsed: %f ms\n", n_iter > 1 ? gpu_avg / (n_iter - 1) : gpu_time); + printf("Wall-clock Time Elapsed: %f ms\n", + n_iter > 1 ? host_avg / (n_iter - 1) : hostTimePerLaunch[0]); + } + + enum class ClockMode + { + Disabled = 0, + OldWallClock = 1, + SeparateClocksSynced = 2, + SeparateClocksNotSynced = 3 + }; + +private: + std::vector hostTimePerLaunch; + + std::vector startEvent; + std::vector endEvent; + + hipStream_t stream; + std::chrono::time_point st; + + ClockMode clockMode; +}; + #endif // GUARD_MIOPEN_TIMER_HPP diff --git a/driver/xorwow_skipahead_generator.hpp b/driver/xorwow_skipahead_generator.hpp deleted file mode 100644 index d7d1fecd16..0000000000 --- a/driver/xorwow_skipahead_generator.hpp +++ /dev/null @@ -1,282 +0,0 @@ -/******************************************************************************* - * - * MIT License - * - * Copyright (c) 2019 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - *******************************************************************************/ -#ifndef GUARD_MIOPEN_XORWOW_SKIPAHEAD_GENERATOR_HPP -#define GUARD_MIOPEN_XORWOW_SKIPAHEAD_GENERATOR_HPP - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define XORWOW_DIM 5 -#define XORWOW_BITS 32 -#define XORWOW_PRECALC_MATRICES_SZ (XORWOW_BITS * XORWOW_DIM * XORWOW_DIM) -#define XORWOW_PRECALC_MATRICES_NUM 32 -#define XORWOW_JUMP_LOG2 2 -#define XORWOW_JUMP_LOG2_MASK ((1 << XORWOW_JUMP_LOG2) - 1) -#define XORWOW_PRECALC_MATRICES_NUM_DEV 64 -#define XORWOW_JUMP_LOG2_DEV 1 -#define XORWOW_SEQUENCE_JUMP_LOG2 67 - -inline unsigned int xorwow_next(prngStates* cur_state) -{ - - const unsigned int t = cur_state->x ^ (cur_state->x >> 2); - cur_state->x = cur_state->y; - cur_state->y = cur_state->z; - cur_state->z = cur_state->w; - cur_state->w = cur_state->v; - cur_state->v = (cur_state->v ^ (cur_state->v << 4)) ^ (t ^ (t << 1)); - - cur_state->d += 362437; - - return cur_state->d + cur_state->v; -} - -// multiply vector by 32-bit-unit matrix, store result in vector -inline void mat_vec(const unsigned int* matrix, unsigned int* vector) -{ - unsigned int result[XORWOW_DIM] = {0}; - for(unsigned int i = 0; i < XORWOW_DIM; i++) - { - for(unsigned int j = 0; j < XORWOW_BITS; j++) - { - if(bool(vector[i] & (1U << j))) - { - std::transform(result, - result + XORWOW_DIM, - matrix + (XORWOW_DIM * (i * XORWOW_BITS + j)), - result, - std::bit_xor{}); - } - } - } - std::copy(std::begin(result), std::end(result), vector); -} - -// multiply matrixA by matrixB, store result in matrixA, matrixA and matrixB address cannot be the -// same -inline void mat_mat(unsigned int* matrixA, const unsigned int* matrixB) -{ - for(int i = 0; i < XORWOW_DIM * XORWOW_BITS; i++) - { - mat_vec(matrixB, matrixA + i * XORWOW_DIM); - } -} - -// generate identity matrix -inline void mat_identity(unsigned int* matrix) -{ - for(unsigned int i = 0; i < XORWOW_DIM; i++) - { - for(unsigned int j = 0; j < XORWOW_BITS; j++) - { - for(unsigned int k = 0; k < XORWOW_DIM; k++) - { - matrix[(i * XORWOW_BITS + j) * XORWOW_DIM + k] = i == k ? (1 << j) : 0; - } - } - } -} - -// compute matrix^power, store result in matrixP -inline void mat_pow(unsigned int* matrixP, const unsigned int* matrix, unsigned long long power) -{ - mat_identity(matrixP); - - unsigned int matrixA[XORWOW_PRECALC_MATRICES_SZ]; - unsigned int matrixB[XORWOW_PRECALC_MATRICES_SZ]; - std::copy(matrix, matrix + XORWOW_PRECALC_MATRICES_SZ, std::begin(matrixA)); - while(bool(power)) - { - if(bool(power & 1)) - { - mat_mat(matrixP, matrixA); - } - - std::copy(std::begin(matrixA), std::end(matrixA), std::begin(matrixB)); - mat_mat(matrixA, matrixB); - power >>= 1; - } -} - -// Generate matrix one-step advanced -inline void skipahead_one_step(unsigned int* matrix) -{ - xorwowStates init_state; - - for(unsigned int i = 0; i < XORWOW_DIM; i++) - { - for(unsigned int j = 0; j < XORWOW_BITS; j++) - { - unsigned int* p = &(init_state.x); - for(unsigned int k = 0; k < XORWOW_DIM; k++) - { - *(p + k) = i == k ? (1 << j) : 0; - } - init_state.d = 0; - - xorwow_next(&init_state); - - for(int k = 0; k < XORWOW_DIM; k++) - { - matrix[(i * XORWOW_BITS + j) * XORWOW_DIM + k] = *(p + k); - } - } - } -} - -// Generate (2^67)-step-ahead matrices -inline void generate_skipahead_matrices(unsigned int* matrix, bool is_skip_seq, bool is_device) -{ - unsigned int matrixA[XORWOW_PRECALC_MATRICES_SZ]; - unsigned int matrixB[XORWOW_PRECALC_MATRICES_SZ]; - skipahead_one_step(matrixA); - - // skipahead sequence - if(is_skip_seq) - { - // split A^(2^67) - // A^(2^33) - mat_pow(matrixB, matrixA, 1ULL << (XORWOW_SEQUENCE_JUMP_LOG2 / 2)); - // (A^(2^33))^(2^34) - mat_pow( - matrixA, matrixB, 1ULL << (XORWOW_SEQUENCE_JUMP_LOG2 - XORWOW_SEQUENCE_JUMP_LOG2 / 2)); - } - - std::copy(std::begin(matrixA), std::end(matrixA), matrix); - for(int k = 1; k < (is_device ? XORWOW_PRECALC_MATRICES_NUM_DEV : XORWOW_PRECALC_MATRICES_NUM); - k++) - { - std::copy(std::begin(matrixA), std::end(matrixA), std::begin(matrixB)); - mat_pow(matrixA, matrixB, 1ULL << (is_device ? XORWOW_JUMP_LOG2_DEV : XORWOW_JUMP_LOG2)); - std::copy(std::begin(matrixA), std::end(matrixA), &matrix[k * XORWOW_PRECALC_MATRICES_SZ]); - } -} - -// write macros in file -inline void write_macro(std::ofstream& os, bool is_device) -{ - os << "#define XORWOW_DIM " << XORWOW_DIM << std::endl; - os << "#define XORWOW_BITS " << XORWOW_BITS << std::endl; - os << "#define XORWOW_PRECALC_MATRICES_SZ (XORWOW_BITS * XORWOW_DIM * XORWOW_DIM)" << std::endl; - os << "#define XORWOW_PRECALC_MATRICES_NUM " - << (is_device ? XORWOW_PRECALC_MATRICES_NUM_DEV : XORWOW_PRECALC_MATRICES_NUM) << std::endl; - os << "#define XORWOW_JUMP_LOG2 " << (is_device ? XORWOW_JUMP_LOG2_DEV : XORWOW_JUMP_LOG2) - << std::endl; - os << "#define XORWOW_JUMP_LOG2_MASK ((1 << XORWOW_JUMP_LOG2) - 1)" << std::endl; - os << "#define XORWOW_SEQUENCE_JUMP_LOG2 67" << std::endl; - os << std::endl; -} - -// write matrices in file -inline void -write_mat(std::ofstream& os, const std::string name, unsigned int* matrix, bool is_device) -{ - os << "static " << (is_device ? "__constant " : "const ") << "unsigned int " << name - << "[XORWOW_PRECALC_MATRICES_NUM][XORWOW_PRECALC_MATRICES_SZ] = {" << std::endl; - for(int k = 0; k < (is_device ? XORWOW_PRECALC_MATRICES_NUM_DEV : XORWOW_PRECALC_MATRICES_NUM); - k++) - { - os << " {"; - for(int j = 0; j < XORWOW_PRECALC_MATRICES_SZ; j++) - { - os << matrix[k * XORWOW_PRECALC_MATRICES_SZ + j] << ", "; - } - os << "}," << std::endl; - } - os << "};" << std::endl; - os << std::endl; -} - -// generate header files with precalculated skip-ahead matrices -inline void generate_skipahead_file() -{ - static unsigned int skipahead_matrices[XORWOW_PRECALC_MATRICES_NUM][XORWOW_PRECALC_MATRICES_SZ]; - static unsigned int skipahead_matrices_sequence[XORWOW_PRECALC_MATRICES_NUM] - [XORWOW_PRECALC_MATRICES_SZ]; - - static unsigned int skipahead_matrices_dev[XORWOW_PRECALC_MATRICES_NUM_DEV] - [XORWOW_PRECALC_MATRICES_SZ]; - static unsigned int skipahead_matrices_sequence_dev[XORWOW_PRECALC_MATRICES_NUM_DEV] - [XORWOW_PRECALC_MATRICES_SZ]; - - generate_skipahead_matrices(&skipahead_matrices[0][0], false, false); - generate_skipahead_matrices(&skipahead_matrices_sequence[0][0], true, false); - - generate_skipahead_matrices(&skipahead_matrices_dev[0][0], false, true); - generate_skipahead_matrices(&skipahead_matrices_sequence_dev[0][0], true, true); - - std::ofstream os; - os.open("../src/include/miopen/precalc_xorwow_skipahead_matrices.hpp"); - write_macro(os, false); - write_mat(os, - "precalc_xorwow_skipahead_matrices", - static_cast(&skipahead_matrices[0][0]), - false); - os.close(); - os.clear(); - - os.open("../src/kernels/precalc_xorwow_skipahead_matrices_kernel.h"); - write_macro(os, true); - write_mat(os, - "precalc_xorwow_skipahead_matrices", - static_cast(&skipahead_matrices_dev[0][0]), - true); - os.close(); - os.clear(); - - os.open("../src/include/miopen/precalc_xorwow_skipahead_sequence_matrices.hpp"); - write_macro(os, false); - write_mat(os, - "precalc_xorwow_skipahead_sequence_matrices", - static_cast(&skipahead_matrices_sequence[0][0]), - false); - os.close(); - os.clear(); - - os.open("../src/kernels/precalc_xorwow_skipahead_sequence_matrices_kernel.h"); - write_macro(os, true); - write_mat(os, - "precalc_xorwow_skipahead_sequence_matrices", - static_cast(&skipahead_matrices_sequence_dev[0][0]), - true); - os.close(); -} - -#endif // GUARD_MIOPEN_XORWOW_SKIPAHEAD_GENERATOR_HPP diff --git a/fin b/fin index 8c40a3c3b4..344cf42f6c 160000 --- a/fin +++ b/fin @@ -1 +1 @@ -Subproject commit 8c40a3c3b41a7d2fb31a8e747155fde4223919b9 +Subproject commit 344cf42f6c18f309f3d1dd08af1cd7b73dd38e46 diff --git a/include/miopen/config.h.in b/include/miopen/config.h.in index fb96368ce2..a89e9232c8 100644 --- a/include/miopen/config.h.in +++ b/include/miopen/config.h.in @@ -49,9 +49,12 @@ #cmakedefine01 MIOPEN_ENABLE_SQLITE_BACKOFF #cmakedefine01 MIOPEN_USE_MLIR #cmakedefine01 MIOPEN_USE_COMPOSABLEKERNEL +#cmakedefine01 MIOPEN_BUILD_DRIVER #cmakedefine01 MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK #cmakedefine01 MIOPEN_ENABLE_AI_KERNEL_TUNING #cmakedefine01 MIOPEN_HIP_COMPILER_HAS_OPTION_OFFLOAD_UNIFORM_BLOCK +#cmakedefine01 MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM +#cmakedefine01 MIOPEN_ENABLE_FIN_INTERFACE // "_PACKAGE_" to avoid name contentions: the macros like // HIP_VERSION_MAJOR are defined in hip_version.h. @@ -203,6 +206,6 @@ const char* getOffloadBundlerBinPath(); #include #endif -#define MIOPEN_GOLDEN_DB_VERSION 20 +#define MIOPEN_GOLDEN_DB_VERSION 21 #endif // GUARD_CONFIG_H_IN diff --git a/include/miopen/miopen.h b/include/miopen/miopen.h index 0f2c2a5cb0..22a34fed01 100644 --- a/include/miopen/miopen.h +++ b/include/miopen/miopen.h @@ -64,11 +64,14 @@ * @defgroup LossFunction * @defgroup TensorReduce * @defgroup find2 - * @defgroup sum * @defgroup ReduceExtreme * @defgroup groupnorm * @defgroup cat * @defgroup SGD + * @defgroup getitem + * @defgroup ReduceCalculation + * @defgroup RotaryPositionalEmbeddings + * @defgroup ReLU * */ @@ -396,7 +399,7 @@ typedef enum miopenTensorCHWNc4 = 5, /*!< CHWNc4 memory layout (Partially supported) */ miopenTensorCHWNc8 = 6, /*!< CHWNc8 memory layout (Partially supported) */ miopenTensorNCDHW = 7, /*!< NCDHW memory layout (Fully supported) */ - miopenTensorNDHWC = 8, /*!< NCDHW memory layout (Fully supported) */ + miopenTensorNDHWC = 8, /*!< NDHWC memory layout (Fully supported) */ } miopenTensorLayout_t; /*! @ingroup pooling @@ -2734,6 +2737,67 @@ miopenBatchNormalizationForwardTraining(miopenHandle_t handle, double epsilon, void* resultSaveMean, void* resultSaveInvVariance); +/*! @brief Execute forward training layer for batch normalization + * + * Batch normalization pass for forward training pass. + * Takes in batch normalization mode bn_mode and input tensor x, output tensor y, bnBias and bnScale + * with their descriptor. + * + * If either resultSaveMean, or resultSaveInvVariance are null pointers then the values for the mean + * and inverse variance will not be used. + * + * Likewise, if either resultRunningMean, or resultRunningVariance are null pointers then the values + * for the running mean and variance will not be saved. + * Running averages and variances are scaled using an exponential averaging factor: \f[ + * \mu_{old} = \mu_{new}*factor + \mu_{old}*(1-factor) + * \f] + * where \f[ + * factor=1/(1+iteration) + * \f] + * + * @param handle MIOpen handle (input) + * @param bn_mode Batch normalization mode (input) + * @param alpha Floating point scaling factor, allocated on the host (input) + * @param beta Floating point shift factor, allocated on the host (input) + * @param xDesc Tensor descriptor for data input tensor x (input) + * @param x Data tensor x (input) + * @param yDesc Tensor descriptor for output data tensor y (input) + * @param y Data tensor y (output) + * @param ScaleDesc Tensor descriptor for BN scaling + * @param biasVarDesc Tensor descriptor for BN bias + * @param savedMeanDesc Tensor descriptor for BN saved Mean + * @param savedVarDesc Tensor descriptor for BN saved Variance + * @param bnScale Batch norm scaling, gamma, tensor (input) + * @param bnBias Batch norm bias, beta, tensor (input) + * @param expAvgFactor Exponential averaging factor (input) + * @param resultRunningMean Running average saved for inference (output) + * @param resultRunningVariance Running variance saved for inference (output) + * @param epsilon Value to stablize inverse variance calculation (input) + * @param resultSaveMean Saved mini-batch mean for backwards pass (output) + * @param resultSaveInvVariance Saved mini-batch inverse variance for backwards pass (output) + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t +miopenBatchNormalizationForwardTraining_V2(miopenHandle_t handle, + miopenBatchNormMode_t bn_mode, + void* alpha, + void* beta, + const miopenTensorDescriptor_t xDesc, + const void* x, + const miopenTensorDescriptor_t yDesc, + void* y, + const miopenTensorDescriptor_t scaleDesc, + const miopenTensorDescriptor_t biasVarDesc, + const miopenTensorDescriptor_t savedMeanDesc, + const miopenTensorDescriptor_t savedVarDesc, + void* bnScale, + void* bnBias, + double expAvgFactor, + void* resultRunningMean, + void* resultRunningVariance, + double epsilon, + void* resultSaveMean, + void* resultSaveInvVariance); /*! @brief Execute forward inference layer for batch normalization * @@ -2780,6 +2844,56 @@ miopenBatchNormalizationForwardInference(miopenHandle_t handle, void* estimatedVariance, double epsilon); +/*! @brief Execute forward inference layer for batch normalization + * + * Batch normalization pass for forward inference pass. + * Takes in batch normalization mode bn_mode and input tensor x, output tensor y, bnBias and bnScale + * with their descriptor. + * + * If either estimatedMean, or estimatedVariance are null pointers then the values for the mean and + * variance will be calculated from input data and this calculated mean and variance will be used + * to update input values. + * If variance is zero and epsilon is also zero, this function outputs NAN values. Input espilon + * value should always be non zero positive value. + * + * @param handle MIOpen handle (input) + * @param bn_mode Batch normalization mode (input) + * @param alpha Floating point scaling factor, allocated on the host (input) + * @param beta Floating point shift factor, allocated on the host (input) + * @param xDesc Tensor descriptor for data input tensor x (input) + * @param x Data tensor x (input) + * @param yDesc Tensor descriptor for output data tensor y (input) + * @param y Data tensor y (output) + * @param ScaleDesc Tensor descriptor for BN scaling + * @param biasVarDesc Tensor descriptor for BN bias + * @param estMeanDesc Tensor descriptor for BN estimated Mean + * @param estVarianceDesc Tensor descriptor for BN estimated Variance + * @param bnScale Batch norm scaling, gamma, tensor (input) + * @param bnBias Batch norm bias, beta, tensor (input) + * @param estimatedMean Running average saved during forward training (input) + * @param estimatedVariance Running variance saved during forward training (input) + * @param epsilon Value to stabilize inverse variance calculation (input) + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t +miopenBatchNormalizationForwardInference_V2(miopenHandle_t handle, + miopenBatchNormMode_t bn_mode, + void* alpha, + void* beta, + const miopenTensorDescriptor_t xDesc, + const void* x, + const miopenTensorDescriptor_t yDesc, + void* y, + const miopenTensorDescriptor_t scaleDesc, + const miopenTensorDescriptor_t biasDesc, + const miopenTensorDescriptor_t estMeanDesc, + const miopenTensorDescriptor_t estVarianceDesc, + void* bnScale, + void* bnBias, + void* estimatedMean, + void* estimatedVariance, + double epsilon); + /*! @brief Execute backwards propagation layer for batch normalization * * Batch normalization pass for backwards propagation training pass. @@ -2835,6 +2949,68 @@ miopenBatchNormalizationBackward(miopenHandle_t handle, const void* savedMean, const void* savedInvVariance); +/*! @brief Execute backwards propagation layer for batch normalization + * + * Batch normalization pass for backwards propagation training pass. + * The method for backwards propagation batch normalization. + * + * Takes in batch normalization mode bn_mode and input tensor data x, input activation tensor dy, + * output tensor dx, the learned tensors resultBNBiasDiff and resultBNScaleDiff with their + * descriptor. + * + * If BOTH savedMean, and savedVariance are not null pointers then the method will use the saved + * mean and variance calculated by the forward training phase. + * + * @param handle MIOpen handle (input) + * @param bn_mode Batch normalization mode (input) + * @param alphaDataDiff Floating point scaling factor, allocated on the host (input) + * @param betaDataDiff Floating point shift factor, allocated on the host (input) + * @param alphaParamDiff Floating point scaling factor, allocated on the host (input) + * @param betaParamDiff Floating point shift factor, allocated on the host (input) + * @param xDesc Tensor descriptor for data input tensor x (input) + * @param x Data tensor x (input) + * @param dyDesc Tensor descriptor for output data tensor y (input) + * @param dy Data tensor y (input) + * @param dxDesc Tensor descriptor for output data tensor dx (input) + * @param dx Data delta tensor dx (output) + * @param scaleDesc Tensor descriptor for scaling descriptor (input) + * @param biasDesc Tensor descriptor for bias/shift descriptor (input) + * @param savedMeanDesc Tensor descriptor for saved Mean descriptor (input) + * @param savedVarDesc Tensor descriptor for saved Variance descriptor (input) + * , shifting, saved variance and + * mean (input) + * @param bnScale Batch norm scaling, gamma, tensor (input) + * @param resultBnScaleDiff Tensor for dscale (output) + * @param resultBnBiasDiff Tensor for dbias (output) + * @param epsilon Value to stabilize inverse variance calculation (input) + * @param savedMean Saved mini-batch mean for backwards pass (input) + * @param savedInvVariance Saved mini-bathc inverse variance for backwards pass (input) + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t +miopenBatchNormalizationBackward_V2(miopenHandle_t handle, + miopenBatchNormMode_t bn_mode, + const void* alphaDataDiff, + const void* betaDataDiff, + const void* alphaParamDiff, + const void* betaParamDiff, + const miopenTensorDescriptor_t xDesc, + const void* x, + const miopenTensorDescriptor_t dyDesc, + const void* dy, + const miopenTensorDescriptor_t dxDesc, + void* dx, + const miopenTensorDescriptor_t scaleDesc, + const miopenTensorDescriptor_t biasDesc, + const miopenTensorDescriptor_t savedMeanDesc, + const miopenTensorDescriptor_t savedVarDesc, + const void* bnScale, + void* resultBnScaleDiff, + void* resultBnBiasDiff, + double epsilon, + const void* savedMean, + const void* savedInvVariance); + /** @} */ // CLOSEOUT BATCHNORM DOXYGEN GROUP @@ -2948,6 +3124,54 @@ miopenDestroyActivationDescriptor(miopenActivationDescriptor_t activDesc); /** @} */ // CLOSEOUT ACTIVATION DOXYGEN GROUP +#ifdef MIOPEN_BETA_API +/** @addtogroup activation + * + * @{ + */ + +/*! @brief Execute a GLU forward layer + * + * @param handle MIOpen handle (input) + * @param inputDesc Tensor descriptor for input tensor (input) + * @param input Input tensor (input) + * @param outputDesc Tensor descriptor for output tensor (input) + * @param output Output tensor (output) + * @param dim Dimension to split the input (input) + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t miopenGLUForward(miopenHandle_t handle, + const miopenTensorDescriptor_t inputDesc, + const void* input, + const miopenTensorDescriptor_t outputDesc, + void* output, + const uint32_t dim); + +/*! @brief Execute a GLU backward layer + * + * @param handle MIOpen handle (input) + * @param inputDesc Tensor descriptor for input tensor (input) + * @param input Input tensor (input) + * @param outputGradDesc Tensor descriptor for delta output tensor (input) + * @param outputGrad Delta output tensor (input) + * @param inputGradDesc Tensor descriptor for delta input tensor (input) + * @param inputGrad Delta input tensor (output) + * @param dim Dimension to split the input (input) + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t miopenGLUBackward(miopenHandle_t handle, + const miopenTensorDescriptor_t inputDesc, + const void* input, + const miopenTensorDescriptor_t outputGradDesc, + const void* outputGrad, + const miopenTensorDescriptor_t inputGradDesc, + void* inputGrad, + const uint32_t dim); + +/** @} */ +// CLOSEOUT ACTIVATION DOXYGEN GROUP +#endif // MIOPEN_BETA_API + // Softmax APIs /** @addtogroup softmax * @@ -5404,40 +5628,42 @@ typedef enum miopenTensorMhaAmaxDK = 33, miopenTensorMhaAmaxDV = 34, miopenTensorMhaAmaxDS = 35, + miopenTensorMhaBias = 36, #ifdef MIOPEN_BETA_API - miopenTensorActivationX = 36, - miopenTensorActivationY = 37, - miopenTensorActivationDX = 38, - miopenTensorActivationDY = 39, - miopenTensorBiasX = 40, - miopenTensorBiasY = 41, - miopenTensorBias = 42, - miopenTensorSoftmaxX = 43, - miopenTensorSoftmaxY = 44, - miopenTensorSoftmaxDX = 45, - miopenTensorSoftmaxDY = 46, - miopenTensorBatchnormX = 47, - miopenTensorBatchnormY = 48, - miopenTensorBatchnormRunningMean = 49, - miopenTensorBatchnormRunningVariance = 50, - miopenTensorBatchnormSavedMean = 51, - miopenTensorBatchnormSavedVariance = 52, - miopenTensorBatchnormScale = 53, - miopenTensorBatchnormScaleDiff = 54, - miopenTensorBatchnormEstimatedMean = 55, - miopenTensorBatchnormEstimatedVariance = 56, - miopenTensorBatchnormBias = 57, - miopenTensorBatchnormBiasDiff = 58, - miopenTensorBatchnormDX = 59, - miopenTensorBatchnormDY = 60, + miopenTensorActivationX = 37, + miopenTensorActivationY = 38, + miopenTensorActivationDX = 39, + miopenTensorActivationDY = 40, + miopenTensorBiasX = 41, + miopenTensorBiasY = 42, + miopenTensorBias = 43, + miopenTensorSoftmaxX = 44, + miopenTensorSoftmaxY = 45, + miopenTensorSoftmaxDX = 46, + miopenTensorSoftmaxDY = 47, + miopenTensorBatchnormX = 48, + miopenTensorBatchnormY = 49, + miopenTensorBatchnormRunningMean = 50, + miopenTensorBatchnormRunningVariance = 51, + miopenTensorBatchnormSavedMean = 52, + miopenTensorBatchnormSavedVariance = 53, + miopenTensorBatchnormScale = 54, + miopenTensorBatchnormScaleDiff = 55, + miopenTensorBatchnormEstimatedMean = 56, + miopenTensorBatchnormEstimatedVariance = 57, + miopenTensorBatchnormBias = 58, + miopenTensorBatchnormBiasDiff = 59, + miopenTensorBatchnormDX = 60, + miopenTensorBatchnormDY = 61, #endif miopenTensorArgumentIsScalar = 1U << 31, + miopenTensorMhaMask = miopenTensorArgumentIsScalar | 1, #ifdef MIOPEN_BETA_API - miopenScalarBatchnormExpAvgFactor = miopenTensorArgumentIsScalar | 1, - miopenScalarBatchnormEpsilon = miopenTensorArgumentIsScalar | 2, + miopenScalarBatchnormExpAvgFactor = miopenTensorArgumentIsScalar | 2, + miopenScalarBatchnormEpsilon = miopenTensorArgumentIsScalar | 3, #endif } miopenTensorArgumentId_t; @@ -5469,6 +5695,15 @@ MIOPEN_EXPORT miopenStatus_t miopenCreateConvProblem(miopenProblem_t* problem, * @return miopenStatus_t */ +/*! @enum miopenMhaMask_t + * Different masks for Mha. + */ +typedef enum +{ + miopenMhaMaskNone = 0, + miopenMhaMaskCausal = 1, +} miopenMhaMask_t; + MIOPEN_EXPORT miopenStatus_t miopenCreateMhaProblem(miopenProblem_t* problem, miopenMhaDescriptor_t operatorDesc, miopenProblemDirection_t direction); @@ -5848,63 +6083,79 @@ MIOPEN_EXPORT miopenStatus_t miopenCreateSoftmaxProblem(miopenProblem_t* problem #ifdef MIOPEN_BETA_API -/*! @ingroup sum - * @enum miopenSumNanPropagation_t - * Nan numbers propagation modes for sum +/*! @ingroup ReduceCalculation + * @enum miopenReduceCalculationNanPropagation_t + * Nan numbers propagation modes for reduce calculation */ typedef enum { - MIOPEN_SUM_NOT_PROPAGATE_NAN = 0, /*!< does not propagate Nan number */ - MIOPEN_SUM_PROPAGATE_NAN = 1, /*!< propagate the Nan number by the Reduction operation */ -} miopenSumNanPropagation_t; + MIOPEN_REDUCE_CALCULATION_NOT_PROPAGATE_NAN = 0, /*!< does not propagate Nan number */ + MIOPEN_REDUCE_CALCULATION_PROPAGATE_NAN = + 1, /*!< propagate the Nan number by the Reduction operation */ +} miopenReduceCalculationNanPropagation_t; -// Sum APIs -/** @addtogroup sum +// ReduceCalculation APIs +/** @addtogroup reducecalculation * * @{ */ -/*! @brief Helper function to query the minimum workspace size required by the ReduceTensor call - * - * @param handle MIOpen Handle (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param dim Dimension to sum. (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param sizeInBytes Pointer to data to return the minimum workspace size - * @return miopenStatus_t +/*! @enum miopenReduceCalculationOp_t + * Reduction Calculation operation types */ -MIOPEN_EXPORT miopenStatus_t miopenGetSumWorkspaceSize(miopenHandle_t handle, - const miopenTensorDescriptor_t xDesc, - const int32_t dim, - const miopenTensorDescriptor_t yDesc, - size_t* sizeInBytes); +typedef enum +{ + MIOPEN_REDUCE_CALCULATION_PROD = + 1, /*!< the operation is multiplying the values of the reduced elements */ + MIOPEN_REDUCE_CALCULATION_SUM = + 2, /*!< the operation is adding the values of the reduced elements */ +} miopenReduceCalculationOp_t; -/*! @brief Execute a sum forward layer +/*! @brief Helper function to query the minimum workspace size required by the ReduceTensor call * - * @param handle MIOpen handle (input) - * @param nanPropagation Nan number propagation mode (input) - * @param workspace Address of the allocated workspace data (input) - * @param workspaceSizeInBytes Size in bytes of the allocated workspace data (input) - * @param xDesc Tensor descriptor for data input tensor x (input) - * @param x Data tensor x (input) - * @param dim Dimension to sum. (input) - * @param yDesc Tensor descriptor for output data tensor y (input) - * @param y Data tensor y (output) - * @return miopenStatus_t + * @param [in] handle MIOpen Handle + * @param [in] xDesc Tensor descriptor for data input tensor x + * @param [in] dim Dimension to calculation. + * @param [in] yDesc Tensor descriptor for output data tensor y + * @param [out] sizeInBytes Pointer to data to return the minimum workspace size + * @return miopenStatus_t */ -MIOPEN_EXPORT miopenStatus_t miopenSumForward(miopenHandle_t handle, - miopenSumNanPropagation_t nanPropagation, - void* workspace, - size_t workspaceSizeInBytes, - const miopenTensorDescriptor_t xDesc, - const void* x, - const int32_t dim, - const miopenTensorDescriptor_t yDesc, - void* y); +MIOPEN_EXPORT miopenStatus_t +miopenGetReduceCalculationWorkspaceSize(miopenHandle_t handle, + const miopenTensorDescriptor_t xDesc, + const int32_t dim, + const miopenReduceCalculationOp_t reduceCalculationOp, + const miopenTensorDescriptor_t reduceDesc, + size_t* sizeInBytes); + +/*! @brief Execute a reducecalculation forward layer + * + * @param [in] handle MIOpen handle + * @param [in] nanPropagation Nan number propagation mode + * @param [in] workspace Address of the allocated workspace data + * @param [in] workspaceSizeInBytes Size in bytes of the allocated workspace data + * @param [in] xDesc Tensor descriptor for data input tensor x + * @param [in] x Data tensor x + * @param [in] dim Dimension to calculation. + * @param [in] yDesc Tensor descriptor for output data tensor y + * @param [out] y Data tensor y + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t +miopenReduceCalculationForward(miopenHandle_t handle, + miopenReduceCalculationNanPropagation_t nanPropagation, + void* workspace, + size_t workspaceSizeInBytes, + const miopenTensorDescriptor_t xDesc, + const void* x, + const int32_t dim, + const miopenReduceCalculationOp_t reduceCalculationOp, + const miopenTensorDescriptor_t reduceDesc, + void* y); /** @} */ -// CLOSEOUT SUM DOXYGEN GROUP -#endif +// CLOSEOUT REDUCE CALCULATION DOXYGEN GROUP +#endif // MIOPEN_BETA_API #ifdef MIOPEN_BETA_API @@ -5957,7 +6208,7 @@ miopenReduceExtremeForward(miopenHandle_t handle, /** @} */ // CLOSEOUT REDUCEEXTREME DOXYGEN GROUP -#endif +#endif // MIOPEN_BETA_API #ifdef MIOPEN_BETA_API // GroupNorm APIs @@ -6004,7 +6255,7 @@ MIOPEN_EXPORT miopenStatus_t miopenGroupNormForward(miopenHandle_t handle, /** @} */ // CLOSEOUT groupnorm DOXYGEN GROUP -#endif +#endif // MIOPEN_BETA_API #ifdef MIOPEN_BETA_API // LayerNorm APIs @@ -6055,7 +6306,7 @@ MIOPEN_EXPORT miopenStatus_t miopenAddLayerNormForward(miopenHandle_t handle, /** @} */ // CLOSEOUT LAYERNORM DOXYGEN GROUP -#endif +#endif // MIOPEN_BETA_API #ifdef MIOPEN_BETA_API // LayerNorm APIs @@ -6153,7 +6404,7 @@ MIOPEN_EXPORT miopenStatus_t miopenT5LayerNormBackward(miopenHandle_t handle, void* dw); /** @} */ // CLOSEOUT LAYERNORM DOXYGEN GROUP -#endif +#endif // MIOPEN_BETA_API #ifdef MIOPEN_BETA_API // Graph API @@ -7538,6 +7789,393 @@ miopenTransformersAdamWWithOutput(miopenHandle_t handle, // CLOSEOUT SGD DOXYGEN GROUP #endif // MIOPEN_BETA_API +#ifdef MIOPEN_BETA_API +// GetItem APIs +/** @addtogroup getitem + * + * @{ + */ +/*! @brief Helper function to query the minimum workspace size required by the getitem call + * + * @param [in] handle MIOpen Handle + * @param [in] indexCount Number of input tensor indexs + * @param [in] indexDescs Tensor descriptor of input tensor indexs + * @param [out] sizeInBytes Pointer to data to return the minimum workspace size + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t +miopenGetGetitemWorkspaceSize(miopenHandle_t handle, + uint32_t indexCount, + const miopenTensorDescriptor_t* indexDescs, + size_t* sizeInBytes); + +/*! @brief Execute a getitem backward layer + * + * Backward of getitem for tensor indexing, slicing, masking. + * + * @param [in] handle MIOpen handle + * @param [in] workspace Address of the allocated workspace data + * @param [in] workspaceSizeInBytes Size in bytes of the allocated workspace data + * @param [in] dyDesc Tensor descriptor of input tensor dy + * @param [in] dy Source data tensor dy + * @param [in] indexCount Number of input tensor indexs + * @param [in] indexDescs Tensor descriptor of input tensor indexs(All indexs same + * size) + * @param [in] indexs Source data tensor indexs + * @param [in] dxDesc Tensor descriptor of output tensor dx + * @param [out] dx Data tensor dx(It must be initialized to 0) + * @param [in] errorDesc Tensor descriptor of output tensor error + * @param [out] error Data tensor error(It must be initialized to 0) + * @param [in] dimCount Number of dimensions + * @param [in] dims Dimensions + * @param [in] sliceCount Number of slices + * @param [in] slices Slices + * @param [in] offset Offset of output tensor dx + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t miopenGetitemBackward(miopenHandle_t handle, + void* workspace, + size_t workspaceSizeInBytes, + const miopenTensorDescriptor_t dyDesc, + const void* dy, + uint32_t indexCount, + const miopenTensorDescriptor_t* indexDescs, + const void* const* indexs, + const miopenTensorDescriptor_t dxDesc, + void* dx, + const miopenTensorDescriptor_t errorDesc, + void* error, + uint32_t dimCount, + const int32_t* dims, + uint32_t sliceCount, + const int32_t* slices, + uint32_t offset); + +/** @} */ +// CLOSEOUT GETITEM DOXYGEN GROUP +#endif // MIOPEN_BETA_API + +#ifdef MIOPEN_BETA_API +// RotaryPositionalEmbeddings APIs +/** @addtogroup RotaryPositionalEmbeddings + * + * @{ + */ +/*! @brief Execute a rope forward layer + * + * @param [in] handle MIOpen handle + * @param [in] xDesc Tensor descriptor for data input tensor x + * @param [in] x Data tensor x + * @param [in] cosDesc Tensor descriptor for data input tensor cos + * @param [in] cos Data tensor cos + * @param [in] sinDesc Tensor descriptor for data input tensor sin + * @param [in] sin Data tensor sin + * @param [in] yDesc Tensor descriptor for output data tensor y + * @param [out] y Data tensor y + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t miopenRoPEForward(miopenHandle_t handle, + const miopenTensorDescriptor_t xDesc, + const void* x, + const miopenTensorDescriptor_t cosDesc, + const void* cos, + const miopenTensorDescriptor_t sinDesc, + const void* sin, + const miopenTensorDescriptor_t yDesc, + void* y); + +/*! @brief Execute a rope backward layer + * + * @param [in] handle MIOpen handle + * @param [in] dyDesc Tensor descriptor for data input tensor dy + * @param [in] dy Data tensor dy + * @param [in] cosDesc Tensor descriptor for output data tensor cos + * @param [in] cos Data tensor cos + * @param [in] sinDesc Tensor descriptor for data input tensor sin + * @param [in] sin Data tensor sin + * @param [in] dxDesc Tensor descriptor for output data tensor dx + * @param [out] dx Data tensor dx + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t miopenRoPEBackward(miopenHandle_t handle, + const miopenTensorDescriptor_t dyDesc, + const void* dy, + const miopenTensorDescriptor_t cosDesc, + const void* cos, + const miopenTensorDescriptor_t sinDesc, + const void* sin, + const miopenTensorDescriptor_t dxDesc, + void* dx); +/** @} */ +// CLOSEOUT ROPE DOXYGEN GROUP +// kthvalue APIs +/** @addtogroup kthvalue + * + * @{ + */ + +/*! @brief Execute a Kthvalue forward layer + * + * @param handle MIOpen handle (input) + * @param inputDesc Tensor descriptor for input tensor (input) + * @param input Data tensor input (input) + * @param outputDesc Tensor descriptor for output tensor (input) + * @param output Data tensor output (output) + * @param indices Data tensor indices (output) + * @param indicesDesc Tensor descriptor for indices tensor (input) + * @param k The k-th smallest element(input) + * @param dim The dimension to find the kth value along (Default = -1)(input) + * @param keepDim Whether the output tensor has dim retained or not (Default = + * false)(input) + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t miopenKthvalueForward(miopenHandle_t handle, + miopenTensorDescriptor_t inputDesc, + const void* input, + miopenTensorDescriptor_t outputDesc, + void* output, + miopenTensorDescriptor_t indicesDesc, + size_t* indices, + size_t k, + int32_t dim = -1, + bool keepDim = false); + +/** @} */ +// CLOSEOUT kthvalue DOXYGEN GROUP +#endif // MIOPEN_BETA_API + +#ifdef MIOPEN_BETA_API +/** @addtogroup ReLU + * + * @{ + */ + +/*! @brief Helper function to query the minimum workspace size required by the PReLU backward call + * + * @param handle MIOpen Handle (input) + * @param inputDesc Tensor descriptor for input tensor (input) + * @param weightDesc Tensor descriptor for weight tensor (input) + * @param sizeInBytes Pointer to data to return the minimum workspace size + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t +miopenGetPReLUBackwardWorkspaceSize(miopenHandle_t handle, + miopenTensorDescriptor_t inputDesc, + miopenTensorDescriptor_t weightDesc, + size_t* sizeInBytes); + +/*! @brief Execute a PReLU backward layer + * + * @param handle MIOpen handle (input) + * @param workspace Address of the allocated workspace data (input) + * @param workspaceSizeInBytes Size in bytes of the allocated workspace data (input) + * @param inputDesc Tensor descriptor for input tensor (input) + * @param input Data tensor input (input) + * @param weightDesc Tensor descriptor for weight tensor (input) + * @param weight Data tensor weight (input) + * @param doutputDesc Tensor descriptor for output gradient (input) + * @param doutput Gradient of output (input) + * @param dinputDesc Tensor descriptor for input gradient (input) + * @param dinput Gradient of input (output) + * @param dweightDesc Tensor descriptor for weight gradient (input) + * @param dweight Gradient of weight (output) + */ +MIOPEN_EXPORT miopenStatus_t miopenPReLUBackward(miopenHandle_t handle, + void* workspace, + size_t workspaceSizeInBytes, + miopenTensorDescriptor_t inputDesc, + const void* input, + miopenTensorDescriptor_t weightDesc, + const void* weight, + miopenTensorDescriptor_t doutputDesc, + const void* doutput, + miopenTensorDescriptor_t dinputDesc, + void* dinput, + miopenTensorDescriptor_t dweightDesc, + void* dweight); + +/** @} */ +// CLOSEOUT RELU DOXYGEN GROUP +#endif // MIOPEN_BETA_API + +#ifdef MIOPEN_BETA_API + +/*! @ingroup LossFunction + * @enum miopenLossReductionMode_t + * Reduction mode for loss function + */ +typedef enum +{ + MIOPEN_LOSS_REDUCTION_NONE = 0, /*!< output tensor elements are not reduced */ + MIOPEN_LOSS_REDUCTION_SUM = 1, /*!< output tensor elements are summed up */ + MIOPEN_LOSS_REDUCTION_MEAN = 2, /*!< output tensor elements are summed up and divided with total + number of elements to get mean value */ +} miopenLossReductionMode_t; + +// SoftMarginLoss APIs +/** @addtogroup LossFunction + * + * @{ + */ + +/*! @brief Helper function to query the minimum workspace size required by the +SoftMarginLossForward call + * + * @param [in] handle MIOpen Handle + * @param [in] inputDesc Tensor descriptor for input tensor + * @param [in] targetDesc Tensor descriptor for target tensor + * @param [in] outputDesc Tensor descriptor for output tensor +* @param [in] reduction Reduction mode (sum, mean). For none reduction we don't need to +use this function + * @param [out] sizeInBytes Pointer to data to return the minimum workspace size + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t +miopenGetSoftMarginLossForwardWorkspaceSize(miopenHandle_t handle, + miopenTensorDescriptor_t inputDesc, + miopenTensorDescriptor_t targetDesc, + miopenTensorDescriptor_t outputDesc, + miopenLossReductionMode_t reduction, + size_t* sizeInBytes); + +/*! @brief Execute a SoftMarginLoss forward layer + * + * @param [in] handle MIOpen handle + * @param [in] inputDesc Tensor descriptor for input tensor + * @param [in] input Data tensor input + * @param [in] targetDesc Tensor descriptor for target tensor + * @param [in] target Data tensor target + * @param [in] outputDesc Tensor descriptor for output tensor + * @param [out] output Data tensor output + * @param [in] reduction Reduction mode. If reduction mode is mean or sum, you must + * provide param workspace and workspaceSizeInBytes. Call + * miopenGetSoftMarginLossForwardWorkspaceSize to get workspaceSizeInBytes + * @param [in] workspace Address of the allocated workspace data (Default = null) + * @param [in] workspaceSizeInBytes Size in bytes of the allocated workspace data (Default = 0) + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t miopenSoftMarginLossForward(miopenHandle_t handle, + miopenTensorDescriptor_t inputDesc, + const void* input, + miopenTensorDescriptor_t targetDesc, + const void* target, + miopenTensorDescriptor_t outputDesc, + void* output, + miopenLossReductionMode_t reduction, + void* workspace = nullptr, + size_t workspaceSizeInBytes = 0); + +/*! @brief Execute a SoftMarginLoss backward layer + * + * @param [in] handle MIOpen handle + * @param [in] inputDesc Tensor descriptor for input tensor + * @param [in] input Data tensor input + * @param [in] targetDesc Tensor descriptor for target tensor + * @param [in] target Data tensor target + * @param [in] doutputDesc Tensor descriptor for output gradient + * @param [in] doutput Output gradient + * @param [in] dinputDesc Tensor descriptor for input gradient + * @param [out] dinput Input gradient + * @param [in] reduction Reduction mode (none, sum, mean) + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t miopenSoftMarginLossBackward(miopenHandle_t handle, + miopenTensorDescriptor_t inputDesc, + const void* input, + miopenTensorDescriptor_t targetDesc, + const void* target, + miopenTensorDescriptor_t doutputDesc, + const void* doutput, + miopenTensorDescriptor_t dinputDesc, + void* dinput, + miopenLossReductionMode_t reduction); + +/** @} */ +// CLOSEOUT LossFunction DOXYGEN GROUP +#endif + +#ifdef MIOPEN_BETA_API +// MultiMarginLoss APIs +/** @addtogroup LossFunction + * + * @{ + */ + +/*! @brief Helper function to query the minimum workspace size required by the +MultiMarginLossForward call + * + * @param [in] handle MIOpen Handle + * @param [in] inputDesc Tensor descriptor for input tensor (N, C) where N is the batch +size and C is the number of classes + * @param [in] targetDesc Tensor descriptor for target tensor, must have shape (N). Each +value is between 0 and C - 1 + * @param [in] weightDesc Tensor descriptor for weight tensor. It is a manual rescaling +weight given to each class. It has to be a Tensor of size C + * @param [in] outputDesc Tensor descriptor for output tensor. If reduction is 'none, +then it must have shape (N). Otherwise, it is a scalar + * @param [in] p Has a default value of 1. The only supported values are 1 and 2 + * @param [in] margin Has a default value of 1 + * @param [in] reduction Reduction mode (sum, mean) + * @param [out] sizeInBytes Pointer to data to return the minimum workspace size + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t +miopenGetMultiMarginLossForwardWorkspaceSize(miopenHandle_t handle, + miopenTensorDescriptor_t inputDesc, + miopenTensorDescriptor_t targetDesc, + miopenTensorDescriptor_t weightDesc, + miopenTensorDescriptor_t outputDesc, + long p, + float margin, + miopenLossReductionMode_t reduction, + size_t* sizeInBytes); + +/*! @brief Execute a MultiMarginLoss forward layer + * + * @param [in] handle MIOpen handle + * @param [in] inputDesc Tensor descriptor for input tensor (N, C) where N is the +batch size and C is the number of classes. + * @param [in] input Data tensor input + * @param [in] targetDesc Tensor descriptor for target tensor, must have shape (N). +Each value is between 0 and C - 1 + * @param [in] target Data tensor target + * @param [in] weightDesc Tensor descriptor for weight tensor. It is a manual +rescaling weight given to each class. It has to be a Tensor of size C + * @param [in] weight Data tensor weight + * @param [in] outputDesc Tensor descriptor for output tensor. If reduction is 'none, +then it must have shape (N). Otherwise, it is a scalar. + * @param [out] output Data tensor output + * @param [in] p Has a default value of 1. The only supported values are 1 +and 2 + * @param [in] margin Has a default value of 1 + * @param [in] reduction Reduction mode. If reduction mode is mean or sum, you must + * provide param workspace and workspaceSizeInBytes. Call + * miopenGetMultiMarginLossForwardWorkspaceSize to get workspaceSizeInBytes + * @param [in] workspace Address of the allocated workspace data. Set = nullptr if +reduction = 'none' + * @param [in] workspaceSizeInBytes Size in bytes of the allocated workspace data. Set = 0 if +reduction = 'none + * @return miopenStatus_t + */ +MIOPEN_EXPORT miopenStatus_t miopenMultiMarginLossForward(miopenHandle_t handle, + miopenTensorDescriptor_t inputDesc, + const void* input, + miopenTensorDescriptor_t targetDesc, + const void* target, + miopenTensorDescriptor_t weightDesc, + const void* weight, + miopenTensorDescriptor_t outputDesc, + void* output, + long p, + float margin, + miopenLossReductionMode_t reduction, + void* workspace, + size_t workspaceSizeInBytes); + +/** @} */ +// CLOSEOUT LossFunction DOXYGEN GROUP +#endif // MIOPEN_BETA_API + #ifdef __cplusplus } #endif diff --git a/install_deps.cmake b/install_deps.cmake index 8e3a71d906..4a0abd9b99 100755 --- a/install_deps.cmake +++ b/install_deps.cmake @@ -113,5 +113,5 @@ cget(init ${TOOLCHAIN_FLAG} -DCMAKE_INSTALL_RPATH=${PREFIX}/lib ${PARSE_UNPARSED cget(ignore pcre) # Install dependencies -cget(install -U ROCm/rocm-recipes@329203d79f9fe77ae5d0d742af0966bc57f4dfc8) +cget(install -U ROCm/rocm-recipes@92c6695449c85887962f45509b376f2eb0d284f7) cget(install -U -f requirements.txt) diff --git a/rbuild.ini b/rbuild.ini index 2ab56e33f2..21c5669442 100755 --- a/rbuild.ini +++ b/rbuild.ini @@ -3,7 +3,7 @@ cxx = ${rocm_path}/llvm/bin/clang++ cc = ${rocm_path}/llvm/bin/clang ignore = pcre deps = - ROCm/rocm-recipes@329203d79f9fe77ae5d0d742af0966bc57f4dfc8 + ROCm/rocm-recipes@92c6695449c85887962f45509b376f2eb0d284f7 -f requirements.txt [develop] @@ -20,5 +20,5 @@ cxx = ${rocm_path}/llvm/bin/clang++ cc = ${rocm_path}/llvm/bin/clang ignore = pcre deps = - ROCm/rocm-recipes@329203d79f9fe77ae5d0d742af0966bc57f4dfc8 + ROCm/rocm-recipes@92c6695449c85887962f45509b376f2eb0d284f7 -f dev-requirements.txt diff --git a/requirements.txt b/requirements.txt index 96b87492f7..73eb8da1cd 100755 --- a/requirements.txt +++ b/requirements.txt @@ -2,10 +2,10 @@ sqlite3@3.43.2 -DCMAKE_POSITION_INDEPENDENT_CODE=On boost@1.83 -DCMAKE_POSITION_INDEPENDENT_CODE=On --build -DCMAKE_CXX_FLAGS=" -std=c++14 -Wno-enum-constexpr-conversion -Wno-deprecated-builtins -Wno-deprecated-declarations " facebook/zstd@v1.4.5 -X subdir -DCMAKE_DIR=build/cmake # ROCm/half@10abd99e7815f0ca5d892f58dd7d15a23b7cf92c --build -ROCm/rocMLIR@rocm-5.5.0 -H sha256:a5f62769d28a73e60bc8d61022820f050e97c977c8f6f6275488db31512e1f42 -DBUILD_FAT_LIBROCKCOMPILER=1 -DCMAKE_IGNORE_PATH="/opt/conda/envs/py_3.8;/opt/conda/envs/py_3.9;/opt/conda/envs/py_3.10" -DCMAKE_IGNORE_PREFIX_PATH=/opt/conda +ROCm/rocMLIR@rocm-5.5.0 -H sha256:a5f62769d28a73e60bc8d61022820f050e97c977c8f6f6275488db31512e1f42 -DBUILD_FAT_LIBROCKCOMPILER=1 -DCMAKE_IGNORE_PATH="/opt/conda/envs/py_3.8;/opt/conda/envs/py_3.9;/opt/conda/envs/py_3.10" -DCMAKE_IGNORE_PREFIX_PATH=/opt/conda -DCMAKE_CXX_FLAGS=" -Wno-deprecated-declarations " nlohmann/json@v3.11.2 -DJSON_MultipleHeaders=ON -DJSON_BuildTests=Off ROCm/FunctionalPlus@v0.2.18-p0 ROCm/eigen@3.4.0 ROCm/frugally-deep@9683d557eb672ee2304f80f6682c51242d748a50 -ROCm/composable_kernel@15baccf2ecad4fb3498b8acb6bbf58fb5359c7a5 -DCMAKE_BUILD_TYPE=Release -DINSTANCES_ONLY=ON +ROCm/composable_kernel@fb948120d2d674607e70d0e7587dcb249c0e74c7 -DCMAKE_BUILD_TYPE=Release google/googletest@v1.14.0 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 14e7c954b1..12bd5a61c9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,6 +89,7 @@ set( MIOpen_Source adam_api.cpp addlayernorm_api.cpp api/find2_0_commons.cpp + base64.cpp batch_norm.cpp batch_norm_api.cpp batchnorm/problem_description.cpp @@ -120,13 +121,18 @@ set( MIOpen_Source env.cpp execution_context.cpp expanduser.cpp + fin/fin_interface.cpp find_controls.cpp find_db.cpp fused_api.cpp fusion.cpp fusion/problem_description.cpp generic_search.cpp + getitem_api.cpp + glu/problem_description.cpp + glu_api.cpp graphapi/convolution.cpp + graphapi/conv_bias_res_add_activ_forward_executor.cpp graphapi/engine.cpp graphapi/enginecfg.cpp graphapi/engineheur.cpp @@ -145,8 +151,11 @@ set( MIOpen_Source groupnorm/problem_description.cpp handle_api.cpp invoker_cache.cpp + getitem/problem_description.cpp kernel_build_params.cpp kernel_warnings.cpp + kthvalue/problem_description.cpp + kthvalue_api.cpp layernorm_api.cpp layernorm/problem_description.cpp load_file.cpp @@ -155,15 +164,20 @@ set( MIOpen_Source lrn_api.cpp mha/mha_descriptor.cpp mha/problem_description.cpp + multimarginloss/problem_description.cpp + multimarginloss_api.cpp op_args.cpp operator.cpp performance_config.cpp pooling/problem_description.cpp pooling_api.cpp + prelu/problem_description.cpp + prelu_api.cpp problem.cpp process.cpp ramdb.cpp readonlyramdb.cpp + reducecalculation_api.cpp reduceextreme_api.cpp reducetensor.cpp reducetensor_api.cpp @@ -171,8 +185,21 @@ set( MIOpen_Source rnn.cpp rnn_api.cpp rnn/rnn_util.cpp + rnn/selector.cpp rnn/Solutions/rnn_transformer.cpp + rnn/Solutions/Base/fw_data_modular.cpp + rnn/Solutions/Base/bw_weights_modular.cpp + rnn/Solutions/Base/bw_data_modular.cpp + rnn/Solutions/fwd_s_stream.cpp + rnn/Solutions/bwd_s_stream.cpp + rnn/Solutions/bwd_multi_stream.cpp + rnn/Solutions/bww_s_steam.cpp + rnn/Solutions/bww_multi_stream.cpp + rope_api.cpp + rope/problem_description.cpp scalar.cpp + softmarginloss/problem_description.cpp + softmarginloss_api.cpp softmax.cpp softmax_api.cpp softmax/problem_description.cpp @@ -241,9 +268,6 @@ set( MIOpen_Source solver/conv/conv_hip_implicit_gemm_3d_grouped_fwd_xdlops.cpp solver/conv/conv_hip_implicit_gemm_3d_grouped_wrw_xdlops.cpp solver/conv/conv_hip_implicit_gemm_3d_grouped_bwd_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_f16f8f16_fwd_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_f16f8f16_bwd_xdlops.cpp - solver/conv/conv_hip_implicit_gemm_f16f8f16_wrw_xdlops.cpp solver/conv/conv_hip_implicit_gemm_nonxdlops_common.cpp solver/conv/conv_hip_implicit_gemm_wrw_v4r4.cpp solver/conv/conv_hip_implicit_gemm_wrw_v4r4_xdlops.cpp @@ -277,29 +301,48 @@ set( MIOpen_Source solver/conv_ck_igemm_fwd_bias_res_add_activ_fused.cpp solver/conv_ocl_dir2Dfwd_fused.cpp solver/conv_winoRxS_fused.cpp + solver/glu/backward_glu.cpp + solver/glu/forward_glu.cpp solver/groupnorm/forward_groupnorm.cpp + solver/getitem/backward_getitem.cpp + solver/kthvalue/forward_kthvalue.cpp solver/layernorm/backward_t5layernorm.cpp solver/layernorm/forward_addlayernorm.cpp solver/layernorm/forward_layernorm.cpp solver/layernorm/forward_layernorm2d_ck.cpp solver/layernorm/forward_layernorm4d_ck.cpp solver/layernorm/forward_t5layernorm.cpp + solver/mha/mha_ck_fa_v2_solver_forward.cpp solver/mha/mha_solver_backward.cpp solver/mha/mha_solver_forward.cpp + solver/multimarginloss/forward_multimarginloss.cpp solver/pooling/forward2d.cpp solver/pooling/forwardNaive.cpp + solver/pooling/forwardNdNhwcNaive.cpp solver/pooling/forwardNd.cpp + solver/pooling/forwardCk2d.cpp + solver/pooling/forwardCkNd.cpp solver/pooling/backward2d.cpp solver/pooling/backwardNd.cpp + solver/pooling/backwardCk2d.cpp + solver/pooling/backwardCkNd.cpp + solver/reduce/forward_argmax.cpp + solver/prelu/backward_prelu_multi_weights.cpp + solver/prelu/backward_prelu_single_weight.cpp + solver/prelu/utils.cpp solver/reduce/forward_argmax.cpp solver/reduce/forward_argmin.cpp solver/reduce/forward_max.cpp solver/reduce/forward_min.cpp + solver/reduce/forward_prod.cpp solver/reduce/forward_sum.cpp + solver/rope/backward_rope.cpp + solver/rope/forward_rope.cpp + solver/softmarginloss/backward_softmarginloss.cpp + solver/softmarginloss/forward_softmarginloss.cpp solver/softmax/attn_softmax.cpp solver/softmax/softmax.cpp subbuffers.cpp - sum_api.cpp t5layernorm_api.cpp target_properties.cpp temp_file.cpp @@ -344,6 +387,7 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN ${COMPOSABLE_KERNEL_DYNAMIC_ASM_INCLUDE} ${GPU_GENERAL_TENSOR_REORDER_KERNEL_HIP_INCLUDE} include/miopen/implicitgemm_params.hpp + kernels/activation_functions.hpp kernels/gpu_reference_kernel/fp8_kern_types.h kernels/Conv_Winograd_v13_3_12_fp16dot_stride1.inc kernels/Conv_Winograd_v13_3_12_fp16dot_stride2_dec.inc @@ -432,12 +476,15 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN kernels/Conv_Winograd_v30_3_1_gfx11_fp32_f3x2_stride1.inc kernels/Conv_Winograd_v30_3_1_gfx11_fp32_f3x2_stride2.inc kernels/Conv_Winograd_v30_3_1_metadata.inc + kernels/MIOpenReduceCalculation.hpp kernels/MIOpenReduceExtreme.hpp kernels/bfloat16_dev.hpp + kernels/block_reduce.hpp kernels/conv_common.inc kernels/conv_sizes.inc kernels/float_types.h kernels/gpr_alloc.inc + kernels/hip_atomic.hpp kernels/hip_f8_impl.hpp kernels/hip_float8.hpp kernels/inst_wrappers.inc @@ -447,9 +494,15 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN kernels/miopen_type_traits.hpp kernels/miopen_utility.hpp kernels/neuron.inc + kernels/pooling_functions.h + include/miopen/pooling/poolingNdNhwcArgs.hpp + kernels/rocm_version.inc + kernels/radix.hpp kernels/rocm_version.inc kernels/stride_array.hpp + kernels/tensor_view.hpp kernels/utilities.inc + kernels/warp_reduce.hpp kernels/winograd/Conv_Winograd_Fury_v2_4_1_gfx11_1536vgprs_fp16_fp16acc_f2x3_c16_stride1.inc kernels/winograd/Conv_Winograd_Fury_v2_4_1_gfx11_1536vgprs_fp16_fp16acc_f2x3_c32_stride1.inc kernels/winograd/Conv_Winograd_Fury_v2_4_1_gfx11_1024vgprs_fp16_fp16acc_f2x3_c16_stride1.inc @@ -486,23 +539,32 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN kernels/MIOpenConvDirUni.cl kernels/MIOpenConvDirBatchNormActiv.cl kernels/MIOpenConvDirGenFwd.cl + kernels/MIOpenGLU.cpp kernels/MIOpenGroupNorm.cpp + kernels/MIOpenGetitem.cpp + kernels/MIOpenKthvalue.cpp kernels/MIOpenLayerNorm.cpp kernels/MIOpenLRNBwd.cl kernels/MIOpenLRNFwd.cl + kernels/MIOpenMultiMarginLoss.cpp kernels/MIOpenNeuron.cl + kernels/MIOpenPReLU.cpp kernels/MIOpenPooling.cl kernels/MIOpenPoolingBwd.cl kernels/MIOpenPoolingBwdND.cl kernels/MIOpenPoolingForwardNaive.cl + kernels/MIOpenPoolingForwardNDNhwcNaive.cpp kernels/MIOpenPoolingND.cl kernels/MIOpenConv1x1S.cl kernels/MIOpenConv1x1J1.cl kernels/MIOpenConv1x1J1_stride.cl + kernels/MIOpenReduceCalculation.cpp kernels/MIOpenReduceExtreme.cpp + kernels/MIOpenReduceSum.cpp + kernels/MIOpenRoPE.cpp + kernels/MIOpenSoftMarginLoss.cpp kernels/MIOpenSoftmax.cl kernels/MIOpenSoftmaxAttn.cpp - kernels/MIOpenSum.cpp kernels/MIOpenUtilKernels3.cl kernels/MIOpenUtilKernels4.cl kernels/MIOpenUtilKernels5.cl @@ -576,8 +638,9 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN kernels/Conv_Winograd_v30_3_1_fp32_f3x2_stride2.s kernels/MIOpenConvBwdBias.cl kernels/MIOpenBatchNormActivInfer.cl + kernels/MIOpenBatchNormActivInferHIP.cpp kernels/MIOpenCTCLoss.cl - kernels/MIOpenDropout.cl + kernels/MIOpenDropoutHIP.cpp kernels/winograd/Conv_Winograd_Fury_v2_4_1_fp16_fp16acc_f2x3_c16_stride1.s kernels/winograd/Conv_Winograd_Fury_v2_4_1_fp16_fp16acc_f2x3_c32_stride1.s kernels/xform_data.s @@ -615,12 +678,16 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN adam.cpp addlayernorm.cpp cat.cpp + exec_utils.cpp groupnorm.cpp + getitem.cpp + glu.cpp kernel_cache.cpp + kthvalue.cpp layernorm.cpp lrn.cpp mlo_dir_conv.cpp - exec_utils.cpp + multimarginloss.cpp ocl/activ_ocl.cpp ocl/batchnormocl.cpp ocl/convolutionocl.cpp @@ -641,8 +708,11 @@ if( MIOPEN_BACKEND MATCHES "OpenCL" OR MIOPEN_BACKEND STREQUAL "HIPOC" OR MIOPEN t5layernorm.cpp ocl/fusionopconvocl.cpp ocl/fusionopbiasbnactivocl.cpp + prelu.cpp + reducecalculation.cpp reduceextreme.cpp - sum.cpp + rope.cpp + softmarginloss.cpp transformers_adam_w.cpp ${PROJECT_BINARY_DIR}/db_path.cpp ) @@ -787,7 +857,7 @@ target_include_directories(MIOpen PUBLIC ) if(MIOPEN_USE_COMPOSABLEKERNEL) -set(MIOPEN_CK_LINK_FLAGS composable_kernel::device_other_operations composable_kernel::device_gemm_operations composable_kernel::device_conv_operations composable_kernel::device_reduction_operations hip::host) +set(MIOPEN_CK_LINK_FLAGS composable_kernel::device_other_operations composable_kernel::device_gemm_operations composable_kernel::device_conv_operations composable_kernel::device_reduction_operations composable_kernel::device_mha_operations hip::host) endif() if(WIN32) @@ -801,6 +871,11 @@ target_include_directories(MIOpen SYSTEM PUBLIC $ #include +#include #include #include #include #include #include -#include #include #include @@ -349,6 +349,8 @@ inline std::ostream& operator<<(std::ostream& stream, const miopenTensorArgument case miopenTensorMhaAmaxDK: stream << "miopenTensorMhaAmaxDK"; break; case miopenTensorMhaAmaxDV: stream << "miopenTensorMhaAmaxDV"; break; case miopenTensorMhaAmaxDS: stream << "miopenTensorMhaAmaxDS"; break; + case miopenTensorMhaBias: stream << "miopenTensorMhaBias"; break; + case miopenTensorMhaMask: stream << "miopenTensorMhaMask"; break; case miopenTensorSoftmaxX: stream << "SoftmaxX"; break; case miopenTensorSoftmaxY: stream << "SoftmaxY"; break; case miopenTensorSoftmaxDX: stream << "SoftmaxDX"; break; diff --git a/src/base64.cpp b/src/base64.cpp new file mode 100644 index 0000000000..f0f734a9f7 --- /dev/null +++ b/src/base64.cpp @@ -0,0 +1,155 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include +#include + +namespace miopen { + +namespace { + +const std::string_view base64Chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + +} // namespace + +std::string base64Encode(const uint8_t* data, std::size_t length) +{ + // Calculate the exact size of the resulting Base64 string + size_t outputSize = ((length + 2) / 3) * 4; + std::string encodedString; + encodedString.reserve(outputSize); // Preallocate memory for the result + + size_t i = 0; + unsigned char charArray3[3]; + unsigned char charArray4[4]; + + while(length-- != 0U) + { + charArray3[i++] = *(data++); + if(i == 3) + { + charArray4[0] = (charArray3[0] & 0xfc) >> 2; + charArray4[1] = ((charArray3[0] & 0x03) << 4) + ((charArray3[1] & 0xf0) >> 4); + charArray4[2] = ((charArray3[1] & 0x0f) << 2) + ((charArray3[2] & 0xc0) >> 6); + charArray4[3] = charArray3[2] & 0x3f; + + std::transform(std::begin(charArray4), + std::end(charArray4), + std::back_inserter(encodedString), + [](unsigned char c) { return base64Chars[c]; }); + i = 0; + } + } + + if(i != 0U) + { + for(size_t j = i; j < 3; j++) + { + charArray3[j] = '\0'; + } + + charArray4[0] = (charArray3[0] & 0xfc) >> 2; + charArray4[1] = ((charArray3[0] & 0x03) << 4) + ((charArray3[1] & 0xf0) >> 4); + charArray4[2] = ((charArray3[1] & 0x0f) << 2) + ((charArray3[2] & 0xc0) >> 6); + charArray4[3] = charArray3[2] & 0x3f; + + for(size_t j = 0; j < i + 1; j++) + { + encodedString += base64Chars[charArray4[j]]; + } + + while(i++ < 3) + { + encodedString += '='; + } + } + + return encodedString; +} + +std::vector base64Decode(const std::string_view& encodedString) +{ + size_t length = encodedString.size(); + MIOPEN_THROW_IF(length % 4 != 0, "Invalid Base64 input"); + + // Reserve space to avoid reallocations + size_t outputLength = (length / 4) * 3; + if(encodedString[length - 1] == '=') + outputLength--; + if(encodedString[length - 2] == '=') + outputLength--; + + std::vector decodedData; + decodedData.reserve(outputLength); + + uint8_t charArray3[3]; + uint8_t charArray4[4]; + size_t i = 0; + + for(char c : encodedString) + { + if(c == '=') + break; + + auto it = std::find(base64Chars.begin(), base64Chars.end(), c); + MIOPEN_THROW_IF(it == base64Chars.end(), "Invalid character in Base64 string"); + + charArray4[i++] = static_cast(std::distance(base64Chars.begin(), it)); + if(i == 4) + { + charArray3[0] = (charArray4[0] << 2) + ((charArray4[1] & 0x30) >> 4); + charArray3[1] = ((charArray4[1] & 0xf) << 4) + ((charArray4[2] & 0x3c) >> 2); + charArray3[2] = ((charArray4[2] & 0x3) << 6) + charArray4[3]; + + decodedData.insert(decodedData.end(), charArray3, charArray3 + 3); + i = 0; + } + } + + if(i != 0U) + { + for(size_t j = i; j < 4; j++) + { + charArray4[j] = 0; + } + + charArray3[0] = (charArray4[0] << 2) + ((charArray4[1] & 0x30) >> 4); + charArray3[1] = ((charArray4[1] & 0xf) << 4) + ((charArray4[2] & 0x3c) >> 2); + charArray3[2] = ((charArray4[2] & 0x3) << 6) + charArray4[3]; + + for(size_t j = 0; j < i - 1; j++) + { + decodedData.push_back(charArray3[j]); + } + } + + return decodedData; +} + +} // namespace miopen diff --git a/src/batch_norm.cpp b/src/batch_norm.cpp index a3c5f93e36..2c5486f307 100644 --- a/src/batch_norm.cpp +++ b/src/batch_norm.cpp @@ -66,15 +66,36 @@ void DeriveBNTensorDescriptor(TensorDescriptor& derivedBnDesc, TensorDescriptor BuildReshaped4DTensorDescriptor(const miopen::TensorDescriptor& tDesc) { - auto dataType = tDesc.GetType(); std::vector dims(tDesc.GetLengths()); - // NxCxDxHxW -> NxCx(D*H)xW - dims[2] *= dims[3]; - dims[3] = dims[4]; - dims.pop_back(); + auto dataType = tDesc.GetType(); + auto layout = tDesc.GetLayout_t(); + if(layout == miopenTensorNCDHW) + { + layout = miopenTensorNCHW; + + // NxCxDxHxW -> NxCx(D*H)xW + dims[2] *= dims[3]; + dims[3] = dims[4]; + dims.pop_back(); + } + else if(layout == miopenTensorNDHWC) + { + layout = miopenTensorNHWC; + + // NxDxHxWxC -> Nx(D*H)xWxC + dims[1] *= dims[2]; + dims[2] = dims[3]; + dims[3] = dims[4]; + dims.pop_back(); + } + else + { + std::cout << "Cannot handle layout : " << layout << "\n"; + exit(EXIT_FAILURE); // NOLINT (concurrency-mt-unsafe) + } - return {dataType, dims}; + return {dataType, layout, dims}; } void profileSequence(const Handle& handle, unsigned char select, float* ctime) diff --git a/src/batch_norm_api.cpp b/src/batch_norm_api.cpp index 8f184a9508..7942b0dac9 100644 --- a/src/batch_norm_api.cpp +++ b/src/batch_norm_api.cpp @@ -50,6 +50,10 @@ namespace miopen { namespace debug { void LogCmdBNorm(const miopenTensorDescriptor_t xDesc, + const miopenTensorDescriptor_t yDesc, + const miopenTensorDescriptor_t scaleDesc, + const miopenTensorDescriptor_t biasDesc, + const miopenTensorDescriptor_t saveMeanDesc, miopenBatchNormMode_t bn_mode, const void* resultRunningMean, const void* resultRunningVariance, @@ -60,6 +64,10 @@ void LogCmdBNorm(const miopenTensorDescriptor_t xDesc, if(miopen::IsLoggingCmd()) { const std::string& str = BnormArgsForMIOpenDriver(xDesc, + yDesc, + scaleDesc, + biasDesc, + saveMeanDesc, bn_mode, resultRunningMean, resultRunningVariance, @@ -88,6 +96,130 @@ miopenBatchNormalizationForwardInference(miopenHandle_t handle, void* estimatedMean, void* estimatedVariance, double epsilon) +{ + return miopenBatchNormalizationForwardInference_V2(handle, + bn_mode, + alpha, + beta, + xDesc, + x, + yDesc, + y, + bnScaleBiasMeanVarDesc, + bnScaleBiasMeanVarDesc, + bnScaleBiasMeanVarDesc, + bnScaleBiasMeanVarDesc, + bnScale, + bnBias, + estimatedMean, + estimatedVariance, + epsilon); +} + +extern "C" miopenStatus_t +miopenBatchNormalizationForwardTraining(miopenHandle_t handle, + miopenBatchNormMode_t bn_mode, + void* alpha, + void* beta, + const miopenTensorDescriptor_t xDesc, + const void* x, + const miopenTensorDescriptor_t yDesc, + void* y, + const miopenTensorDescriptor_t bnScaleBiasMeanVarDesc, + void* bnScale, + void* bnBias, + double expAvgFactor, + void* resultRunningMean, + void* resultRunningVariance, + double epsilon, + void* resultSaveMean, + void* resultSaveInvVariance) +{ + return miopenBatchNormalizationForwardTraining_V2(handle, + bn_mode, + alpha, + beta, + xDesc, + x, + yDesc, + y, + bnScaleBiasMeanVarDesc, + bnScaleBiasMeanVarDesc, + bnScaleBiasMeanVarDesc, + bnScaleBiasMeanVarDesc, + bnScale, + bnBias, + expAvgFactor, + resultRunningMean, + resultRunningVariance, + epsilon, + resultSaveMean, + resultSaveInvVariance); +} + +extern "C" miopenStatus_t +miopenBatchNormalizationBackward(miopenHandle_t handle, + miopenBatchNormMode_t bn_mode, + const void* alphaDataDiff, + const void* betaDataDiff, + const void* alphaParamDiff, + const void* betaParamDiff, + const miopenTensorDescriptor_t xDesc, + const void* x, + const miopenTensorDescriptor_t dyDesc, + const void* dy, + const miopenTensorDescriptor_t dxDesc, + void* dx, + const miopenTensorDescriptor_t bnScaleBiasDiffDesc, + const void* bnScale, + void* resultBnScaleDiff, + void* resultBnBiasDiff, + double epsilon, + const void* savedMean, + const void* savedInvVariance) +{ + return miopenBatchNormalizationBackward_V2(handle, + bn_mode, + alphaDataDiff, + betaDataDiff, + alphaParamDiff, + betaParamDiff, + xDesc, + x, + dyDesc, + dy, + dxDesc, + dx, + bnScaleBiasDiffDesc, + bnScaleBiasDiffDesc, + bnScaleBiasDiffDesc, + bnScaleBiasDiffDesc, + bnScale, + resultBnScaleDiff, + resultBnBiasDiff, + epsilon, + savedMean, + savedInvVariance); +} + +extern "C" miopenStatus_t +miopenBatchNormalizationForwardInference_V2(miopenHandle_t handle, + miopenBatchNormMode_t bn_mode, + void* alpha, + void* beta, + const miopenTensorDescriptor_t xDesc, + const void* x, + const miopenTensorDescriptor_t yDesc, + void* y, + const miopenTensorDescriptor_t scaleDesc, + const miopenTensorDescriptor_t biasDesc, + const miopenTensorDescriptor_t estMeanDesc, + const miopenTensorDescriptor_t estVarianceDesc, + void* bnScale, + void* bnBias, + void* estimatedMean, + void* estimatedVariance, + double epsilon) { MIOPEN_LOG_FUNCTION(handle, bn_mode, @@ -95,7 +227,10 @@ miopenBatchNormalizationForwardInference(miopenHandle_t handle, x, yDesc, y, - bnScaleBiasMeanVarDesc, + scaleDesc, + biasDesc, + estMeanDesc, + estVarianceDesc, bnScale, bnBias, estimatedMean, @@ -103,65 +238,76 @@ miopenBatchNormalizationForwardInference(miopenHandle_t handle, epsilon); miopen::debug::LogCmdBNorm(xDesc, + yDesc, + scaleDesc, + biasDesc, + estMeanDesc, bn_mode, - estimatedMean, - estimatedVariance, nullptr, nullptr, + estMeanDesc, + estimatedVariance, miopen::debug::BatchNormDirection_t::ForwardInference); - // In case of NxCxDxHxW int size{0}; miopenGetTensorDescriptorSize(xDesc, &size); + // In case of NxCxDxHxW + auto ReshapeIfNeeded = [size](const auto desc) { + return (size == 5) ? miopen::BuildReshaped4DTensorDescriptor(miopen::deref(desc)) + : miopen::deref(desc); + }; return miopen::try_([&] { - miopen::BatchNormForwardInference( - miopen::deref(handle), - bn_mode, - alpha, - beta, - (size == 5) ? miopen::BuildReshaped4DTensorDescriptor(miopen::deref(xDesc)) - : miopen::deref(xDesc), - DataCast(x), - (size == 5) ? miopen::BuildReshaped4DTensorDescriptor(miopen::deref(yDesc)) - : miopen::deref(yDesc), - DataCast(y), - (size == 5) - ? miopen::BuildReshaped4DTensorDescriptor(miopen::deref(bnScaleBiasMeanVarDesc)) - : miopen::deref(bnScaleBiasMeanVarDesc), - DataCast(bnScale), - DataCast(bnBias), - DataCast(estimatedMean), - DataCast(estimatedVariance), - epsilon); + miopen::BatchNormForwardInference(miopen::deref(handle), + bn_mode, + alpha, + beta, + ReshapeIfNeeded(xDesc), + DataCast(x), + ReshapeIfNeeded(yDesc), + DataCast(y), + ReshapeIfNeeded(scaleDesc), + ReshapeIfNeeded(biasDesc), + ReshapeIfNeeded(estMeanDesc), + ReshapeIfNeeded(estVarianceDesc), + DataCast(bnScale), + DataCast(bnBias), + DataCast(estimatedMean), + DataCast(estimatedVariance), + epsilon); }); } extern "C" miopenStatus_t -miopenBatchNormalizationForwardTraining(miopenHandle_t handle, - miopenBatchNormMode_t bn_mode, - void* alpha, - void* beta, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t yDesc, - void* y, - const miopenTensorDescriptor_t bnScaleBiasMeanVarDesc, - void* bnScale, - void* bnBias, - double expAvgFactor, - void* resultRunningMean, - void* resultRunningVariance, - double epsilon, - void* resultSaveMean, - void* resultSaveInvVariance) +miopenBatchNormalizationForwardTraining_V2(miopenHandle_t handle, + miopenBatchNormMode_t bn_mode, + void* alpha, + void* beta, + const miopenTensorDescriptor_t xDesc, + const void* x, + const miopenTensorDescriptor_t yDesc, + void* y, + const miopenTensorDescriptor_t scaleDesc, + const miopenTensorDescriptor_t biasDesc, + const miopenTensorDescriptor_t savedMeanDesc, + const miopenTensorDescriptor_t savedVarianceDesc, + void* bnScale, + void* bnBias, + double expAvgFactor, + void* resultRunningMean, + void* resultRunningVariance, + double epsilon, + void* resultSaveMean, + void* resultSaveInvVariance) { - MIOPEN_LOG_FUNCTION(handle, bn_mode, xDesc, x, yDesc, y, - bnScaleBiasMeanVarDesc, + scaleDesc, + biasDesc, + savedMeanDesc, + savedVarianceDesc, bnScale, bnBias, expAvgFactor, @@ -172,63 +318,72 @@ miopenBatchNormalizationForwardTraining(miopenHandle_t handle, resultSaveInvVariance); miopen::debug::LogCmdBNorm(xDesc, + yDesc, + scaleDesc, + biasDesc, + savedMeanDesc, bn_mode, resultRunningMean, resultRunningVariance, resultSaveMean, resultSaveInvVariance, miopen::debug::BatchNormDirection_t::ForwardTraining); - // In case of NxCxDxHxW + int size{0}; miopenGetTensorDescriptorSize(xDesc, &size); + // In case of NxCxDxHxW + auto ReshapeIfNeeded = [size](const auto desc) { + return (size == 5) ? miopen::BuildReshaped4DTensorDescriptor(miopen::deref(desc)) + : miopen::deref(desc); + }; return miopen::try_([&] { - miopen::BatchNormForwardTraining( - miopen::deref(handle), - bn_mode, - alpha, - beta, - (size == 5) ? miopen::BuildReshaped4DTensorDescriptor(miopen::deref(xDesc)) - : miopen::deref(xDesc), - DataCast(x), - (size == 5) ? miopen::BuildReshaped4DTensorDescriptor(miopen::deref(yDesc)) - : miopen::deref(yDesc), - DataCast(y), - (size == 5) - ? miopen::BuildReshaped4DTensorDescriptor(miopen::deref(bnScaleBiasMeanVarDesc)) - : miopen::deref(bnScaleBiasMeanVarDesc), - DataCast(bnScale), - DataCast(bnBias), - expAvgFactor, - DataCast(resultRunningMean), - DataCast(resultRunningVariance), - epsilon, - DataCast(resultSaveMean), - DataCast(resultSaveInvVariance)); + miopen::BatchNormForwardTraining(miopen::deref(handle), + bn_mode, + alpha, + beta, + ReshapeIfNeeded(xDesc), + DataCast(x), + ReshapeIfNeeded(yDesc), + DataCast(y), + ReshapeIfNeeded(scaleDesc), + ReshapeIfNeeded(biasDesc), + ReshapeIfNeeded(savedMeanDesc), + ReshapeIfNeeded(savedVarianceDesc), + DataCast(bnScale), + DataCast(bnBias), + expAvgFactor, + DataCast(resultRunningMean), + DataCast(resultRunningVariance), + epsilon, + DataCast(resultSaveMean), + DataCast(resultSaveInvVariance)); }); } extern "C" miopenStatus_t -miopenBatchNormalizationBackward(miopenHandle_t handle, - miopenBatchNormMode_t bn_mode, - const void* alphaDataDiff, - const void* betaDataDiff, - const void* alphaParamDiff, - const void* betaParamDiff, - const miopenTensorDescriptor_t xDesc, - const void* x, - const miopenTensorDescriptor_t dyDesc, - const void* dy, - const miopenTensorDescriptor_t dxDesc, - void* dx, - const miopenTensorDescriptor_t bnScaleBiasDiffDesc, - const void* bnScale, - void* resultBnScaleDiff, - void* resultBnBiasDiff, - double epsilon, - const void* savedMean, - const void* savedInvVariance) +miopenBatchNormalizationBackward_V2(miopenHandle_t handle, + miopenBatchNormMode_t bn_mode, + const void* alphaDataDiff, + const void* betaDataDiff, + const void* alphaParamDiff, + const void* betaParamDiff, + const miopenTensorDescriptor_t xDesc, + const void* x, + const miopenTensorDescriptor_t dyDesc, + const void* dy, + const miopenTensorDescriptor_t dxDesc, + void* dx, + const miopenTensorDescriptor_t scaleDesc, + const miopenTensorDescriptor_t biasDesc, + const miopenTensorDescriptor_t savedMeanDesc, + const miopenTensorDescriptor_t savedVarianceDesc, + const void* bnScale, + void* resultBnScaleDiff, + void* resultBnBiasDiff, + double epsilon, + const void* savedMean, + const void* savedInvVariance) { - MIOPEN_LOG_FUNCTION(handle, bn_mode, xDesc, @@ -237,7 +392,10 @@ miopenBatchNormalizationBackward(miopenHandle_t handle, dy, dxDesc, dx, - bnScaleBiasDiffDesc, + scaleDesc, + biasDesc, + savedMeanDesc, + savedVarianceDesc, bnScale, resultBnScaleDiff, resultBnBiasDiff, @@ -245,40 +403,45 @@ miopenBatchNormalizationBackward(miopenHandle_t handle, savedMean, savedInvVariance); miopen::debug::LogCmdBNorm(xDesc, + dyDesc, + scaleDesc, + biasDesc, + savedMeanDesc, bn_mode, nullptr, nullptr, savedMean, savedInvVariance, miopen::debug::BatchNormDirection_t::Backward); - // In case of NxCxDxHxW int size{0}; miopenGetTensorDescriptorSize(xDesc, &size); + // In case of NxCxDxHxW + auto ReshapeIfNeeded = [size](const auto desc) { + return (size == 5) ? miopen::BuildReshaped4DTensorDescriptor(miopen::deref(desc)) + : miopen::deref(desc); + }; return miopen::try_([&] { - miopen::BatchNormBackward( - miopen::deref(handle), - bn_mode, - alphaDataDiff, - betaDataDiff, - alphaParamDiff, - betaParamDiff, - (size == 5) ? miopen::BuildReshaped4DTensorDescriptor(miopen::deref(xDesc)) - : miopen::deref(xDesc), - DataCast(x), - (size == 5) ? miopen::BuildReshaped4DTensorDescriptor(miopen::deref(dyDesc)) - : miopen::deref(dyDesc), - DataCast(dy), - (size == 5) ? miopen::BuildReshaped4DTensorDescriptor(miopen::deref(dxDesc)) - : miopen::deref(dxDesc), - DataCast(dx), - (size == 5) - ? miopen::BuildReshaped4DTensorDescriptor(miopen::deref(bnScaleBiasDiffDesc)) - : miopen::deref(bnScaleBiasDiffDesc), - DataCast(bnScale), - DataCast(resultBnScaleDiff), - DataCast(resultBnBiasDiff), - epsilon, - DataCast(savedMean), - DataCast(savedInvVariance)); + miopen::BatchNormBackward(miopen::deref(handle), + bn_mode, + alphaDataDiff, + betaDataDiff, + alphaParamDiff, + betaParamDiff, + ReshapeIfNeeded(xDesc), + DataCast(x), + ReshapeIfNeeded(dyDesc), + DataCast(dy), + ReshapeIfNeeded(dxDesc), + DataCast(dx), + ReshapeIfNeeded(scaleDesc), + ReshapeIfNeeded(biasDesc), + ReshapeIfNeeded(savedMeanDesc), + ReshapeIfNeeded(savedVarianceDesc), + DataCast(bnScale), + DataCast(resultBnScaleDiff), + DataCast(resultBnBiasDiff), + epsilon, + DataCast(savedMean), + DataCast(savedInvVariance)); }); } diff --git a/src/batchnorm/problem_description.cpp b/src/batchnorm/problem_description.cpp index 39bc909aed..daebf49208 100644 --- a/src/batchnorm/problem_description.cpp +++ b/src/batchnorm/problem_description.cpp @@ -36,6 +36,109 @@ namespace miopen { namespace batchnorm { +bool is_fp16_or_bfp16(miopenDataType_t type) +{ + return ((type == miopenHalf) || (type == miopenBFloat16)); +} + +bool is_fp32_or_fp64(miopenDataType_t type) +{ + return ((type == miopenFloat) || (type == miopenDouble)); +} + +bool is_fp32(miopenDataType_t type) { return (type == miopenFloat); } + +bool IsOCLInferTypeValid(const ProblemDescription& bn_problem) +{ + // case 1 : mix type + return ( + (is_fp16_or_bfp16(bn_problem.GetXDesc().GetType()) && + is_fp16_or_bfp16(bn_problem.GetYDesc().GetType()) && + is_fp32(bn_problem.GetBnScale().GetType()) && is_fp32(bn_problem.GetBnBias().GetType())) || + // case 2 : float type + (is_fp32(bn_problem.GetXDesc().GetType()) && is_fp32(bn_problem.GetYDesc().GetType()) && + is_fp32(bn_problem.GetBnScale().GetType()) && is_fp32(bn_problem.GetBnBias().GetType()))); +} + +bool IsCKInferTypeValid(const ProblemDescription& bn_problem) +{ + // case 1 : mix type + return ((is_fp16_or_bfp16(bn_problem.GetXDesc().GetType()) && + is_fp16_or_bfp16(bn_problem.GetYDesc().GetType()) && + is_fp16_or_bfp16(bn_problem.GetBnScale().GetType()) && + is_fp16_or_bfp16(bn_problem.GetBnBias().GetType()) && + is_fp32(bn_problem.GetBnSMean().GetType()) && + is_fp32(bn_problem.GetBnSVar().GetType())) || + // case 2 : fp32 or fp64 + (is_fp32_or_fp64(bn_problem.GetXDesc().GetType()) && + is_fp32_or_fp64(bn_problem.GetYDesc().GetType()) && + is_fp32_or_fp64(bn_problem.GetBnScale().GetType()) && + is_fp32_or_fp64(bn_problem.GetBnBias().GetType()) && + is_fp32_or_fp64(bn_problem.GetBnSMean().GetType()) && + is_fp32_or_fp64(bn_problem.GetBnSVar().GetType()))); +} + +bool IsOCLFwdTrainTypeValid(const ProblemDescription& bn_problem) +{ + // case 1 : mix type + return ( + (is_fp16_or_bfp16(bn_problem.GetXDesc().GetType()) && + is_fp16_or_bfp16(bn_problem.GetYDesc().GetType()) && + is_fp32(bn_problem.GetBnScale().GetType()) && is_fp32(bn_problem.GetBnBias().GetType())) || + // case 2 : float type + (is_fp32(bn_problem.GetXDesc().GetType()) && is_fp32(bn_problem.GetYDesc().GetType()) && + is_fp32(bn_problem.GetBnScale().GetType()) && is_fp32(bn_problem.GetBnBias().GetType()))); +} + +bool IsCKFwdTrainTypeValid(const ProblemDescription& bn_problem) +{ + // case 1 : mix type + return ((is_fp16_or_bfp16(bn_problem.GetXDesc().GetType()) && + is_fp16_or_bfp16(bn_problem.GetYDesc().GetType()) && + is_fp16_or_bfp16(bn_problem.GetBnScale().GetType()) && + is_fp16_or_bfp16(bn_problem.GetBnBias().GetType()) && + is_fp32(bn_problem.GetBnSMean().GetType()) && + is_fp32(bn_problem.GetBnSVar().GetType())) || + // case 2 : fp32 or fp64 + (is_fp32_or_fp64(bn_problem.GetXDesc().GetType()) && + is_fp32_or_fp64(bn_problem.GetYDesc().GetType()) && + is_fp32_or_fp64(bn_problem.GetBnScale().GetType()) && + is_fp32_or_fp64(bn_problem.GetBnBias().GetType()) && + is_fp32_or_fp64(bn_problem.GetBnSMean().GetType()) && + is_fp32_or_fp64(bn_problem.GetBnSVar().GetType()))); +} + +bool IsOCLBwdTypeValid(const ProblemDescription& bn_problem) +{ + return ( + (is_fp16_or_bfp16(bn_problem.GetXDesc().GetType()) && + is_fp16_or_bfp16(bn_problem.GetDXDesc().GetType()) && + is_fp16_or_bfp16(bn_problem.GetDYDesc().GetType()) && + is_fp32(bn_problem.GetBnScale().GetType()) && is_fp32(bn_problem.GetBnSMean().GetType()) && + is_fp32(bn_problem.GetBnSVar().GetType())) || + // case 1 : fp32 + (is_fp32(bn_problem.GetXDesc().GetType()) && is_fp32(bn_problem.GetDXDesc().GetType()) && + is_fp32(bn_problem.GetBnScale().GetType()) && is_fp32(bn_problem.GetBnBias().GetType()) && + is_fp32(bn_problem.GetBnSMean().GetType()) && is_fp32(bn_problem.GetBnSVar().GetType()))); +} + +bool IsCKBwdTypeValid(const ProblemDescription& bn_problem) +{ + return ((is_fp16_or_bfp16(bn_problem.GetXDesc().GetType()) && + bn_problem.GetDXDesc().GetType() == miopenFloat && + is_fp16_or_bfp16(bn_problem.GetBnScale().GetType()) && + bn_problem.GetDYDesc().GetType() == miopenFloat && + bn_problem.GetBnSMean().GetType() == miopenFloat && + bn_problem.GetBnSVar().GetType() == miopenFloat) || + // case 1 : fp32 or fp64 + (is_fp32_or_fp64(bn_problem.GetXDesc().GetType()) && + is_fp32_or_fp64(bn_problem.GetDXDesc().GetType()) && + is_fp32_or_fp64(bn_problem.GetBnScale().GetType()) && + is_fp32_or_fp64(bn_problem.GetBnBias().GetType()) && + is_fp32_or_fp64(bn_problem.GetBnSMean().GetType()) && + is_fp32_or_fp64(bn_problem.GetBnSVar().GetType()))); +} + NetworkConfig ProblemDescription::MakeNetworkConfig() const { switch(direction) @@ -67,7 +170,7 @@ NetworkConfig ProblemDescription::MakeForwardTrainingNetworkConfig() const size_t ygridsize = 1; bool bfpmixparm = false; - if(xDesc.GetType() == miopenHalf && GetBnScaleBiasMeanVarDesc().GetType() == miopenFloat) + if(IsMix()) { bfpmixparm = true; } @@ -137,7 +240,8 @@ NetworkConfig ProblemDescription::MakeForwardTrainingNetworkConfig() const ss << "fp16" << static_cast(IsFp16()); ss << "fp32" << static_cast(IsFp32()); ss << "fp64" << static_cast(IsFp64()); - ss << "fbf16" << static_cast(IsBfp16()); + ss << "fbf16" << static_cast(IsBFp16()); + ss << "fmix" << static_cast(IsMix()); ss << "c" << c; } else @@ -153,7 +257,8 @@ NetworkConfig ProblemDescription::MakeForwardTrainingNetworkConfig() const ss << "fp16" << static_cast(IsFp16()); ss << "fp32" << static_cast(IsFp32()); ss << "fp64" << static_cast(IsFp64()); - ss << "fbf16" << static_cast(IsBfp16()); + ss << "fbf16" << static_cast(IsBFp16()); + ss << "fmix" << static_cast(IsMix()); ss << "single" << static_cast(single); ss << "n" << n; ss << "c" << c; @@ -171,7 +276,8 @@ NetworkConfig ProblemDescription::MakeForwardTrainingNetworkConfig() const ss << "fp16" << static_cast(IsFp16()); ss << "fp32" << static_cast(IsFp32()); ss << "fp64" << static_cast(IsFp64()); - ss << "fbf16" << static_cast(IsBfp16()); + ss << "fbf16" << static_cast(IsBFp16()); + ss << "fmix" << static_cast(IsMix()); ss << "gx" << xgridsize; ss << "gy" << ygridsize; ss << "lx" << xlocalsize; @@ -183,7 +289,9 @@ NetworkConfig ProblemDescription::MakeForwardTrainingNetworkConfig() const ss << "c" << c; ss << "hw" << in_cstride; } - ss << "layout" << xDesc.GetLayout_str(); + ss << "layout" << in_layout; + ss << "scaleType" << static_cast(IsScaleFp16()); + ss << "scaleType" << static_cast(IsScaleFp32()); return NetworkConfig{ss.str()}; } @@ -200,11 +308,14 @@ NetworkConfig ProblemDescription::MakeForwardInferenceNetworkConfig() const ss << "fp16" << static_cast(IsFp16()); ss << "fp32" << static_cast(IsFp32()); ss << "fp64" << static_cast(IsFp64()); - ss << "fbf16" << static_cast(IsBfp16()); + ss << "fbf16" << static_cast(IsBFp16()); + ss << "fmix" << static_cast(IsMix()); ss << "mode" << bn_mode; ss << "HWdims" << in_cstride; ss << "C" << c; - ss << "layout" << xDesc.GetLayout_str(); + ss << "layout" << in_layout; + ss << "scaleType" << static_cast(IsScaleFp16()); + ss << "scaleType" << static_cast(IsScaleFp32()); return NetworkConfig{ss.str()}; } @@ -214,7 +325,7 @@ NetworkConfig ProblemDescription::MakeBackwardNetworkConfig() const std::ostringstream ss; bool bfpmixparm = false; - if(xDesc.GetType() == miopenHalf && GetScaleBiasDiffDesc().GetType() == miopenFloat) + if(xDesc.GetType() == miopenHalf && GetBnScale().GetType() == miopenFloat) { bfpmixparm = true; } @@ -307,7 +418,8 @@ NetworkConfig ProblemDescription::MakeBackwardNetworkConfig() const ss << "fp16" << static_cast(IsFp16()); ss << "fp32" << static_cast(IsFp32()); ss << "fp64" << static_cast(IsFp64()); - ss << "fbf16" << static_cast(IsBfp16()); + ss << "fbf16" << static_cast(IsBFp16()); + ss << "fmix" << static_cast(IsMix()); ss << "single" << static_cast(single); ss << "gcn" << ldsgcn; } @@ -329,10 +441,13 @@ NetworkConfig ProblemDescription::MakeBackwardNetworkConfig() const ss << "fp16" << static_cast(IsFp16()); ss << "fp32" << static_cast(IsFp32()); ss << "fp64" << static_cast(IsFp64()); - ss << "fbf16" << static_cast(IsBfp16()); + ss << "fbf16" << static_cast(IsBFp16()); + ss << "fmix" << static_cast(IsMix()); ss << "nhw" << in_nhw; } - ss << "layout" << xDesc.GetLayout_str(); + ss << "layout" << in_layout; + ss << "scaleType" << static_cast(IsScaleFp16()); + ss << "scaleType" << static_cast(IsScaleFp32()); return NetworkConfig{ss.str()}; } diff --git a/src/comgr.cpp b/src/comgr.cpp index aa53b71bb5..8d2aebaf9b 100644 --- a/src/comgr.cpp +++ b/src/comgr.cpp @@ -701,6 +701,7 @@ void BuildAsm(const std::string& name, #define WORKAROUND_ISSUE_HIPRTC_HIPRTC_HEADER_H 1 // See SWDEV-307838, issue #1648. #define WORKAROUND_ISSUE_1674 (HIP_PACKAGE_VERSION_FLAT >= 5003022305ULL) +#define WORKAROUND_ISSUE_3188 (HIP_PACKAGE_VERSION_FLAT >= 6002041133ULL) // See WORKAROUND_SWDEV_413293 in ./CmakeLists.txt #define WORKAROUND_SWDEV_413293 MIOPEN_HIP_COMPILER_HAS_OPTION_OFFLOAD_UNIFORM_BLOCK @@ -975,6 +976,9 @@ void BuildHip(const std::string& name, #endif #if WORKAROUND_ISSUE_1674 opts.push_back("-Wno-gnu-line-marker"); +#endif +#if WORKAROUND_ISSUE_3188 + opts.push_back("-Wno-pass-failed"); #endif opts.push_back("-Wno-cuda-compat"); opts.push_back("-fno-gpu-rdc"); @@ -988,8 +992,12 @@ void BuildHip(const std::string& name, opts.push_back("-std=c++17"); HiprtcProgram prog(name, text); - prog.Compile(opts); - prog.GetCode(binary); + try{ // TEMPCODE RJS + + std::cout << "Compling HIP: '" << name << "'" << std::endl; // TEMPCODE RJS + prog.Compile(opts); + } catch(Error& ex) { std::cout << __FUNCTION__ << " : Exception calling prog.Compile!: " << ex.text << std::endl; throw(ex); } + prog.GetCode(binary); } catch(Error& ex) { diff --git a/src/conv/heuristics/ai_heuristics.cpp b/src/conv/heuristics/ai_heuristics.cpp index a8a456e2d6..dc922bd2eb 100644 --- a/src/conv/heuristics/ai_heuristics.cpp +++ b/src/conv/heuristics/ai_heuristics.cpp @@ -102,16 +102,24 @@ size_t Metadata::EncodePrecision(miopenDataType_t data_type) const return precision_encodings.at("FP16"); else if(data_type == miopenFloat) return precision_encodings.at("FP32"); - MIOPEN_THROW("Unsupported data type passed through TunaNet applicability check"); + MIOPEN_THROW("Unsupported data type passed to TunaNet"); } size_t Metadata::EncodeLayout(const std::string& layout) const { - if(layout != "NCDHW" && layout != "NCHW") - MIOPEN_THROW("Unsupported layout passed through TunaNet applicability check"); + if(layout != "NCDHW" && layout != "NCHW") // TunaNet supports NCHW and NCDHW layouts only atm + MIOPEN_THROW("Unsupported layout passed to TunaNet"); return layout_encodings.at(layout); } +/** `Model` encapuslates the machinery required to run inference on a TunaNet model + * + * The `Model` class encapuslates all the machinery needed to run inference on a + * TunaNet model, including loading the TunaNet model, formatting a problem so that it + * can be fed into TunaNet for inference, and getting TunaNet's predictions etc. + * + * @param arch Architecture + */ class Model { public: @@ -123,9 +131,30 @@ class Model offset(metadata.num_outputs - metadata.num_solvers) { } - virtual ~Model() = default; + virtual ~Model() = default; + /** Is given problem supported by TunaNet? + * + * A TunaNet model can only work with problems "similar" to the problems it was trained on. + * Since our training data has changed over time, a TunaNet model trained for an earlier + * GPU might not support the same set of problems as a TunaNet model trained for a later + * GPU might. Thus, each subclass of `Model`, specializing `Model` to a specific GPU, must + * implement its own `IsProblemSupported` function. + * + * @param problem Problem + * @param ctx Execution context + */ virtual bool IsProblemSupported(const conv::ProblemDescription& problem, const ExecutionContext& ctx) const = 0; + /** Forward (i.e., run inference on) problem through TunaNet + * + * This function takes in a problem, converts it to a numeric vector and feeds it TunaNet + * for inference. Its output is a numeric vector that represents a probability distribution. + * Each index in this vector represents a solver (as given in metadata.solver_map) and the + * value at each index represents the probability that that solver is the fastest for given + * convolution problem. + * + * @param problem Problem + */ std::vector Forward(const conv::ProblemDescription& problem) const { std::vector features = ToFeatures(problem); @@ -136,9 +165,18 @@ class Model } protected: - const fdeep::model model; - const fdeep::tensor_shape input_shape; - const size_t offset; + const fdeep::model model; // TunaNet model + const fdeep::tensor_shape input_shape; // Shape of input tensor required by TunaNet + const size_t offset; // Some TunaNet models output some "fluff" before they output kernel + // probabilites. This offset tells how many indexes of fluff need to + // be skipped in order to get to kernel probabilities. + /** Path to model file for given GPU + * + * The model files for each GPU are identified by the GPU architecture. This function takes + * in a GPU architecture and returns the path to its TunaNet model. + * + * @param arch Architecture + */ static std::string ModelPath(const std::string& arch) { const auto file_path = GetSystemDbPath() / (arch + ".tn.model"); @@ -146,6 +184,15 @@ class Model MIOPEN_THROW(miopenStatusInternalError, "Unable to load AI model file:" + file_path); return file_path.string(); } + /** Convert given problem to a numeric vector + * + * TunaNet takes in a numeric vector representing the given problem. The exact details + * of this vector vary from one TunaNet model to another, and thus this function, which + * converts a problem into a numeric vector that can be fed to TunaNet, must be implemented + * by each sub-class of `Model` on its own. + * + * @param problem Problem + */ virtual std::vector ToFeatures(const conv::ProblemDescription& problem) const = 0; }; @@ -433,7 +480,6 @@ class Gfx942Model final : public Model static_cast(problem.GetDilationH()), static_cast(problem.GetDilationW()), static_cast(problem.GetOutBatchSize()), - static_cast(metadata.EncodeLayout(problem.GetInLayout())), static_cast(metadata.EncodePrecision(problem.GetInDataType())), static_cast(metadata.EncodeDirection(problem.GetDirection())), static_cast(problem.GetGroupCount())}; @@ -453,7 +499,7 @@ std::unique_ptr GetModel(const std::string& device) return std::make_unique(); if(device == "gfx90a") return std::make_unique(); - return std::make_unique(); + return std::make_unique(); // default model if GPU-specific model is not available } std::vector PredictSolver(const conv::ProblemDescription& problem, @@ -486,13 +532,14 @@ std::vector PredictSolver(const conv::ProblemDescription& problem, } MIOPEN_LOG_I2("Evaluating TunaNet"); + std::vector res = model->Forward(problem); // res[i] gives the probability that the + // i-th solver is the fastest for given + // problem. ( The exact name of the i-th + // solver may be obtained as follows: + // model->metadata.solver_map.at(i) ) - std::vector res = model->Forward(problem); + // sort solvers in order of their probabilities std::vector> sort_res(res.size()); - // sorts result based upon magnitude of result in vector, returned from Model, - // paired with original index (idx). Sort magnitudes in descending order. - // Greater magnitude = better solver. Indexes (idx), which will be used to map to solvers, - // with greater corresponding magnitude are at front of the vector so they get priority. for(auto idx = 0; idx < res.size(); idx++) sort_res[idx] = {idx, res[idx]}; const auto cmp = [](const std::pair& a, const std::pair& b) -> bool { @@ -500,12 +547,12 @@ std::vector PredictSolver(const conv::ProblemDescription& problem, }; std::sort(sort_res.begin(), sort_res.end(), cmp); - // map idx to solver id and then anysolver + // map solver idx to solver id and then to anysolver std::vector sol; std::vector any_sol; for(const auto& kinder : sort_res) { - const auto id = kinder.first; + const auto id = kinder.first; // index of solver in probability vector const auto sol_id = solver::Id{model->metadata.solver_map.at(id)}; if(!sol_id.IsValid()) { @@ -553,13 +600,36 @@ class Model { } virtual ~Model() = default; + /** + * Encode the input features into a "context" tensor + * + * @param features Input features + * @param dim Dimension (must be equal to len(features) if transform + * is True and sqrt(len(features)) otherwise) + * @param transform Reshape input features into a square matrix? + */ fdeep::tensors Encode(const std::vector& features, std::size_t dim, bool transform) const { + // if transform==True, reshape input features into a matrix of `dim x dim` dimensions. + // otherwise, have them as a vector of size `dim`. const auto tensor_shape_depth = transform ? dim : 1; fdeep::tensor input_tensor = fdeep::tensor(fdeep::tensor_shape(dim, tensor_shape_depth), features); + return encoder.predict({input_tensor}); } + /** + * Decode the next token based on the previous token and the encoded context. + * + * Decoder predicts the next token based on the previous token and the context predicted + * by the Encoder. A token is a representation of a kernel parameter, i.e., each unique + * token maps to a unique kernel parameter, with the only exception being the token '-1' + * which signals the end of the decoding process (i.e., all kernel parameters have been + * obtained). + * + * @param prev_token Previous token + * @param context Context vector obtained from encoder + */ fdeep::tensors Decode(const float prev_token, const fdeep::tensors& context) const { return decoder.predict( @@ -589,6 +659,17 @@ class Model } }; +/** + * Return the KernelTuningNet model for given architecture and solver + * + * KernelTuningNet models are specific to each solver and are fine-tuned for each + * GPU skew. This function constructs the KernelTuningNet model for the given + * architecture and solver and stores it in a static map, so that the next time + * the same model is required it doesn't have to be constructed anew. + * + * @param arch GPU Architecture + * @param solver Solver + */ std::shared_ptr GetModel(const std::string& arch, const std::string& solver) { static std::map> models; @@ -605,6 +686,18 @@ std::shared_ptr GetModel(const std::string& arch, const std::string& solv } } +/** + * Set kernel parameters for given solver + * + * @param arch GPU Architecture + * @param solver Solver + * @param direction Convolution Direction + * @param features Input features for KernelTuningNet model + * @param transform_features Whether or not to reshape features into a square + * matrix before feeding them to KernelTuningNet + * @param validator A boolean function that accepts an index `i` and a string `v`, and returns + * True iff `v` is a valid kernel parameter value at index `i` + */ bool ModelSetParams(const std::string& arch, const std::string& solver, miopen::conv::Direction direction, @@ -613,7 +706,9 @@ bool ModelSetParams(const std::string& arch, std::function validator) { auto model = GetModel(arch, solver); - int dim = 0; + + // get context + int dim = 0; if(transform_features) dim = std::sqrt(features.size()); else @@ -621,6 +716,8 @@ bool ModelSetParams(const std::string& arch, auto start = std::chrono::high_resolution_clock::now(); fdeep::tensors context = model->Encode(features, dim, transform_features); float decoder_input = 0.0; + + // set direction string std::string dir; switch(direction) { @@ -630,33 +727,38 @@ bool ModelSetParams(const std::string& arch, default: return false; } + // run decoder to set kernel parameters for(size_t i = 0, num_tuning_params = 1; i < num_tuning_params; ++i) { if(i == 0 && (model->metadata.predict_type == 0u)) num_tuning_params = model->metadata.num_tuning_params[dir]; - fdeep::tensors decoder_output = model->Decode(decoder_input, context); - auto token_scores = decoder_output[0].to_vector(); + fdeep::tensors decoder_output = model->Decode(decoder_input, context); + auto token_scores = decoder_output[0].to_vector(); // token_scores[k] gives the + // score of the k-th token + // order tokens according to their scores std::priority_queue> pq; for(int j = 0; j < token_scores.size(); j++) pq.push(std::make_pair(token_scores[j], j)); // sort by value at index + // find a token whose value is a valid kernel parameter for the i-th position int output_token_index = -1; while(!pq.empty()) { - int token = pq.top().second; - // convert index to token value + // get the token with the highest score and look up its value + int token = pq.top().second; std::string value = model->metadata.tuning_decodings[std::to_string(token)]; pq.pop(); - if(value == "-1") + + if(value == "-1") // if token-value is "-1", then decoding has finished { auto stop = std::chrono::high_resolution_clock::now(); auto duration = std::chrono::duration_cast(stop - start); MIOPEN_LOG_I2("Model ran for " << duration.count() << " micro-seconds"); return false; } - if(validator(i, value)) + if(validator(i, value)) // if token-value is a valid kernel parameter, it's set { output_token_index = token; // index with largest value that is valid = predicted index diff --git a/src/conv/invokers/impl_gemm_dynamic.cpp b/src/conv/invokers/impl_gemm_dynamic.cpp index 80a8cb8bc3..c2a2ef06ca 100644 --- a/src/conv/invokers/impl_gemm_dynamic.cpp +++ b/src/conv/invokers/impl_gemm_dynamic.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -177,9 +178,7 @@ InvokerFactory MakeImplGemmDynamicBackwardDataInvokerFactory(const ProblemDescri const int gemm_k_gid = k * y_dot_slice_gid[gemm_id] * x_dot_slice_gid[gemm_id]; is_gemm_not_empty.emplace_back(gemm_k_gid > 0); } - bool need_set_zero = false; - if(y < stride_h || x < stride_w || dilation_h != 1 || dilation_w != 1) - need_set_zero = true; + bool need_set_zero = true; return [=](const std::vector& kernels) { const auto kernel = kernels[0]; @@ -314,13 +313,10 @@ MakeImplGemmDynamicBackwardDataInvokerFactory(const ProblemDescription& problem, const int gemm_k_gid = k * y_dot_slice_gid[gemm_id] * x_dot_slice_gid[gemm_id]; is_gemm_not_empty.emplace_back(gemm_k_gid > 0); } - bool need_set_zero = false; - if(y < stride_h || x < stride_w || dilation_h != 1 || dilation_w != 1) - need_set_zero = true; - - int nxb = cfg.nxb; - int b = h_tilda_slice * w_tilda_slice; - b = (cfg.nxe == 0) ? (b) : ((b + nxb - 1) / nxb) * nxb; // pad to nxb modulo when nxe != 0 + bool need_set_zero = true; + int nxb = cfg.nxb; + int b = h_tilda_slice * w_tilda_slice; + b = (cfg.nxe == 0) ? (b) : ((b + nxb - 1) / nxb) * nxb; // pad to nxb modulo when nxe != 0 uint32_t nb_n0 = cfg.tensor_b_cluster_lengths[2] * cfg.tensor_b_thread_lengths[2]; uint32_t nb_n1b = cfg.tensor_b_cluster_lengths[3] * cfg.tensor_b_thread_lengths[3]; @@ -495,7 +491,8 @@ InvokerFactory MakeImplGemmDynamicForwardXdlopsNHWCInvokerFactory( shift_pack_1 = 0; } - bool need_set_zero = config.gemm_k_global_split > 0; + // Clear buffer for all condition to resolve the NaN issue. + bool need_set_zero = true; bool use_fp32_global_split_on_fp16 = config.vector_store == 1 && config.gemm_k_global_split > 0; std::vector opArgs; @@ -539,7 +536,7 @@ InvokerFactory MakeImplGemmDynamicForwardXdlopsNHWCInvokerFactory( if(problem.GetOut().GetType() == miopenHalf) return use_fp32_global_split_on_fp16; if(problem.GetOut().GetType() == miopenBFloat16) - return need_set_zero; + return config.gemm_k_global_split > 0; return false; }(); const auto is_nchw = problem.IsLayoutDefault(); @@ -796,11 +793,8 @@ InvokerFactory MakeImplGemmDynamicBackwardDataXdlopsNHWCInvokerFactory( int dtile_h = num_of_gemms > 1 ? static_cast(mdiv_group_mn.magic) : h_tilda; int dtile_w = num_of_gemms > 1 ? static_cast(mdiv_group_mn.shift) : w_tilda; - bool need_set_zero = false; + bool need_set_zero = true; bool use_fp32_global_split_on_fp16 = config.vector_store == 1 && config.gemm_k_global_split > 0; - if(y < stride_h || x < stride_w || dilation_h != 1 || dilation_w != 1) - need_set_zero = true; - need_set_zero |= config.gemm_k_global_split > 0; std::vector opArgs; opArgs.emplace_back(0); // placeholder @@ -855,7 +849,10 @@ InvokerFactory MakeImplGemmDynamicBackwardDataXdlopsNHWCInvokerFactory( if(problem.GetOut().GetType() == miopenHalf) return use_fp32_global_split_on_fp16; if(problem.GetOut().GetType() == miopenBFloat16) - return need_set_zero; + { + return (y < stride_h || x < stride_w || dilation_h != 1 || dilation_w != 1 || + config.gemm_k_global_split > 0); + } return false; }(); const auto is_nchw = problem.IsLayoutDefault(); diff --git a/src/conv/problem_description.cpp b/src/conv/problem_description.cpp index 3f4e0abb37..f868e0544d 100644 --- a/src/conv/problem_description.cpp +++ b/src/conv/problem_description.cpp @@ -121,17 +121,12 @@ std::string ProblemDescription::GetAlphaBetaCaseStr() const void ProblemDescription::HeuristicUpdateLayouts() { - const std::string labels = tensor_layout_get_default(in_layout.size()); - static const std::vector supported_layouts = {"NCHW", "NHWC", "CHWN", "NCDHW"}; + for(const std::string& layout : supported_layouts) { - // Skip layouts that doesn't match dimension sizes - if(layout.size() != labels.size()) - continue; - - if(in.IsPossibleLayout(labels, layout) && out.IsPossibleLayout(labels, layout) && - weights.IsPossibleLayout(labels, layout)) + if(in.IsPossibleLayout4D5D(layout) && out.IsPossibleLayout4D5D(layout) && + weights.IsPossibleLayout4D5D(layout)) { in_layout = layout; weights_layout = layout; @@ -294,5 +289,16 @@ void ProblemDescription::SetupFloats(ExecutionContext& ctx) const << "x" << GetDataTypeName(GetOutDataType())); } +std::string ProblemDescription::ComputeLayout(const TensorDescriptor& td) const +{ + return td.GetLayout_str(); +} + +std::string ProblemDescription::ComputeInLayout() const { return ComputeLayout(in); } + +std::string ProblemDescription::ComputeOutLayout() const { return ComputeLayout(out); } + +std::string ProblemDescription::ComputeWeightsLayout() const { return ComputeLayout(weights); } + } // namespace conv } // namespace miopen diff --git a/src/conv/solver_finders.cpp b/src/conv/solver_finders.cpp index b6c561fdaf..43d5e72031 100644 --- a/src/conv/solver_finders.cpp +++ b/src/conv/solver_finders.cpp @@ -264,7 +264,8 @@ static std::vector EvaluateInvokers(Handle& handle, // Run invoker max 6 times, with ~5 sec time limit. using elapsed_t = decltype(handle.GetKernelTime()); constexpr elapsed_t TIME_MS_MAX = 5000.0; - constexpr int N_RUNS_MAX = 6; + constexpr int N_RUNS_MAX = 8; + constexpr int N_RUNS_DISCARD = 3; auto elapsed = static_cast(0); auto first_elapsed = static_cast(0); int i = 0; @@ -272,14 +273,22 @@ static std::vector EvaluateInvokers(Handle& handle, { invoker(handle, invoke_ctx); elapsed += handle.GetKernelTime(); - if(i == 0) + if(i == (N_RUNS_DISCARD - 1)) first_elapsed = elapsed; ++i; } // If the execution time was not too long, // then the 1st run is not counted (assume it's warm-up): - if(i > 1) - elapsed = (elapsed - first_elapsed) / static_cast(i - 1); + if(i > N_RUNS_DISCARD) + { + elapsed = (elapsed - first_elapsed) / static_cast(i - N_RUNS_DISCARD); + } + else if(i > 0) + { + elapsed /= i; + } + + MIOPEN_THROW_IF(elapsed <= 0, "Invalid elapsed time detected in EvaluateInvokers"); MIOPEN_LOG_I(sol << ": " << elapsed << (elapsed < best ? " < " : " >= ") << best); if(elapsed < best) diff --git a/src/convolution.cpp b/src/convolution.cpp index 5110bf2974..51d94ece7b 100644 --- a/src/convolution.cpp +++ b/src/convolution.cpp @@ -35,7 +35,7 @@ #include #include #include -#include +#include #include #include #include @@ -373,8 +373,7 @@ TensorDescriptor ConvolutionDescriptor::GetForwardOutputTensor(const TensorDescr miopenDataType_t yType) const { // output layout same as input - const std::string default_layout = tensor_layout_get_default(xDesc.GetNumDims()); - const std::string in_layout = xDesc.GetLayout(default_layout); + const std::string in_layout = xDesc.GetLayout_str(); return GetForwardOutputTensorWithLayout(xDesc, wDesc, in_layout, yType); } diff --git a/src/convolution_api.cpp b/src/convolution_api.cpp index 9c166278a8..2e8233e9f7 100644 --- a/src/convolution_api.cpp +++ b/src/convolution_api.cpp @@ -237,7 +237,8 @@ miopenConvolutionABBackwardWeightsGetWorkSpaceSize(const miopenAlphaBetaCase_t a size_t K_per_group = K / G; return (alpha_beta_case == BILINEAR || alpha_beta_case == SCALE) || - (data_type == miopenHalf && (is_odd(C_per_group) || is_odd(K_per_group))); + ((data_type == miopenHalf || data_type == miopenBFloat16) && + (is_odd(C_per_group) || is_odd(K_per_group))); }; size_t output_tensor_size = miopen::deref(outputTensorDesc).GetElementSize(); diff --git a/src/db.cpp b/src/db.cpp index 1361597020..62a82931d4 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -67,7 +67,7 @@ PlainTextDb::PlainTextDb(DbKinds db_kind_, const fs::path& filename_, bool is_sy if(!fs::create_directories(directory)) MIOPEN_LOG_W("Unable to create a directory: " << directory); else - fs::permissions(directory, fs::perms::all); + fs::permissions(directory, FS_ENUM_PERMS_ALL); } } } @@ -251,7 +251,7 @@ bool PlainTextDb::FlushUnsafe(const DbRecord& record, const RecordPositions* pos record.WriteContents(file); } - fs::permissions(filename, fs::perms::all); + fs::permissions(filename, FS_ENUM_PERMS_ALL); } else { @@ -286,7 +286,7 @@ bool PlainTextDb::FlushUnsafe(const DbRecord& record, const RecordPositions* pos fs::remove(filename); fs::rename(temp_name, filename); /// \todo What if rename fails? Thou shalt not loose the original file. - fs::permissions(filename, fs::perms::all); + fs::permissions(filename, FS_ENUM_PERMS_ALL); } return true; } diff --git a/src/driver_arguments.cpp b/src/driver_arguments.cpp index bfc6892aad..b730325add 100644 --- a/src/driver_arguments.cpp +++ b/src/driver_arguments.cpp @@ -60,15 +60,82 @@ void ConvDataType(std::stringstream& ss, const miopen::TensorDescriptor& desc) } } -void BnDataType(std::stringstream& ss, const miopen::TensorDescriptor& desc) +// test based on the input tensor and scaleMean. +// We choose scaleMean because its a accumulator type. +void BnDataType(std::stringstream& ss, + const miopen::TensorDescriptor& xDesc, + const miopen::TensorDescriptor& yDesc, + const miopen::TensorDescriptor& scaleDesc, + const miopen::TensorDescriptor& biasDesc, + const miopen::TensorDescriptor& sMeanDesc, + const BatchNormDirection_t bn_mode) { - if(desc.GetType() == miopenHalf) + if(bn_mode == BatchNormDirection_t::ForwardInference || + bn_mode == BatchNormDirection_t::ForwardTraining) { - ss << "bnormfp16"; + if(xDesc.GetType() == miopenHalf && yDesc.GetType() == miopenHalf && + scaleDesc.GetType() == miopenFloat && biasDesc.GetType() == miopenFloat && + sMeanDesc.GetType() == miopenFloat) + { + ss << "bnormfp16"; + } + else if(xDesc.GetType() == miopenBFloat16 && yDesc.GetType() == miopenBFloat16 && + scaleDesc.GetType() == miopenFloat && biasDesc.GetType() == miopenFloat && + sMeanDesc.GetType() == miopenFloat) + { + ss << "bnormbfp16"; + } + else if(xDesc.GetType() == miopenHalf && yDesc.GetType() == miopenHalf && + scaleDesc.GetType() == miopenHalf && biasDesc.GetType() == miopenHalf && + sMeanDesc.GetType() == miopenFloat) + { + ss << "bnormfp16fp32"; + } + else if(xDesc.GetType() == miopenBFloat16 && yDesc.GetType() == miopenBFloat16 && + scaleDesc.GetType() == miopenBFloat16 && biasDesc.GetType() == miopenBFloat16 && + sMeanDesc.GetType() == miopenFloat) + { + ss << "bnormbfp16fp32"; + } + else + { + ss << "bnorm"; + } + } + else if(bn_mode == BatchNormDirection_t::Backward) + { + if(xDesc.GetType() == miopenHalf && yDesc.GetType() == miopenHalf && + scaleDesc.GetType() == miopenFloat && biasDesc.GetType() == miopenFloat && + sMeanDesc.GetType() == miopenFloat) + { + ss << "bnormfp16"; + } + else if(xDesc.GetType() == miopenBFloat16 && yDesc.GetType() == miopenBFloat16 && + scaleDesc.GetType() == miopenFloat && biasDesc.GetType() == miopenFloat && + sMeanDesc.GetType() == miopenFloat) + { + ss << "bnormbfp16"; + } + else if(xDesc.GetType() == miopenHalf && yDesc.GetType() == miopenFloat && + scaleDesc.GetType() == miopenHalf && biasDesc.GetType() == miopenFloat && + sMeanDesc.GetType() == miopenFloat) + { + ss << "bnormfp16fp32"; + } + else if(xDesc.GetType() == miopenBFloat16 && yDesc.GetType() == miopenFloat && + scaleDesc.GetType() == miopenBFloat16 && biasDesc.GetType() == miopenFloat && + sMeanDesc.GetType() == miopenFloat) + { + ss << "bnormbfp16fp32"; + } + else + { + ss << "bnorm"; + } } else { - ss << "bnorm"; + MIOPEN_THROW("Bad Op direction"); } } @@ -142,9 +209,9 @@ std::string ConvArgsForMIOpenDriver(const miopen::TensorDescriptor& xDesc, << " -v " << convDesc.GetConvStrides()[1] // << " -l " << convDesc.GetConvDilations()[0] // << " -j " << convDesc.GetConvDilations()[1]; - std::string x_layout = xDesc.GetLayout("NCHW"); - std::string w_layout = wDesc.GetLayout("NCHW"); - std::string y_layout = yDesc.GetLayout("NCHW"); + std::string x_layout = xDesc.GetLayout_str(); + std::string w_layout = wDesc.GetLayout_str(); + std::string y_layout = yDesc.GetLayout_str(); if(x_layout != "NCHW") { ss << " --in_layout " << x_layout; @@ -182,9 +249,9 @@ std::string ConvArgsForMIOpenDriver(const miopen::TensorDescriptor& xDesc, << " -l " << convDesc.GetConvDilations()[1] // << " -j " << convDesc.GetConvDilations()[2] // << " --spatial_dim 3"; - std::string x_layout = xDesc.GetLayout("NCDHW"); - std::string w_layout = wDesc.GetLayout("NCDHW"); - std::string y_layout = yDesc.GetLayout("NCDHW"); + std::string x_layout = xDesc.GetLayout_str(); + std::string w_layout = wDesc.GetLayout_str(); + std::string y_layout = yDesc.GetLayout_str(); if(x_layout != "NCDHW") { ss << " --in_layout " << x_layout; @@ -211,7 +278,11 @@ std::string ConvArgsForMIOpenDriver(const miopen::TensorDescriptor& xDesc, return ss.str(); } -std::string BnormArgsForMIOpenDriver(miopenTensorDescriptor_t xDesc, +std::string BnormArgsForMIOpenDriver(const miopenTensorDescriptor_t xDesc, + const miopenTensorDescriptor_t yDesc, + const miopenTensorDescriptor_t scaleDesc, + const miopenTensorDescriptor_t biasDesc, + const miopenTensorDescriptor_t saveMeanDesc, miopenBatchNormMode_t bn_mode, const void* resultRunningMean, const void* resultRunningVariance, @@ -224,7 +295,15 @@ std::string BnormArgsForMIOpenDriver(miopenTensorDescriptor_t xDesc, miopenGetTensorDescriptorSize(xDesc, &size); std::stringstream ss; if(print_for_bn_driver) - BnDataType(ss, miopen::deref(xDesc)); + { + BnDataType(ss, + miopen::deref(xDesc), + miopen::deref(yDesc), + miopen::deref(scaleDesc), + miopen::deref(biasDesc), + miopen::deref(saveMeanDesc), + dir); + } ss << " -n " << miopen::deref(xDesc).GetLengths()[0] // clang-format off << " -c " << miopen::deref(xDesc).GetLengths()[1]; @@ -239,7 +318,7 @@ std::string BnormArgsForMIOpenDriver(miopenTensorDescriptor_t xDesc, ss << " -H " << miopen::deref(xDesc).GetLengths()[2] << " -W " << miopen::deref(xDesc).GetLengths()[3]; } - ss << " -M " << bn_mode; // clang-format on + ss << " -m " << bn_mode; // clang-format on if(print_for_bn_driver) { BnDriverInfo(ss, @@ -248,6 +327,7 @@ std::string BnormArgsForMIOpenDriver(miopenTensorDescriptor_t xDesc, resultRunningVariance, resultSaveMean, resultSaveInvVariance); + ss << " --layout " << miopen::deref(xDesc).GetLayout_str(); } return ss.str(); } diff --git a/src/dropout_api.cpp b/src/dropout_api.cpp index 721340b2f7..c846ec5e6b 100644 --- a/src/dropout_api.cpp +++ b/src/dropout_api.cpp @@ -30,6 +30,16 @@ #include #include +// disable __device__ qualifiers +#ifdef FQUALIFIERS +#error rocrand FQUALIFIERS defined externally, probably one of rocrand device header included prior to this +#endif +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-macros" +#define FQUALIFIERS inline +#pragma clang diagnostic pop +#include "kernels/miopen_rocrand.hpp" + extern "C" miopenStatus_t miopenCreateDropoutDescriptor(miopenDropoutDescriptor_t* dropoutDesc) { @@ -63,7 +73,7 @@ extern "C" miopenStatus_t miopenDropoutGetStatesSize(miopenHandle_t handle, return miopen::try_([&] { miopen::deref(stateSizeInBytes) = std::min(size_t(MAX_PRNG_STATE), miopen::deref(handle).GetImage3dMaxWidth()) * - sizeof(prngStates); + sizeof(rocrand_state_xorwow); }); } @@ -177,14 +187,18 @@ extern "C" miopenStatus_t miopenDropoutForward(miopenHandle_t handle, LogCmdDropout(dropoutDesc, xDesc, true); return miopen::try_([&] { miopen::deref(dropoutDesc) - .DropoutForward(miopen::deref(handle), + .Dropout(miopen::deref(handle), miopen::deref(noise_shape), miopen::deref(xDesc), DataCast(x), miopen::deref(yDesc), DataCast(y), DataCast(reserveSpace), - reserveSpaceSizeInBytes); + reserveSpaceSizeInBytes, + static_cast(0), + static_cast(0), + static_cast(0), + false /* is_backward */); }); } @@ -203,13 +217,17 @@ extern "C" miopenStatus_t miopenDropoutBackward(miopenHandle_t handle, LogCmdDropout(dropoutDesc, dxDesc, false); return miopen::try_([&] { miopen::deref(dropoutDesc) - .DropoutBackward(miopen::deref(handle), + .Dropout(miopen::deref(handle), miopen::deref(noise_shape), miopen::deref(dyDesc), DataCast(dy), miopen::deref(dxDesc), DataCast(dx), DataCast(reserveSpace), - reserveSpaceSizeInBytes); + reserveSpaceSizeInBytes, + static_cast(0), + static_cast(0), + static_cast(0), + true /* is_backward */); }); } diff --git a/src/expanduser.cpp b/src/expanduser.cpp index c87d5d10cb..5503688c3f 100644 --- a/src/expanduser.cpp +++ b/src/expanduser.cpp @@ -42,10 +42,11 @@ #include #include +// The magic number 0x0BD00BD0 (LL_SUPER_MAGIC) is defined in the Lustre filesystem source code. +// Please refer to the Lustre project's documentation or source code for details. #ifndef LL_SUPER_MAGIC -#define LL_SUPER_MAGIC \ - 0x0BD00BD0 // LUSTRE - // https://github.com/whamcloud/lustre/blob/a336d7c7c1cd62a5a5213835aa85b8eaa87b076a/lustre/include/uapi/linux/lustre/lustre_user.h#L252 +#define LL_SUPER_MAGIC 0x0BD00BD0 // LUSTRE + #endif #ifndef CEPH_SUPER_MAGIC #define CEPH_SUPER_MAGIC 0x00c36400 diff --git a/src/fin/fin_interface.cpp b/src/fin/fin_interface.cpp new file mode 100644 index 0000000000..d321a687a0 --- /dev/null +++ b/src/fin/fin_interface.cpp @@ -0,0 +1,609 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include + +#if MIOPEN_ENABLE_FIN_INTERFACE + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace miopen { +namespace fin_interface { + +// ================== AnySolver ================== +// This class is an ugly trick. The only reason for it is that each tunable solver has its own +// PerformanceConfig. Needs to be refactored in the future. Although, it might be worth leaving it +// as is to be able to add things needed for tuning infrastructure. +template +class AnySolver +{ +public: + AnySolver(const miopen::solver::SolverBase* solver_base, uint64_t id); + + /// \todo Move to SolverInterface + miopen::solver::ConvSolution FindSolution(const Context& ctx, + const Problem& problem, + miopen::PerformanceDb& db, + const miopen::AnyInvokeParams& invoke_ctx, + const std::string& perf_cfg) const + { + assert(obj != nullptr); + return obj->FindSolution(sbase, ctx, problem, db, invoke_ctx, perf_cfg); + } + + std::vector GetAllSolutions(const Context& ctx, + const Problem& problem) const + { + assert(obj != nullptr); + return obj->GetAllSolutions(sbase, ctx, problem); + } + + std::string + GetPerfCfgParams(const Context& ctx, const Problem& problem, miopen::PerformanceDb& db) const + { + assert(obj != nullptr); + return obj->GetPerfCfgParams(sbase, ctx, problem, db); + } + + bool + TestPerfCfgParams(const Context& ctx, const Problem& problem, const std::string& params) const + { + assert(obj != nullptr); + return obj->TestPerfCfgParams(sbase, ctx, problem, params); + } + +private: + using NonTunableSolver = miopen::solver::SolverInterfaceNonTunable; + + // Virtual base class + class AnySolver_base + { + public: + virtual ~AnySolver_base() = default; + virtual miopen::solver::ConvSolution + FindSolution(const miopen::solver::SolverBase* solver_base, + const Context& ctx, + const Problem& problem, + miopen::PerformanceDb& db, + const miopen::AnyInvokeParams& invoke_ctx, + const std::string& perf_cfg) const = 0; + virtual std::vector + GetAllSolutions(const miopen::solver::SolverBase* solver_base, + const Context& ctx, + const Problem& problem) const = 0; + virtual std::string GetPerfCfgParams(const miopen::solver::SolverBase* solver_base, + const Context& ctx, + const Problem& problem, + miopen::PerformanceDb& db) const = 0; + virtual bool TestPerfCfgParams(const miopen::solver::SolverBase* solver_base, + const Context& ctx, + const Problem& problem, + const std::string& params) const = 0; + }; + + // Templated derived class + template + class AnySolver_impl final : public AnySolver_base + { + miopen::solver::ConvSolution FindSolution(const miopen::solver::SolverBase* solver_base, + const Context& ctx, + const Problem& problem, + miopen::PerformanceDb& db, + const miopen::AnyInvokeParams& invoke_ctx, + const std::string& perf_cfg) const override + { + const auto& solver = GetSolver(solver_base); + return miopen::solver::FindSolution(solver, ctx, problem, db, invoke_ctx, perf_cfg); + } + + std::vector + GetAllSolutions(const miopen::solver::SolverBase* solver_base, + const Context& ctx, + const Problem& problem) const override + { + const auto& solver = GetSolver(solver_base); + if constexpr(std::is_same_v) + { + // Non-tunable solver + return {solver.GetSolution(ctx, problem)}; + } + else + { + // Tunable solver + using PerformanceConfig = + decltype(solver.GetDefaultPerformanceConfig(ctx, problem)); + if constexpr(std::is_same_v) + { + // Legacy tunable solver + MIOPEN_THROW("No solutions returned for Legacy Solvers."); + } + else + { + return miopen::solver::GetAllSolutions(solver, ctx, problem); + } + } + } + + std::string GetPerfCfgParams(const miopen::solver::SolverBase* solver_base, + const Context& ctx, + const Problem& problem, + miopen::PerformanceDb& db) const override + { + const auto& solver = GetSolver(solver_base); + + if constexpr(std::is_same_v) + { + // Non-tunable solver + MIOPEN_LOG_I2("PerformanceDb: No Config: " << solver.SolverDbId()); + return {}; + } + else + { + // Tunable solver + using PerformanceConfig = + decltype(solver.GetDefaultPerformanceConfig(ctx, problem)); + PerformanceConfig config; + + if(db.Load(problem, solver.SolverDbId(), config)) + { + MIOPEN_LOG_I2("PerformanceDb: Record Loaded: " << solver.SolverDbId()); + if(solver.IsValidPerformanceConfig(ctx, problem, config)) + { + return config.ToString(); + } + MIOPEN_LOG_I2("PerformanceDb: Invalid Config: " << solver.SolverDbId()); + } + else if(!solver.AltSolverDbId().empty() && + db.Load(problem, solver.AltSolverDbId(), config)) + { + MIOPEN_LOG_I( + "PerformanceDb: alternate record loaded: " << solver.AltSolverDbId()); + if(solver.IsValidPerformanceConfig(ctx, problem, config)) + { + return config.ToString(); + } + MIOPEN_LOG_I2("PerformanceDb: Invalid alternate record: " + << solver.AltSolverDbId() << ": " << config); + } + + MIOPEN_LOG_I2( + "PerformanceDb: Failed Loading, Using Default: " << solver.SolverDbId()); + config = solver.GetDefaultPerformanceConfig(ctx, problem); + return config.ToString(); + } + } + + bool TestPerfCfgParams(const miopen::solver::SolverBase* solver_base, + const Context& ctx, + const Problem& problem, + const std::string& params) const override + { + if constexpr(std::is_same_v) + { + // Non-tunable solver + return false; + } + else + { + // Tunable solver + const auto& solver = GetSolver(solver_base); + using PerformanceConfig = + decltype(solver.GetDefaultPerformanceConfig(ctx, problem)); + PerformanceConfig config; + + if(!config.Deserialize(params)) + { + MIOPEN_LOG_WE("Perf params are obsolete or corrupt: " + << params << ". Performance may degrade."); + return false; + } + + return solver.IsValidPerformanceConfig(ctx, problem, config); + } + } + + const auto& GetSolver(const miopen::solver::SolverBase* solver_base) const + { + return *static_cast(solver_base); + } + }; + + template + void SetObject() + { + // Test the cast in the constructor using dynamic_cast, so that later we can use static_cast + // everywhere + const T* ptr = dynamic_cast(sbase); + if(ptr == nullptr) + { + std::ostringstream ss; + ss << "Wrong object (T = " << type_name_bare(); + ss << ", name = " << sbase->SolverDbId(); + ss << ")"; + MIOPEN_THROW(miopenStatusInternalError, ss.str()); + } + + static const AnySolver_impl impl; + obj = &impl; + } + + void SetObjectNonTunable() { SetObject(); } + + const AnySolver_base* obj = nullptr; + const miopen::solver::SolverBase* const sbase; +}; + +template <> +AnySolver::AnySolver( + const miopen::solver::SolverBase* solver_base, uint64_t id) + : sbase(solver_base) +{ + if(!sbase->IsTunable()) + { + SetObjectNonTunable(); + return; + } + + switch(id) + { + case 1: SetObject(); break; + case 2: SetObject(); break; + case 3: SetObject(); break; + case 11: SetObject(); break; + case 13: SetObject(); break; + case 16: SetObject(); break; + case 17: SetObject(); break; + case 18: SetObject>(); break; + case 19: SetObject>(); break; + case 20: SetObject>(); break; + case 21: SetObject>(); break; + case 22: SetObject>(); break; + case 26: SetObject(); break; + case 31: SetObject(); break; + case 37: SetObject>(); break; + case 53: SetObject>(); break; + case 54: SetObject(); break; + case 55: SetObject(); break; + case 56: SetObject(); break; + case 57: SetObject(); break; + case 60: SetObject(); break; + case 61: SetObject(); break; + case 64: SetObject(); break; + case 73: SetObject(); break; + case 75: SetObject>(); break; + case 76: SetObject>(); break; + case 77: SetObject>(); break; + case 78: SetObject>(); break; + case 79: SetObject>(); break; + case 80: SetObject(); break; + case 81: + SetObject(); + break; + case 83: SetObject(); break; + case 98: SetObject(); break; + case 99: SetObject(); break; + case 100: SetObject(); break; + case 103: SetObject(); break; + case 104: SetObject(); break; + case 105: SetObject(); break; + case 107: SetObject(); break; + case 108: SetObject(); break; + case 110: SetObject(); break; + case 114: SetObject(); break; + case 127: SetObject(); break; + case 128: SetObject(); break; + case 129: SetObject(); break; + case 137: SetObject(); break; + case 138: SetObject(); break; + case 140: SetObject(); break; + case 141: SetObject(); break; + case 155: SetObject(); break; + case 156: SetObject(); break; + // New tunable solver should be added here + default: + MIOPEN_THROW(miopenStatusInternalError, "Unknown solver ID (" + std::to_string(id) + ")"); + } +} + +template <> +AnySolver::AnySolver( + const miopen::solver::SolverBase* solver_base, uint64_t id) + : sbase(solver_base) +{ + if(!sbase->IsTunable()) + { + SetObjectNonTunable(); + return; + } + + switch(id) + { + case 142: SetObject(); break; + case 143: SetObject(); break; + case 144: SetObject(); break; + // New tunable solver should be added here + default: + MIOPEN_THROW(miopenStatusInternalError, "Unknown solver ID (" + std::to_string(id) + ")"); + } +} + +// ================== Solver ================== +Solver::Solver(const miopen::solver::SolverBase* solver_base, uint64_t solver_id) + : sbase(solver_base), id(solver_id) +{ + if(sbase == nullptr) + MIOPEN_THROW(miopenStatusInternalError); +} + +Solver::Solver(const std::string& requested_name) : rname(requested_name) {} + +bool Solver::IsValid() const { return sbase != nullptr; } + +uint64_t Solver::GetId() const +{ + if(sbase == nullptr) + MIOPEN_THROW(miopenStatusNotInitialized); + + return id; +} + +const std::string& Solver::GetName() const +{ + if(sbase != nullptr) + return sbase->SolverDbId(); + else + return rname; +} + +bool Solver::IsTunable() const +{ + if(sbase == nullptr) + MIOPEN_THROW(miopenStatusNotInitialized); + + return sbase->IsTunable(); +} + +bool Solver::IsDynamic() const +{ + if(sbase == nullptr) + MIOPEN_THROW(miopenStatusNotInitialized); + + return sbase->IsDynamic(); +} + +// ================== SolverMixin ================== +template +SolverMixin::SolverMixin(const miopen::solver::SolverBase* solver_base, + uint64_t solver_id) + : Solver(solver_base, solver_id), asolver(AnySolver(sbase, id)) +{ + // std::any: avoid dynamic allocations for small objects + static_assert(std::is_nothrow_move_constructible_v>); +} + +template +bool SolverMixin::IsApplicable(const Context& ctx, const Problem& problem) const +{ + if(sbase == nullptr) + MIOPEN_THROW(miopenStatusNotInitialized); + + using SolverInterface = miopen::solver::SolverInterface; + return static_cast(sbase)->IsApplicable(ctx, problem); +} + +template +size_t SolverMixin::GetWorkspaceSize(const Context& ctx, + const Problem& problem) const +{ + if(sbase == nullptr) + MIOPEN_THROW(miopenStatusNotInitialized); + + using SolverInterface = miopen::solver::SolverInterface; + return static_cast(sbase)->GetWorkspaceSize(ctx, problem); +} + +template +miopen::solver::ConvSolution +SolverMixin::FindSolution(const Context& ctx, + const Problem& problem, + miopen::PerformanceDb& db, + const miopen::AnyInvokeParams& invoke_ctx, + const std::string& perf_cfg) const +{ + if(sbase == nullptr) + MIOPEN_THROW(miopenStatusNotInitialized); + + const auto& solver = std::any_cast&>(asolver); + return solver.FindSolution(ctx, problem, db, invoke_ctx, perf_cfg); +} + +template +std::vector +SolverMixin::GetAllSolutions(const Context& ctx, const Problem& problem) const +{ + if(sbase == nullptr) + MIOPEN_THROW(miopenStatusNotInitialized); + + const auto& solver = std::any_cast&>(asolver); + return solver.GetAllSolutions(ctx, problem); +} + +template +std::string SolverMixin::GetPerfCfgParams(const Context& ctx, + const Problem& problem, + miopen::PerformanceDb& db) const +{ + if(sbase == nullptr) + MIOPEN_THROW(miopenStatusNotInitialized); + + const auto& solver = std::any_cast&>(asolver); + return solver.GetPerfCfgParams(ctx, problem, db); +} + +template +bool SolverMixin::TestPerfCfgParams(const Context& ctx, + const Problem& problem, + const std::string& params) const +{ + if(sbase == nullptr) + MIOPEN_THROW(miopenStatusNotInitialized); + + const auto& solver = std::any_cast&>(asolver); + return solver.TestPerfCfgParams(ctx, problem, params); +} + +// Explicit instantiation +template class SolverMixin; +template class SolverMixin; + +// ================== ConvSolver ================== +ConvSolver::ConvSolver(const miopen::solver::SolverBase* solver_base, + uint64_t solver_id, + miopenConvAlgorithm_t algo_) + : SolverMixin(solver_base, solver_id), algo(algo_) +{ +} + +std::string ConvSolver::GetAlgo(miopen::conv::Direction dir) const +{ + if(sbase == nullptr) + MIOPEN_THROW(miopenStatusNotInitialized); + + return ConvolutionAlgoToDirectionalString(algo, dir); +} + +// ================== FinInterface ================== +namespace { + +template +struct SolverToPrimitive; + +template <> +struct SolverToPrimitive +{ + static auto GetPrimitive() { return miopen::solver::Primitive::Convolution; } +}; + +template <> +struct SolverToPrimitive +{ + static auto GetPrimitive() { return miopen::solver::Primitive::Batchnorm; } +}; + +} // namespace + +template +const std::vector& GetAllSolvers() +{ + static const auto solvers = [] { + const auto& ids = GetSolversByPrimitive(SolverToPrimitive::GetPrimitive()); + std::vector solvers; + solvers.reserve(ids.size()); + + for(const auto& id : ids) + { + if(!id.IsValid()) + MIOPEN_THROW(miopenStatusInternalError); + + if constexpr(std::is_same_v) + solvers.emplace_back(Solver{id.GetSolverBase(), id.Value(), id.GetAlgo()}); + else + solvers.emplace_back(Solver{id.GetSolverBase(), id.Value()}); + } + + return solvers; + }(); + return solvers; +} + +template +Solver GetSolver(const std::string& name) +{ + const auto id = miopen::solver::Id{name}; + if(!id.IsValid()) + return {name}; + + if constexpr(std::is_same_v) + return {id.GetSolverBase(), id.Value(), id.GetAlgo()}; + else + return {id.GetSolverBase(), id.Value()}; +} + +namespace { + +template +std::vector GetSolvers(const std::vector& names) +{ + std::vector solvers; + solvers.reserve(names.size()); + for(const auto& name : names) + solvers.emplace_back(GetSolver(name)); + return solvers; +} + +} // namespace + +const std::vector& GetAllConvSolvers() { return GetAllSolvers(); } + +std::vector GetConvSolvers(const std::vector& names) +{ + return GetSolvers(names); +} + +ConvSolver GetConvSolver(const std::string& name) { return GetSolver(name); } + +const std::vector& GetAllBatchNormSolvers() +{ + return GetAllSolvers(); +} + +std::vector GetBatchNormSolvers(const std::vector& names) +{ + return GetSolvers(names); +} + +BatchNormSolver GetBatchNormSolver(const std::string& name) +{ + return GetSolver(name); +} + +} // namespace fin_interface +} // namespace miopen + +#endif // MIOPEN_ENABLE_FIN_INTERFACE diff --git a/src/find_controls.cpp b/src/find_controls.cpp index 118743e5d7..da66644ded 100644 --- a/src/find_controls.cpp +++ b/src/find_controls.cpp @@ -39,10 +39,13 @@ #include #include #include +#include +#include MIOPEN_DECLARE_ENV_VAR_STR(MIOPEN_FIND_ENFORCE) MIOPEN_DECLARE_ENV_VAR_STR(MIOPEN_DEBUG_FIND_ONLY_SOLVER) MIOPEN_DECLARE_ENV_VAR_STR(MIOPEN_FIND_MODE) +MIOPEN_DECLARE_ENV_VAR_STR(MIOPEN_FIND_MODE_FUSION) namespace miopen { @@ -196,11 +199,12 @@ std::ostream& operator<<(std::ostream& os, const FindMode::Values& v) return os << ToCString(v) << "(" << static_cast(v) << ')'; } -FindMode::Values GetFindModeValueImpl2() +template +std::optional GetFindModeValueImpl2(Variable variable) { - auto str = env::value(MIOPEN_FIND_MODE); + auto str = env::value(variable); if(str.empty()) - return FindMode::Values::Default_; + return std::nullopt; for(auto& c : str) c = toupper(static_cast(c)); if(str == "NORMAL") @@ -225,26 +229,34 @@ FindMode::Values GetFindModeValueImpl2() const auto val = static_cast(stoul(str)); if(FindMode::Values::Begin_ <= val && val < FindMode::Values::End_) return val; - MIOPEN_LOG_NQE("Wrong MIOPEN_FIND_MODE, using default."); - return FindMode::Values::Default_; + MIOPEN_LOG_NQE("Wrong " << variable.GetName() << ", using default."); + return std::nullopt; } -FindMode::Values GetFindModeValueImpl() +template +FindMode::Values GetFindModeValue(Variable variable, FindMode::Values defaultValue) { - auto rv = GetFindModeValueImpl2(); - MIOPEN_LOG_NQI("MIOPEN_FIND_MODE = " << rv); - return rv; -} - -FindMode::Values GetFindModeValue() -{ - static const FindMode::Values val = GetFindModeValueImpl(); + static const FindMode::Values val = [&]() { + auto rv = GetFindModeValueImpl2(variable).value_or(defaultValue); + MIOPEN_LOG_NQI(variable.GetName() << " = " << rv); + return rv; + }(); return val; } } // namespace -FindMode::FindMode() { value = GetFindModeValue(); } +FindMode::FindMode(solver::Primitive primitive) +{ + switch(primitive) + { + case solver::Primitive::Fusion: + value = GetFindModeValue(MIOPEN_FIND_MODE_FUSION, FindMode::Values::Fast); + break; + default: value = GetFindModeValue(MIOPEN_FIND_MODE, FindMode::Values::Default_); break; + } +} + std::ostream& operator<<(std::ostream& os, const FindMode& obj) { return os << obj.value; } static_assert(miopenConvolutionFindModeNormal == diff --git a/src/fused_api.cpp b/src/fused_api.cpp index 0dbd125710..6a4ac9a866 100644 --- a/src/fused_api.cpp +++ b/src/fused_api.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include diff --git a/src/fusion.cpp b/src/fusion.cpp index 24c4448335..5541409a67 100644 --- a/src/fusion.cpp +++ b/src/fusion.cpp @@ -390,13 +390,17 @@ std::string LogCmdBnormFusion(const miopenFusionPlanDescriptor_t fusePlanDesc, i if(bn_op != nullptr) { str += BnormArgsForMIOpenDriver(&bn_op->input_desc, + nullptr, + nullptr, + nullptr, + nullptr, bn_op->mode, nullptr, nullptr, nullptr, nullptr, miopen::debug::BatchNormDirection_t::ForwardInference, - false); + false); // having false allows safe handling of nullptrs } else { @@ -822,7 +826,8 @@ class FusionSolverFinder : public SolversFinderMixin& options) const override { - return solvers.SearchForAllSolutions(dynamic_cast(ctx), + const auto fusion_ctx = FusionContext(ctx); + return solvers.SearchForAllSolutions(fusion_ctx, problem, miopen::GetDb(ctx), invoke_ctx, @@ -865,9 +870,8 @@ FindFusion(const ExecutionContext& ctx, // fusion_ctx.use_dynamic_solutions_only = findMode.IsDynamicHybrid(fusion_ctx); // We need buffers for find, thus we lazily get them, possibly allocating. - auto fusion_ctx = FusionContext(ctx.GetStream()); return FindCore(invoke_params(), - fusion_ctx, + ctx, fusion_problem, FusionFindParameters{}, GetFusionSolverFinders(), @@ -983,7 +987,7 @@ miopenStatus_t FusionPlanDescriptor::Compile(Handle& handle) } { - FindMode findMode; + FindMode findMode(solver::Primitive::Fusion); auto sol = boost::optional{}; if(findMode.IsFast(fusion_problem) || findMode.IsHybrid(fusion_problem)) @@ -994,19 +998,35 @@ miopenStatus_t FusionPlanDescriptor::Compile(Handle& handle) if(fallback) { - bool found = false; + auto fallback_failed = true; + bool found = false; + GetAllFusionSolvers().Foreach([&](auto solver) { if(found || !solver.IsApplicable(ctx, fusion_problem)) return; - const auto id = solver::Id(solver.SolverDbId()); - sols.push_back({0, 0, id.Value(), miopenConvolutionAlgoDirect}); + const auto id = solver::Id(solver.SolverDbId()); + const auto wti = solver.GetWti(ctx, fusion_problem); + // Assume WTI == 1.0 (100%) is 10 ms. + // Return negative values as is, avoid DIV/0. + const auto time = wti <= 0.0f ? wti : (10.f / wti); + sols.push_back({time, 0, id.Value(), miopenConvolutionAlgoDirect}); + fallback_failed = false; }); + + if(fallback_failed) + { + MIOPEN_LOG_I("No supported fusion solvers found"); + return miopenStatusUnsupportedOp; + } } // override the normal find with immed mode with env var if(!sols.empty() && (!(findMode.IsHybrid(fusion_problem) && fallback))) - // || env::enabled(MIOPEN_DEBUG_FORCE_IMMED_MODE_FALLBACK) + // || env::enabled(MIOPEN_DEBUG_FORCE_IMMED_MODE_FALLBACK) + { + std::sort(sols.begin(), sols.end(), SolutionTimeComparator()); sol = sols.front(); + } // In Hybrid Find mode, we use Normal Find instead of Immediate fallback kernels. } @@ -1060,7 +1080,7 @@ miopenStatus_t FusionPlanDescriptor::Compile(Handle& handle) continue; } - handle.RegisterInvoker(*invoker, network_config, id.ToString(), AlgorithmName{"fusion"}); + handle.RegisterInvoker(*invoker, network_config, id.ToString()); invokers.push_back(std::move(*invoker)); MIOPEN_LOG_I2(miopen::ConvolutionAlgoToString(algorithm)); } @@ -1071,6 +1091,8 @@ miopenStatus_t FusionPlanDescriptor::Compile(Handle& handle) return miopenStatusUnsupportedOp; } + handle.SetAsFound1_0( + network_config, AlgorithmName{"fusion"}, find_results.front().GetSolver().ToString()); return miopenStatusSuccess; } @@ -1079,7 +1101,10 @@ FusionPlanDescriptor::Find(Handle& handle, const std::function& invoke_params, const std::optional& options) const { - return FindFusion(&handle, this, invoke_params, options); + auto ctx = ExecutionContext(&handle); + if(options) + ctx.do_search = options->exhaustive_search; + return FindFusion(ctx, this, invoke_params, options); } miopenStatus_t FusionPlanDescriptor::Execute(const Handle& handle, diff --git a/src/getitem.cpp b/src/getitem.cpp new file mode 100644 index 0000000000..c3b1b0c3bc --- /dev/null +++ b/src/getitem.cpp @@ -0,0 +1,106 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace miopen { + +std::size_t GetGetitemWorkspaceSize(Handle& handle, + uint32_t indexCount, + const TensorDescriptor* const* indexDescs) +{ + auto ctx = ExecutionContext{&handle}; + const auto problem = getitem::ProblemDescription{indexCount, indexDescs}; + + const auto algo = AlgorithmName{"GetitemBackward"}; + const auto solvers = solver::SolverContainer{}; + + auto pair_size_vector = solvers.GetWorkspaceSizes(ctx, problem, true); + + return pair_size_vector.empty() ? static_cast(0) : pair_size_vector.front().second; +} + +miopenStatus_t GetitemBackward(Handle& handle, + Data_t workspace, + size_t workspaceSizeInBytes, + const TensorDescriptor& dyDesc, + ConstData_t dy, + uint32_t indexCount, + const TensorDescriptor* const* indexDescs, + ConstData_t* indexs, + const TensorDescriptor& dxDesc, + Data_t dx, + const TensorDescriptor& errorDesc, + Data_t error, + uint32_t dimCount, + const int32_t* dims, + uint32_t sliceCount, + const int32_t* slices, + uint32_t offset) +{ + const auto problem = getitem::ProblemDescription{dyDesc, + indexCount, + indexDescs, + dxDesc, + errorDesc, + dimCount, + dims, + sliceCount, + slices, + offset}; + + const auto invoke_params = getitem::GetitemInvokeParams{workspace, + workspaceSizeInBytes, + dyDesc, + dy, + indexCount, + indexDescs, + indexs, + dxDesc, + dx, + errorDesc, + error, + dimCount, + dims, + sliceCount, + slices, + offset}; + + const auto algo = AlgorithmName{"GetitemBackward"}; + const auto solvers = solver::SolverContainer{}; + solvers.ExecutePrimitive(handle, problem, algo, invoke_params); + + return miopenStatusSuccess; +} + +} // namespace miopen diff --git a/src/getitem/problem_description.cpp b/src/getitem/problem_description.cpp new file mode 100644 index 0000000000..b8b32109d6 --- /dev/null +++ b/src/getitem/problem_description.cpp @@ -0,0 +1,69 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include +#include + +#include + +namespace miopen { + +namespace getitem { + +NetworkConfig ProblemDescription::MakeNetworkConfig() const +{ + auto dy_dims = dyDesc.GetLengths(); + auto input_dtype = dyDesc.GetType(); + auto error_dtype = errorDesc.GetType(); + + auto input_size = + std::accumulate(dy_dims.begin(), dy_dims.end(), 1ULL, std::multiplies()); + + std::ostringstream ss; + + ss << "getitembwd"; + ss << "input_size" << input_size; + ss << "input_dtype" << input_dtype; + ss << "error_dtype" << error_dtype; + ss << "indexCount" << indexCount; + + for(int i = 0; i < indexCount; ++i) + { + if(i == 0) + ss << "indexs_size"; + const auto& index_dims = (*indexDescs)[i].GetLengths(); + auto index_size = std::accumulate( + index_dims.begin(), index_dims.begin(), 1ULL, std::multiplies()); + ss << index_size << "_"; + } + + return NetworkConfig{ss.str()}; +} + +} // namespace getitem + +} // namespace miopen diff --git a/src/getitem_api.cpp b/src/getitem_api.cpp new file mode 100644 index 0000000000..094f44620f --- /dev/null +++ b/src/getitem_api.cpp @@ -0,0 +1,206 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#include +#include +#include +#include +#include + +static void LogCmdGetitem(const miopenTensorDescriptor_t dyDesc, + uint32_t indexCount, + const miopenTensorDescriptor_t* indexDescs, + const miopenTensorDescriptor_t dxDesc, + uint32_t dimCount, + const int32_t* dims, + uint32_t sliceCount, + const int32_t* slices, + uint32_t offset, + bool is_fwd) +{ + if(miopen::IsLoggingCmd()) + { + std::stringstream ss; + auto dtype = miopen::deref(dyDesc).GetType(); + if(dtype == miopenHalf) + { + ss << "getitemfp16"; + } + else if(dtype == miopenFloat) + { + ss << "getitemfp32"; + } + else if(dtype == miopenBFloat16) + { + ss << "getitemf16"; + } + + std::string dy_s; + auto dy_dims = miopen::deref(dyDesc).GetLengths(); + for(int i = 0; i < dy_dims.size(); i++) + { + dy_s += std::to_string(dy_dims[i]); + if(i != dy_dims.size() - 2) + dy_s += ","; + } + ss << " -doutput " << dy_s; + + for(int i = 0; i < indexCount; i++) + { + std::string index_s; + auto index_dims = miopen::deref(indexDescs[i]).GetLengths(); + for(int j = 0; j < index_dims.size(); j++) + { + index_s += std::to_string(index_dims[j]); + if(j != index_dims.size() - 2) + index_s += ","; + } + ss << " -index" << i + 1 << " " << index_s; + } + + std::string dx_s; + auto dx_dims = miopen::deref(dxDesc).GetLengths(); + + for(int i = 0; i < dx_dims.size(); i++) + { + dx_s += std::to_string(dx_dims[i]); + if(i != dx_dims.size() - 2) + dx_s += ","; + } + + ss << " -dx " << dx_s; + + std::string dims_s; + for(int i = 0; i < dimCount; i++) + { + dims_s += std::to_string(dims[i]); + if(i != dimCount - 2) + dims_s += ","; + } + ss << " -dims" << dims_s; + + std::string slices_s; + for(int i = 0; i < sliceCount; i++) + { + slices_s += std::to_string(slices[i]); + if(i != sliceCount - 2) + slices_s += ","; + } + ss << " -slice" << slices_s; + + ss << " -offset" << offset; + ss << " -F " << ((is_fwd) ? "1" : "2"); + + MIOPEN_LOG_DRIVER_CMD(ss.str()); + } +} + +extern "C" miopenStatus_t miopenGetGetitemWorkspaceSize(miopenHandle_t handle, + uint32_t indexCount, + const miopenTensorDescriptor_t* indexDescs, + size_t* sizeInBytes) +{ + MIOPEN_LOG_FUNCTION(handle, indexCount, indexDescs); + + return miopen::try_([&] { + std::vector indexDescsCast; + std::transform(indexDescs, + indexDescs + indexCount, + std::back_inserter(indexDescsCast), + [](const auto& indexDesc) { return &miopen::deref(indexDesc); }); + miopen::deref(sizeInBytes) = miopen::GetGetitemWorkspaceSize( + miopen::deref(handle), indexCount, indexDescsCast.data()); + }); +}; + +extern "C" miopenStatus_t miopenGetitemBackward(miopenHandle_t handle, + void* workspace, + size_t workspaceSizeInBytes, + const miopenTensorDescriptor_t dyDesc, + const void* dy, + uint32_t indexCount, + const miopenTensorDescriptor_t* indexDescs, + const void* const* indexs, + const miopenTensorDescriptor_t dxDesc, + void* dx, + const miopenTensorDescriptor_t errorDesc, + void* error, + uint32_t dimCount, + const int32_t* dims, + uint32_t sliceCount, + const int32_t* slices, + uint32_t offset) +{ + MIOPEN_LOG_FUNCTION(handle, + workspace, + workspaceSizeInBytes, + dyDesc, + dy, + indexCount, + indexDescs, + indexs, + dxDesc, + dx, + errorDesc, + error, + dimCount, + dims, + sliceCount, + slices, + offset); + + LogCmdGetitem( + dyDesc, indexCount, indexDescs, dxDesc, dimCount, dims, sliceCount, slices, offset, true); + return miopen::try_([&] { + std::vector indexsCast; + std::vector indexDescsCast; + std::transform(indexDescs, + indexDescs + indexCount, + std::back_inserter(indexDescsCast), + [](const auto& indexDesc) { return &miopen::deref(indexDesc); }); + std::transform(indexs, + indexs + indexCount, + std::back_inserter(indexsCast), + [](const void* index) { return DataCast(index); }); + + miopen::GetitemBackward(miopen::deref(handle), + DataCast(workspace), + workspaceSizeInBytes, + miopen::deref(dyDesc), + DataCast(dy), + indexCount, + indexDescsCast.data(), + indexsCast.data(), + miopen::deref(dxDesc), + DataCast(dx), + miopen::deref(errorDesc), + DataCast(error), + dimCount, + dims, + sliceCount, + slices, + offset); + }); +} diff --git a/src/glu.cpp b/src/glu.cpp new file mode 100644 index 0000000000..9d1181d32e --- /dev/null +++ b/src/glu.cpp @@ -0,0 +1,102 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include + +#include +#include +#include +#include +#include +#include + +namespace miopen { + +namespace glu { + +miopenStatus_t GLUForward(Handle& handle, + const TensorDescriptor& inputDesc, + ConstData_t input, + const TensorDescriptor& outputDesc, + Data_t output, + uint32_t dim) +{ + const auto problem = glu::ProblemDescription{inputDesc, outputDesc, dim}; + + const auto invoke_params = [&]() { + auto tmp = glu::FwdInvokeParams{}; + tmp.type = InvokeType::Run; + tmp.inputDesc = &inputDesc; + tmp.outputDesc = &outputDesc; + tmp.input = input; + tmp.output = output; + tmp.dim = dim; + return tmp; + }(); + + const auto algo = AlgorithmName{"GLUForward"}; + const auto solvers = solver::SolverContainer{}; + + solvers.ExecutePrimitive(handle, problem, algo, invoke_params); + + return miopenStatusSuccess; +} + +miopenStatus_t GLUBackward(Handle& handle, + const TensorDescriptor& inputDesc, + ConstData_t input, + const TensorDescriptor& outputGradDesc, + ConstData_t outputGrad, + const TensorDescriptor& inputGradDesc, + Data_t inputGrad, + uint32_t dim) +{ + const auto problem = glu::ProblemDescription{inputDesc, outputGradDesc, inputGradDesc, dim}; + + const auto invoke_params = [&]() { + auto tmp = glu::BwdInvokeParams{}; + tmp.type = InvokeType::Run; + tmp.inputDesc = &inputDesc; + tmp.input = input; + tmp.inputGradDesc = &inputGradDesc; + tmp.inputGrad = inputGrad; + tmp.outputGradDesc = &outputGradDesc; + tmp.outputGrad = outputGrad; + tmp.dim = dim; + return tmp; + }(); + + const auto algo = AlgorithmName{"GLUBackward"}; + const auto solvers = solver::SolverContainer{}; + + solvers.ExecutePrimitive(handle, problem, algo, invoke_params); + + return miopenStatusSuccess; +} + +} // namespace glu + +} // namespace miopen diff --git a/src/glu/problem_description.cpp b/src/glu/problem_description.cpp new file mode 100644 index 0000000000..53caaf1fc7 --- /dev/null +++ b/src/glu/problem_description.cpp @@ -0,0 +1,208 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include +#include + +#include + +namespace miopen { + +namespace glu { + +ProblemDescription::ProblemDescription(const TensorDescriptor& inputDesc_, + const TensorDescriptor& outputDesc_, + uint32_t dim_) + : direction(Direction::Forward), inputDesc(inputDesc_), outputDesc(outputDesc_), dim(dim_) +{ + if(inputDesc.GetNumDims() != outputDesc.GetNumDims()) + { + MIOPEN_THROW(miopenStatusBadParm, + "GLU::ProblemDescription: Number of dimensions between input and output " + "tensor do not match."); + } + + if(dim >= inputDesc.GetNumDims()) + { + MIOPEN_THROW(miopenStatusBadParm, "GLU::ProblemDescription: Dimension is out of range."); + } + + if(inputDesc.GetLengths()[dim] % 2 != 0) + { + MIOPEN_THROW(miopenStatusBadParm, + "GLU::ProblemDescription: The split dimension size of input tensor should " + "be divisible by 2."); + } + + for(auto i = 0; i < inputDesc.GetNumDims(); i++) + { + if(i == dim) + { + if(inputDesc.GetLengths()[i] / 2 != outputDesc.GetLengths()[i]) + { + MIOPEN_THROW(miopenStatusBadParm, + "GLU::ProblemDescription: Dimension sizes don't match between " + "input tensor and output tensor."); + } + } + else + { + if(inputDesc.GetLengths()[i] != outputDesc.GetLengths()[i]) + { + MIOPEN_THROW(miopenStatusBadParm, + "GLU::ProblemDescription: Dimension sizes don't match between " + "input tensor and output tensor."); + } + } + } + + if(!IsSameType()) + { + MIOPEN_THROW(miopenStatusBadParm, "GLU::ProblemDescription: Tensor types do not match."); + } +} + +ProblemDescription::ProblemDescription(const TensorDescriptor& inputDesc_, + const TensorDescriptor& outputGradDesc_, + const TensorDescriptor& inputGradDesc_, + uint32_t dim_) + : direction(Direction::Backward), + inputDesc(inputDesc_), + outputGradDesc(outputGradDesc_), + inputGradDesc(inputGradDesc_), + dim(dim_) +{ + if(inputDesc.GetNumDims() != inputGradDesc.GetNumDims() || + inputDesc.GetNumDims() != outputGradDesc.GetNumDims()) + { + MIOPEN_THROW(miopenStatusBadParm, + "GLU::ProblemDescription: Number of tensor dimensions between input and " + "output tensor do not match."); + } + + if(dim >= inputDesc.GetNumDims()) + { + MIOPEN_THROW(miopenStatusBadParm, "GLU::ProblemDescription: Dimension is out of range."); + } + + if(inputDesc.GetLengths()[dim] % 2 != 0) + { + MIOPEN_THROW(miopenStatusBadParm, + "GLU::ProblemDescription: The split dimension size of input tensor should " + "be divisible by 2."); + } + + for(auto i = 0; i < inputDesc.GetNumDims(); i++) + { + if(i == dim) + { + if(inputDesc.GetLengths()[i] / 2 != outputGradDesc.GetLengths()[i] || + inputDesc.GetLengths()[i] != inputGradDesc.GetLengths()[i]) + { + MIOPEN_THROW(miopenStatusBadParm, + "GLU::ProblemDescription: Dimension sizes don't match between " + "input tensor and output tensor."); + } + } + else + { + if(inputDesc.GetLengths()[i] != inputGradDesc.GetLengths()[i] || + inputDesc.GetLengths()[i] != outputGradDesc.GetLengths()[i]) + { + MIOPEN_THROW(miopenStatusBadParm, + "GLU::ProblemDescription: Dimension sizes don't match between " + "input tensor and output tensor."); + } + } + } + + if(!IsSameType()) + { + MIOPEN_THROW(miopenStatusBadParm, "GLU::ProblemDescription: Tensor types do not match."); + } +} + +bool ProblemDescription::IsSameType() const +{ + if(direction == Direction::Forward) + { + if(inputDesc.GetType() != outputDesc.GetType()) + { + return false; + } + } + else + { + if(inputDesc.GetType() != inputGradDesc.GetType() || + inputGradDesc.GetType() != outputGradDesc.GetType()) + { + return false; + } + } + + return true; +} + +bool ProblemDescription::IsAllContiguous() const +{ + if(direction == Direction::Forward) + { + if(!(inputDesc.IsContiguous() && outputDesc.IsContiguous())) + { + return false; + } + } + else + { + if(!(inputDesc.IsContiguous() && inputGradDesc.IsContiguous() && + outputGradDesc.IsContiguous())) + { + return false; + } + } + + return true; +} + +NetworkConfig ProblemDescription::MakeNetworkConfig() const +{ + auto input_numel = inputDesc.GetElementSize(); + auto io_dtype = miopen::GetDataType(inputDesc.GetType()); + + std::ostringstream ss; + + ss << "io_dtype" << io_dtype; + ss << "dim" << dim; + ss << "input_numel" << input_numel; + ss << IsAllContiguous(); + + return NetworkConfig{ss.str()}; +} + +} // namespace glu + +} // namespace miopen diff --git a/src/glu_api.cpp b/src/glu_api.cpp new file mode 100644 index 0000000000..856eeb6920 --- /dev/null +++ b/src/glu_api.cpp @@ -0,0 +1,74 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include +#include +#include +#include +#include + +extern "C" miopenStatus_t miopenGLUForward(miopenHandle_t handle, + const miopenTensorDescriptor_t inputDesc, + const void* input, + const miopenTensorDescriptor_t outputDesc, + void* output, + const uint32_t dim) +{ + MIOPEN_LOG_FUNCTION(handle, inputDesc, input, outputDesc, output, dim); + + return miopen::try_([&] { + miopen::glu::GLUForward(miopen::deref(handle), + miopen::deref(inputDesc), + DataCast(input), + miopen::deref(outputDesc), + DataCast(output), + dim); + }); +} + +extern "C" miopenStatus_t miopenGLUBackward(miopenHandle_t handle, + const miopenTensorDescriptor_t inputDesc, + const void* input, + const miopenTensorDescriptor_t outputGradDesc, + const void* outputGrad, + const miopenTensorDescriptor_t inputGradDesc, + void* inputGrad, + const uint32_t dim) +{ + MIOPEN_LOG_FUNCTION( + handle, inputDesc, input, outputGradDesc, outputGrad, inputGradDesc, inputGrad, dim); + return miopen::try_([&] { + miopen::glu::GLUBackward(miopen::deref(handle), + miopen::deref(inputDesc), + DataCast(input), + miopen::deref(outputGradDesc), + DataCast(outputGrad), + miopen::deref(inputGradDesc), + DataCast(inputGrad), + dim); + }); +} diff --git a/src/graphapi/conv_bias_res_add_activ_forward_executor.cpp b/src/graphapi/conv_bias_res_add_activ_forward_executor.cpp new file mode 100644 index 0000000000..36d798e4fb --- /dev/null +++ b/src/graphapi/conv_bias_res_add_activ_forward_executor.cpp @@ -0,0 +1,133 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#include +#include +#include +#include +#include +#include + +namespace miopen { + +namespace graphapi { + +namespace { +std::vector Convert(const std::vector& values) +{ + std::vector converted(values.size()); + std::transform(values.begin(), values.end(), converted.begin(), [](int64_t value) { + assert(value <= std::numeric_limits::max() && + value >= std::numeric_limits::min()); + return static_cast(value); + }); + + return converted; +} + +ConvolutionDescriptor Convert(const Convolution& conv, int groupCount) +{ + return {conv.getSpatialDims(), + conv.getMode(), + miopenPaddingMode_t::miopenPaddingDefault, + Convert(conv.getPrePaddings()), + Convert(conv.getFilterStrides()), + Convert(conv.getDilations()), + Convert(conv.getPostPaddings()), + groupCount}; +} +} // namespace + +ConvBiasResAddActivForwardExecutor::ConvBiasResAddActivForwardExecutor(const nlohmann::json& json) + : GraphPatternExecutor(), + mXTensor(json.at(JsonFields::XTensor)), + mWTensor(json.at(JsonFields::WTensor)), + mConvolution(json.at(JsonFields::Convolution)), + mGroupCount(json.at(JsonFields::GroupCount)), + mZTensor(json.at(JsonFields::ZTensor)), + mBiasTensor(json.at(JsonFields::BiasTensor)), + mYTensor(json.at(JsonFields::YTensor)), + mAlpha1(json.at(JsonFields::Alpha1)), + mAlpha2(json.at(JsonFields::Alpha2)), + mActivationAlpha(json.at(JsonFields::ActivationAlpha)) +{ +} + +void ConvBiasResAddActivForwardExecutor::execute(miopenHandle_t handle, const VariantPack& vpk) +{ + auto convDesc = Convert(mConvolution, mGroupCount); + + ActivationDescriptor activDesc{miopenActivationRELU, mActivationAlpha, 1.0, 1.0}; + + auto* xData = vpk.getDataPointer(mXTensor.getId()); + auto* wData = vpk.getDataPointer(mWTensor.getId()); + auto* zData = vpk.getDataPointer(mZTensor.getId()); + auto* biasData = vpk.getDataPointer(mBiasTensor.getId()); + auto* yData = vpk.getDataPointer(mYTensor.getId()); + + auto status = + ConvBiasActivFusion(miopen::deref(handle), + &mAlpha1, + mXTensor, + xData, + mWTensor, + wData, + convDesc, + miopenConvFwdAlgorithm_t::miopenConvolutionFwdAlgoImplicitGEMM, + nullptr, + 0, + &mAlpha2, + mZTensor, + zData, + mBiasTensor, + biasData, + activDesc, + mYTensor, + yData); + + MIOPEN_THROW_IF(status != miopenStatusSuccess, "execute failed"); +} + +nlohmann::json ConvBiasResAddActivForwardExecutor::getJson() +{ + return { + {GraphPatternExecutor::JsonFields::Name, name}, + {JsonFields::XTensor, mXTensor}, + {JsonFields::WTensor, mWTensor}, + {JsonFields::Convolution, mConvolution}, + {JsonFields::GroupCount, mGroupCount}, + {JsonFields::ZTensor, mZTensor}, + {JsonFields::BiasTensor, mBiasTensor}, + {JsonFields::YTensor, mYTensor}, + {JsonFields::Alpha1, mAlpha1}, + {JsonFields::Alpha2, mAlpha2}, + {JsonFields::ActivationAlpha, mActivationAlpha}, + }; +} + +} // namespace graphapi + +} // namespace miopen diff --git a/src/graphapi/convolution.cpp b/src/graphapi/convolution.cpp index 2279f793d7..e5ce03c7a5 100644 --- a/src/graphapi/convolution.cpp +++ b/src/graphapi/convolution.cpp @@ -26,6 +26,10 @@ #include #include #include +#include + +#include +#include namespace miopen { @@ -1020,6 +1024,74 @@ OperationConvolution& BackendOperationConvolutionBackwardFilterDescriptor::getOp return mOperation; } +void to_json(nlohmann::json& json, const Convolution& conv) +{ + static const std::map CompType2Str{ + {miopenHalf, "miopenHalf"}, + {miopenFloat, "miopenFloat"}, + {miopenInt32, "miopenInt32"}, + {miopenInt8, "miopenInt8"}, + {miopenBFloat16, "miopenBFloat16"}, + {miopenDouble, "miopenDouble"}, + {miopenFloat8, "miopenFloat8"}, + {miopenBFloat8, "miopenBFloat8"}, + {miopenInt64, "miopenInt64"}, + }; + + static const std::map Mode2Str{ + {miopenConvolution, "miopenConvolution"}, + {miopenTranspose, "miopenTranspose"}, + {miopenGroupConv, "miopenGroupConv"}, + {miopenDepthwise, "miopenDepthwise"}, + }; + + std::string sMode = Mode2Str.at(conv.mMode); + std::string sCompType = CompType2Str.at(conv.mCompType); + + json = nlohmann::json{ + {Convolution::JsonFields::SpatialDims, conv.mSpatialDims}, + {Convolution::JsonFields::Dilations, conv.mDilations}, + {Convolution::JsonFields::FilterStrides, conv.mFilterStrides}, + {Convolution::JsonFields::PrePaddings, conv.mPrePaddings}, + {Convolution::JsonFields::PostPaddings, conv.mPostPaddings}, + {Convolution::JsonFields::CompType, sCompType}, + {Convolution::JsonFields::Mode, sMode}, + }; +} + +void from_json(const nlohmann::json& json, Convolution& conv) +{ + json.at(Convolution::JsonFields::SpatialDims).get_to(conv.mSpatialDims); + json.at(Convolution::JsonFields::Dilations).get_to(conv.mDilations); + json.at(Convolution::JsonFields::FilterStrides).get_to(conv.mFilterStrides); + json.at(Convolution::JsonFields::PrePaddings).get_to(conv.mPrePaddings); + json.at(Convolution::JsonFields::PostPaddings).get_to(conv.mPostPaddings); + auto sCompType = json.at(Convolution::JsonFields::CompType).get(); + auto sMode = json.at(Convolution::JsonFields::Mode).get(); + + static const std::map Str2CompType{ + {"miopenHalf", miopenHalf}, + {"miopenFloat", miopenFloat}, + {"miopenInt32", miopenInt32}, + {"miopenInt8", miopenInt8}, + {"miopenBFloat16", miopenBFloat16}, + {"miopenDouble", miopenDouble}, + {"miopenFloat8", miopenFloat8}, + {"miopenBFloat8", miopenBFloat8}, + {"miopenInt64", miopenInt64}, + }; + + static const std::map Str2Mode{ + {"miopenConvolution", miopenConvolution}, + {"miopenTranspose", miopenTranspose}, + {"miopenGroupConv", miopenGroupConv}, + {"miopenDepthwise", miopenDepthwise}, + }; + + conv.mCompType = Str2CompType.at(sCompType); + conv.mMode = Str2Mode.at(sMode); +} + } // namespace graphapi } // namespace miopen diff --git a/src/graphapi/engine.cpp b/src/graphapi/engine.cpp index 8dd1674c8f..cc3ec9d275 100644 --- a/src/graphapi/engine.cpp +++ b/src/graphapi/engine.cpp @@ -25,8 +25,17 @@ *******************************************************************************/ #include +#include #include #include +#include +#include +#include + +#include +#include +#include +#include namespace miopen { @@ -34,9 +43,59 @@ namespace graphapi { GraphPatternExecutor::~GraphPatternExecutor() = default; -size_t GraphExecutorFind20::getWorkspaceSize() const +size_t GraphExecutorFind20::getWorkspaceSize() const { return mSolution.GetWorkspaceSize(); } + +nlohmann::json GraphExecutorFind20::getJson() { - return miopen::deref(mSolution).GetWorkspaceSize(); + std::map id2ArgumentMap{}; + + for(const auto& [tensorId, tensorInfo] : *mTensorInfoMap) + { + id2ArgumentMap.try_emplace(tensorId, tensorInfo.mEnumId); + } + + std::size_t size{0U}; + auto status = miopenGetSolutionSize(&mSolution, &size); + MIOPEN_THROW_IF(status != miopenStatusSuccess, + "Serialization size for Solution wasn't obtained"); + + std::vector serializedSolution(size); + status = miopenSaveSolution(&mSolution, reinterpret_cast(serializedSolution.data())); + MIOPEN_THROW_IF(status != miopenStatusSuccess, "Solution failed to be serialized"); + + std::string base64edSolution = base64Encode(serializedSolution); + + return { + {GraphPatternExecutor::JsonFields::Name, name}, + {GraphExecutorFind20::JsonFields::Solution, base64edSolution}, + {GraphExecutorFind20::JsonFields::Id2ArgumentMap, id2ArgumentMap}, + }; +} + +GraphExecutorFind20::GraphExecutorFind20(const nlohmann::json& json) +{ + auto base64edSolution = json.at(GraphExecutorFind20::JsonFields::Solution).get(); + auto serializedSolution = base64Decode(base64edSolution); + + miopenSolution_t solutionDescriptor; + auto status = miopenLoadSolution(&solutionDescriptor, + reinterpret_cast(serializedSolution.data()), + serializedSolution.size()); + MIOPEN_THROW_IF(status != miopenStatusSuccess, "Failed to deserialize Solution"); + + // Ensure miopenDestroySolution() will be called + scope_exit finallyForSolution([=]() { miopenDestroySolution(solutionDescriptor); }); + + mSolution = std::move(deref(solutionDescriptor)); + + auto id2ArgumentMap = json.at(GraphExecutorFind20::JsonFields::Id2ArgumentMap) + .get>(); + + mTensorInfoMap = std::make_shared(); + for(const auto [tensorId, argumentId] : id2ArgumentMap) + { + mTensorInfoMap->try_emplace(tensorId, argumentId, nullptr); + } } void GraphExecutorFind20::execute(miopenHandle_t handle, const VariantPack& vpk) @@ -79,7 +138,7 @@ void GraphExecutorFind20::execute(miopenHandle_t handle, const VariantPack& vpk) } auto s = miopenRunSolution(handle, - mSolution, + &mSolution, tens_args.size(), tens_args.data(), vpk.getWorkspace(), @@ -92,6 +151,39 @@ void GraphExecutorFind20::execute(miopenHandle_t handle, const VariantPack& vpk) } } +void to_json(nlohmann::json& json, const Engine& engine) +{ + MIOPEN_THROW_IF(!engine.mExecutor, "Cannot serialize an Engine without an Executor"); + + json = nlohmann::json{ + {Engine::JsonFields::Executor, engine.mExecutor->getJson()}, + {Engine::JsonFields::GlobalIndex, engine.mGlobalIndex}, + {Engine::JsonFields::SmCount, engine.mSmCount}, + }; +} + +void from_json(const nlohmann::json& json, Engine& engine) +{ + static const std::map< + std::string, + std::function(const nlohmann::json& json)>> + name2Maker{ + {GraphExecutorFind20::name, + std::make_shared}, + {ConvBiasResAddActivForwardExecutor::name, + std::make_shared}, + }; + + auto jExecutor = json.at(Engine::JsonFields::Executor); + auto executorName = jExecutor.at(GraphPatternExecutor::JsonFields::Name).get(); + auto maker = name2Maker.at(executorName); + + engine.mExecutor = maker(jExecutor); + engine.mGraph = nullptr; + json.at(Engine::JsonFields::GlobalIndex).get_to(engine.mGlobalIndex); + json.at(Engine::JsonFields::SmCount).get_to(engine.mSmCount); +} + EngineBuilder& EngineBuilder::setGraph(OpGraph* g) { assert(g); diff --git a/src/graphapi/execution_plan.cpp b/src/graphapi/execution_plan.cpp index ec45fcf8c7..bf332786d0 100644 --- a/src/graphapi/execution_plan.cpp +++ b/src/graphapi/execution_plan.cpp @@ -32,8 +32,11 @@ namespace graphapi { std::string ExecutionPlan::getJsonRepresentation() const { - /// \todo Implement ExecutionPlan::getJsonRepresentation --Sergei May, 2024 - return {}; + std::string result; + nlohmann::json json = *this; + // We don't support text/json as we have binary data inside + nlohmann::json::to_msgpack(json, result); + return result; } ExecutionPlanBuilder& ExecutionPlanBuilder::setHandle(miopenHandle_t handle) & @@ -70,14 +73,15 @@ ExecutionPlanBuilder& ExecutionPlanBuilder::setIntermediateIds(std::vector #include #include +#include #include #include #include +#include +#include +#include namespace miopen { namespace graphapi { GraphPatternMatcher::~GraphPatternMatcher() = default; +class ConvBiasResAddActive_Fwd_Pattern : public GraphPatternMatcher +{ + struct OperationPointwiseWithOneVirtualInput + { + Tensor* concreteTensor; + Tensor* virtualTensor; + float concreteAlpha; + float virtualAlpha; + + OperationPointwiseWithOneVirtualInput(const OperationPointwise* pointwise) + { + auto convertToFloat = [](auto&& arg) { return static_cast(arg); }; + if(pointwise->getX()->isVirtual()) + { + virtualTensor = pointwise->getX(); + virtualAlpha = std::visit(convertToFloat, pointwise->getAlpha1()); + + concreteTensor = pointwise->getB(); + concreteAlpha = std::visit(convertToFloat, pointwise->getAlpha2()); + } + else + { + concreteTensor = pointwise->getX(); + concreteAlpha = std::visit(convertToFloat, pointwise->getAlpha1()); + + virtualTensor = pointwise->getB(); + virtualAlpha = std::visit(convertToFloat, pointwise->getAlpha2()); + } + } + }; + + static const OpGraph& getPatternGraph() + { + static auto graph_gen = + PatternGraphGenerator::Make({{"OP_CONVOLUTION_FORWARD", {"X", "W"}, {"T_C_0"}}, + {"OP_POINTWISE:ADD", {"T_C_0", "Z"}, {"T_A_0"}}, + {"OP_POINTWISE:ADD", {"T_A_0", "BIAS"}, {"T_A_1"}}, + {"OP_POINTWISE:RELU_FWD", {"T_A_1"}, {"Y"}}}); + + return graph_gen->graph(); + } + + static bool isBiasNode(OperationPointwise* addNode) + { + OperationPointwiseWithOneVirtualInput add(addNode); + + auto& lengthsToCheck = add.concreteTensor->GetLengths(); + + return std::count_if(lengthsToCheck.cbegin(), lengthsToCheck.cend(), [](std::size_t value) { + return value > size_t{1}; + }) <= 1; + } + + static bool hasBiasNode(const OpGraph& graph) + { + auto* conv = dynamic_cast( + graph.findOutNeighByName(graph.getSourceNode(), "OP_CONVOLUTION_FORWARD")); + + auto* add1 = + dynamic_cast(graph.findOutNeighByName(conv, "OP_POINTWISE:ADD")); + + auto* add2 = + dynamic_cast(graph.findOutNeighByName(add1, "OP_POINTWISE:ADD")); + + return isBiasNode(add1) || isBiasNode(add2); + } + +public: + static std::unique_ptr Make() + { + return std::make_unique(); + } + + std::string_view name() const final + { + static const std::string_view n{"convbiasresaddactivation_fwd"}; + return n; + } + + bool matches(const OpGraph* graph_ptr) const final + { + assert(graph_ptr); + + if(!isIsomorphic(*graph_ptr, getPatternGraph())) + { + return false; + } + + return hasBiasNode(*graph_ptr); + } + + std::vector getEngines(OpGraph* graph_ptr) const override + { + assert(graph_ptr); + assert(matches(graph_ptr)); + auto& graph = *graph_ptr; + + auto* conv = dynamic_cast( + graph.findOutNeighByName(graph.getSourceNode(), "OP_CONVOLUTION_FORWARD")); + + std::size_t in_c = conv->getX()->GetLengths()[1]; + std::size_t wei_c = conv->getW()->GetLengths()[1]; + + if(wei_c == std::size_t{0}) + { + MIOPEN_THROW( + miopenStatusBadParm, + "invalid weight tensor provided for graph matching ConvBiasResAddActive pattern"); + } + else if(in_c % wei_c != std::size_t{0}) + { + MIOPEN_THROW(miopenStatusBadParm, + "invalid group count from input and weight tensor for graph matching " + "ConvBiasResAddActive pattern"); + } + + int groupCount = in_c / wei_c; + + auto* add1 = + dynamic_cast(graph.findOutNeighByName(conv, "OP_POINTWISE:ADD")); + + auto* add2 = + dynamic_cast(graph.findOutNeighByName(add1, "OP_POINTWISE:ADD")); + + auto* activ = dynamic_cast( + graph.findOutNeighByName(add2, "OP_POINTWISE:RELU_FWD")); + + auto [addTemp, biasTemp] = + isBiasNode(add1) ? std::make_pair(add2, add1) : std::make_pair(add1, add2); + + OperationPointwiseWithOneVirtualInput add(addTemp); + OperationPointwiseWithOneVirtualInput bias(biasTemp); + + // The virtual tensor for the add is the result of the convolution, and combining the + // alpha1's allow for users to specify the alpha on either, or both nodes, and have it be + // correct. + float alpha1 = conv->getAlpha() * add.virtualAlpha; + float alpha2 = add.concreteAlpha; + + std::shared_ptr exec = + std::make_shared( + conv->getX(), + conv->getW(), + conv->getConvolution(), + groupCount, + add.concreteTensor, + bias.concreteTensor, + activ->getY(), + alpha1, + alpha2, + std::visit([](auto&& arg) { return static_cast(arg); }, activ->getAlpha1())); + return {EngineBuilder().setGraph(graph_ptr).setExecutor(exec).setGlobalIndex(0).build()}; + } +}; + class MHA_Fwd_F8_Pattern : public GraphPatternMatcher { static const OpGraph& getPatternGraph() @@ -258,7 +417,6 @@ class MHA_Fwd_F8_Pattern : public GraphPatternMatcher std::vector getEngines(OpGraph* graph_ptr) const override { - assert(graph_ptr); assert(matches(graph_ptr)); auto& graph = *graph_ptr; @@ -275,6 +433,9 @@ class MHA_Fwd_F8_Pattern : public GraphPatternMatcher auto s = miopenCreateMhaProblem(&mha_prob, &mha_desc, miopenProblemDirectionForward); MIOPEN_THROW_IF(s != miopenStatusSuccess, "failed while creating problem for mha fwd"); + // Ensure miopenDestroyProblem() will be called even if an exception occurs + scope_exit finallyForProblem([=]() { miopenDestroyProblem(mha_prob); }); + for(auto& [k, v] : *tensor_map) { s = miopenSetProblemTensorDescriptor(mha_prob, v.mEnumId, v.mGraphTensor); @@ -290,12 +451,22 @@ class MHA_Fwd_F8_Pattern : public GraphPatternMatcher solutions.resize(num_found); + // Ensure miopenDestroySolution() will be called even if an exception occurs + scope_exit finallyForSolutions([&]() { + for(miopenSolution_t sol : solutions) + { + miopenDestroySolution(sol); + } + }); + std::vector engines; + engines.reserve(num_found); size_t i = 0; - for(const auto& sol : solutions) + for(miopenSolution_t sol : solutions) { - std::shared_ptr exec = GraphExecutorFind20::make(sol, tensor_map); + std::shared_ptr exec = + std::make_shared(std::move(deref(sol)), tensor_map); engines.emplace_back( EngineBuilder().setGraph(graph_ptr).setExecutor(exec).setGlobalIndex(i).build()); @@ -306,16 +477,563 @@ class MHA_Fwd_F8_Pattern : public GraphPatternMatcher } }; -/* -class FwdConvResAddBiasActPattern : public GraphPattern +class MHA_Bwd_F8_Pattern : public GraphPatternMatcher { + static const OpGraph& getPatternGraph() + { + static auto graph_gen = PatternGraphGenerator::Make({ + {"OP_RESHAPE", {"K"}, {"KT"}}, + {"OP_MATMUL", {"Q", "KT"}, {"MM0"}}, + {"OP_POINTWISE:IDENTITY", {"MM0"}, {"PWS0"}}, + // {"OP_POINTWISE:MUL", {"MM0", "ATTN_S"}, {"PWS0"}}, + // replacing the MUL node above with IDENTITY node below for now + // because Find 2.0 mha descriptor expects attention scale as a scalar + // parameter on host side + {"OP_POINTWISE:MUL", {"PWS0", "DSCL_Q"}, {"PWS1"}}, + {"OP_POINTWISE:MUL", {"PWS1", "DSCL_K"}, {"PWS2"}}, + {"OP_POINTWISE:SUB", {"PWS2", "M"}, {"SUB0"}}, + {"OP_POINTWISE:EXP", {"SUB0"}, {"EXP0"}}, + {"OP_POINTWISE:MUL", {"EXP0", "ZINV"}, {"MULT0"}}, + {"OP_RNG", {"SEED", "OFFSET"}, {"RND"}}, + {"OP_POINTWISE:MUL", {"MULT0", "RND"}, {"MULT1"}}, + {"OP_POINTWISE:MUL", {"MULT1", "I_PROB"}, {"PWS3"}}, + {"OP_POINTWISE:MUL", {"PWS3", "SCL_S"}, {"PWS4"}}, + {"OP_RESHAPE", {"PWS4"}, {"PWS4T"}}, + {"OP_MATMUL", {"PWS4T", "DO"}, {"MM1"}}, + {"OP_POINTWISE:MUL", {"MM1", "DSCL_S"}, {"PWS5"}}, + {"OP_POINTWISE:MUL", {"PWS5", "DSCL_DO"}, {"PWS6"}}, + {"OP_REDUCTION:MAX", {"PWS6"}, {"AMAX_DV"}}, + {"OP_POINTWISE:MUL", {"PWS6", "SCL_DV"}, {"DV"}}, + + {"OP_RESHAPE", {"V"}, {"VT"}}, + {"OP_MATMUL", {"DO", "VT"}, {"MM2"}}, + {"OP_POINTWISE:MUL", {"MM2", "DSCL_DO"}, {"PWS7"}}, + {"OP_POINTWISE:MUL", {"PWS7", "DSCL_V"}, {"PWS8"}}, + {"OP_POINTWISE:MUL", {"PWS8", "RND"}, {"PWS9"}}, + {"OP_POINTWISE:MUL", {"PWS9", "PROB"}, {"PWS10"}}, + + {"OP_POINTWISE:MUL", {"DO", "DSCL_DO"}, {"PWS11"}}, + {"OP_POINTWISE:MUL", {"O", "DSCL_O"}, {"PWS12"}}, + {"OP_POINTWISE:MUL", {"PWS11", "PWS12"}, {"MULT2"}}, + {"OP_POINTWISE:MUL", {"MULT2", "PROB"}, {"PWS13"}}, + {"OP_REDUCTION:ADD", {"PWS13"}, {"SUM0"}}, + + {"OP_POINTWISE:SUB", {"PWS10", "SUM0"}, {"SUB1"}}, + {"OP_POINTWISE:IDENTITY", {"SUB1"}, {"PWS14"}}, + // {"OP_POINTWISE:MUL", {"SUB1", "ATTN_S"}, {"PWS0"}}, + // replacing the MUL node above with IDENTITY node below for now + // because Find 2.0 mha descriptor expects attention scale as a scalar + // parameter on host side + {"OP_POINTWISE:MUL", {"PWS14", "PWS3"}, {"MULT3"}}, + {"OP_REDUCTION:MAX", {"MULT3"}, {"AMAX_DS"}}, + {"OP_POINTWISE:MUL", {"MULT3", "SCL_DS"}, {"PWS15"}}, + + {"OP_MATMUL", {"PWS15", "K"}, {"MM3"}}, + {"OP_POINTWISE:MUL", {"MM3", "DSCL_DS"}, {"PWS16"}}, + {"OP_POINTWISE:MUL", {"PWS16", "DSCL_K"}, {"PWS17"}}, + {"OP_REDUCTION:MAX", {"PWS17"}, {"AMAX_DQ"}}, + {"OP_POINTWISE:MUL", {"PWS17", "SCL_DQ"}, {"DQ"}}, + + {"OP_RESHAPE", {"PWS15"}, {"PWS15T"}}, + {"OP_MATMUL", {"PWS15T", "Q"}, {"MM4"}}, + {"OP_POINTWISE:MUL", {"MM4", "DSCL_DS"}, {"PWS18"}}, + {"OP_POINTWISE:MUL", {"PWS18", "DSCL_Q"}, {"PWS19"}}, + {"OP_REDUCTION:MAX", {"PWS19"}, {"AMAX_DK"}}, + {"OP_POINTWISE:MUL", {"PWS19", "SCL_DK"}, {"DK"}}, + + }); + + return graph_gen->graph(); + } + + std::shared_ptr extractFind20Tensors(const OpGraph& graph, + float* attnScale) const + { + assert(attnScale); + + auto tensorMap = std::make_shared(); + + auto addMapping = [&](miopenTensorArgumentId_t enumId, Tensor* tensPtr) { + assert(tensPtr); + assert(enumId != miopenTensorArgumentIdInvalid); + tensorMap->try_emplace(tensPtr->getId(), TensorInfo(enumId, tensPtr)); + }; + + const auto& starts = graph.getOutEdges(graph.getSourceNode()); + + // Find top left and center parts by `transpose` nodes + + auto isTranspose = [](const OpNode::Edge& edge) -> bool { + auto [node, tensor] = edge; + return node->signName() == "OP_RESHAPE"; + }; + + auto leftOrCenterStartIt1 = std::find_if(starts.cbegin(), starts.cend(), isTranspose); + assert(leftOrCenterStartIt1 != starts.cend()); + auto leftOrCenterStartIt2 = + std::find_if(leftOrCenterStartIt1 + 1, starts.cend(), isTranspose); + assert(leftOrCenterStartIt2 != starts.cend()); + + // Tell left part from center one by `identity` node + + auto* leftOrCenterMatmulPtr = + graph.findOutNeighByName(leftOrCenterStartIt1->first, "OP_MATMUL"); + assert(leftOrCenterMatmulPtr); + + auto* leftIdentityPtr = + graph.findOutNeighByName(leftOrCenterMatmulPtr, "OP_POINTWISE:IDENTITY"); + + OpNode* leftHead = nullptr; + OpNode* centerHead = nullptr; + + if(leftIdentityPtr != nullptr) + { + leftHead = leftOrCenterStartIt1->first; + centerHead = leftOrCenterStartIt2->first; + } + else + { + leftHead = leftOrCenterStartIt2->first; + centerHead = leftOrCenterStartIt1->first; + } + + // Walk the left part + + OpNode* currentNode = leftHead; + auto& leftReshape = dynamic_cast(*currentNode); + addMapping(miopenTensorMhaK, leftReshape.getX()); + Tensor* prevOutputTensor = leftReshape.getY(); + + currentNode = graph.findOutNeighByName(currentNode, "OP_MATMUL"); + assert(currentNode); + auto& leftMatmul0 = dynamic_cast(*currentNode); + if(leftMatmul0.getA() != prevOutputTensor) + { + addMapping(miopenTensorMhaQ, leftMatmul0.getA()); + } + else + { + addMapping(miopenTensorMhaQ, leftMatmul0.getB()); + } + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:IDENTITY"); + assert(currentNode); + auto& leftIdentity = dynamic_cast(*currentNode); + *attnScale = std::get(leftIdentity.getAlpha1()); + prevOutputTensor = leftIdentity.getY(); + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(currentNode); + auto& leftMul0 = dynamic_cast(*currentNode); + if(leftMul0.getX() != prevOutputTensor) + { + addMapping(miopenTensorMhaDescaleQ, leftMul0.getX()); + } + else + { + addMapping(miopenTensorMhaDescaleQ, leftMul0.getB()); + } + prevOutputTensor = leftMul0.getY(); + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(currentNode); + auto& leftMul1 = dynamic_cast(*currentNode); + if(leftMul1.getX() != prevOutputTensor) + { + addMapping(miopenTensorMhaDescaleK, leftMul1.getX()); + } + else + { + addMapping(miopenTensorMhaDescaleK, leftMul1.getB()); + } + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:SUB"); + assert(currentNode); + addMapping(miopenTensorMhaM, dynamic_cast(*currentNode).getB()); + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:EXP"); + assert(currentNode); + prevOutputTensor = dynamic_cast(*currentNode).getY(); + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(currentNode); + auto& leftMul2 = dynamic_cast(*currentNode); + if(leftMul2.getX() != prevOutputTensor) + { + addMapping(miopenTensorMhaZInv, leftMul2.getX()); + } + else + { + addMapping(miopenTensorMhaZInv, leftMul2.getB()); + } + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(currentNode); + + // Stop at diversion point and remember it + + OpNode* leftDiversionPoint0 = graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(leftDiversionPoint0); + + // Take a look at RNG node + + currentNode = graph.findInNeighByName(currentNode, "OP_RNG"); + assert(currentNode); + auto& rng = dynamic_cast(*currentNode); + addMapping(miopenTensorMhaDropoutSeed, std::get(rng.getSeed())); + addMapping(miopenTensorMhaDropoutOffset, rng.getOffset()); + + // Walk the center part + + currentNode = centerHead; + auto& centerReshape = dynamic_cast(*currentNode); + addMapping(miopenTensorMhaV, centerReshape.getX()); + prevOutputTensor = centerReshape.getY(); + + currentNode = graph.findOutNeighByName(currentNode, "OP_MATMUL"); + assert(currentNode); + auto& centerMatmul0 = dynamic_cast(*currentNode); + int64_t doId = 0; // save id of DO tensor + if(centerMatmul0.getA() != prevOutputTensor) + { + addMapping(miopenTensorMhaDO, centerMatmul0.getA()); + doId = centerMatmul0.getA()->getId(); + } + else + { + addMapping(miopenTensorMhaDO, centerMatmul0.getB()); + doId = centerMatmul0.getB()->getId(); + } + prevOutputTensor = centerMatmul0.getC(); + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(currentNode); + auto& centerMul0 = dynamic_cast(*currentNode); + int64_t descaleDoId = 0; // save id of DescaleDO tensor + if(centerMul0.getX() != prevOutputTensor) + { + addMapping(miopenTensorMhaDescaleDO, centerMul0.getX()); + descaleDoId = centerMul0.getX()->getId(); + } + else + { + addMapping(miopenTensorMhaDescaleDO, centerMul0.getB()); + descaleDoId = centerMul0.getB()->getId(); + } + prevOutputTensor = centerMul0.getY(); + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(currentNode); + auto& centerMul1 = dynamic_cast(*currentNode); + if(centerMul1.getX() != prevOutputTensor) + { + addMapping(miopenTensorMhaDescaleV, centerMul1.getX()); + } + else + { + addMapping(miopenTensorMhaDescaleV, centerMul1.getB()); + } + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(currentNode); + prevOutputTensor = dynamic_cast(*currentNode).getY(); + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(currentNode); + auto& centerMul3 = dynamic_cast(*currentNode); + if(centerMul3.getX() != prevOutputTensor) + { + addMapping(miopenTensorMhaDropoutProbability, centerMul3.getX()); + } + else + { + addMapping(miopenTensorMhaDropoutProbability, centerMul3.getB()); + } + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:SUB"); + assert(currentNode); + + // save the tail of the right part + OpNode* rightTail = graph.findInNeighByName(currentNode, "OP_REDUCTION:ADD"); + assert(rightTail); + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:IDENTITY"); + assert(currentNode); + + // save the neighbor of the left part's diversion point + // it also happened to be the first diversion point of the center part + OpNode* centerDiversionPoint0 = graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(centerDiversionPoint0); + + currentNode = graph.findOutNeighByName(centerDiversionPoint0, "OP_REDUCTION:MAX"); + assert(currentNode); + addMapping(miopenTensorMhaAmaxDS, dynamic_cast(*currentNode).getY()); + + currentNode = graph.findOutNeighByName(centerDiversionPoint0, "OP_POINTWISE:MUL"); + assert(currentNode); + auto& centerMul5 = dynamic_cast(*currentNode); + if(centerMul5.getX() != dynamic_cast(*centerDiversionPoint0).getY()) + { + addMapping(miopenTensorMhaScaleDS, centerMul5.getX()); + } + else + { + addMapping(miopenTensorMhaScaleDS, centerMul5.getB()); + } + + // save the head of the right bottom part + OpNode* rightBottomHead = graph.findOutNeighByName(currentNode, "OP_RESHAPE"); + assert(rightBottomHead); + + currentNode = graph.findOutNeighByName(currentNode, "OP_MATMUL"); + assert(currentNode); + prevOutputTensor = dynamic_cast(*currentNode).getC(); + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(currentNode); + auto& centerMul6 = dynamic_cast(*currentNode); + if(centerMul6.getX() != prevOutputTensor) + { + addMapping(miopenTensorMhaDescaleDS, centerMul6.getX()); + } + else + { + addMapping(miopenTensorMhaDescaleDS, centerMul6.getB()); + } + + // save the second center part's diversion point + OpNode* centerDiversionPoint1 = graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(centerDiversionPoint1); + + currentNode = graph.findOutNeighByName(centerDiversionPoint1, "OP_REDUCTION:MAX"); + assert(currentNode); + addMapping(miopenTensorMhaAmaxDQ, dynamic_cast(*currentNode).getY()); + + currentNode = graph.findOutNeighByName(centerDiversionPoint1, "OP_POINTWISE:MUL"); + assert(currentNode); + auto& centerMul8 = dynamic_cast(*currentNode); + if(centerMul8.getX() != dynamic_cast(*centerDiversionPoint1).getY()) + { + addMapping(miopenTensorMhaScaleDQ, centerMul8.getX()); + } + else + { + addMapping(miopenTensorMhaScaleDQ, centerMul8.getB()); + } + addMapping(miopenTensorMhaDQ, centerMul8.getY()); + + // Walk the right bottom part + + currentNode = graph.findOutNeighByName(rightBottomHead, "OP_MATMUL"); + assert(currentNode); + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(currentNode); + + // save right bottom part's diversion point + OpNode* rightBottomDiversionPoint = + graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(rightBottomDiversionPoint); + + currentNode = graph.findOutNeighByName(rightBottomDiversionPoint, "OP_REDUCTION:MAX"); + assert(currentNode); + addMapping(miopenTensorMhaAmaxDK, dynamic_cast(*currentNode).getY()); + + currentNode = graph.findOutNeighByName(rightBottomDiversionPoint, "OP_POINTWISE:MUL"); + assert(currentNode); + auto& rightBottomMul2 = dynamic_cast(*currentNode); + if(rightBottomMul2.getX() != + dynamic_cast(*rightBottomDiversionPoint).getY()) + { + addMapping(miopenTensorMhaScaleDK, rightBottomMul2.getX()); + } + else + { + addMapping(miopenTensorMhaScaleDK, rightBottomMul2.getB()); + } + addMapping(miopenTensorMhaDK, rightBottomMul2.getY()); + + // Walk the right part backwards + + currentNode = graph.findInNeighByName(rightTail, "OP_POINTWISE:MUL"); + assert(currentNode); + + currentNode = graph.findInNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(currentNode); + + // tell right part's heads from each other by ids of + // DO and DescaleDO tensors which were saved above + const auto& rightHeads = graph.getInEdges(currentNode); + auto headWithO = std::find_if( + rightHeads.cbegin(), rightHeads.cend(), [=](const OpNode::Edge& edge) -> bool { + if(edge.first->signName() == "OP_POINTWISE:MUL") + { + auto& operation = dynamic_cast(*edge.first); + auto xId = operation.getX()->getId(); + auto bId = operation.getB()->getId(); + return xId != doId && xId != descaleDoId && bId != doId && bId != descaleDoId; + } + else + { + return false; + } + }); + assert(headWithO != rightHeads.cend()); + auto& rightMulWithO = dynamic_cast(*headWithO->first); + + // we cannot distinguish O from DescaleO but it doesn't matter + addMapping(miopenTensorMhaO, rightMulWithO.getX()); + addMapping(miopenTensorMhaDescaleO, rightMulWithO.getB()); + + // Continue walking the remaining left part after the first diversion point + + // Tell between pointwise:mul descendants + const auto& leftDivPointOutEdges = graph.getOutEdges(leftDiversionPoint0); + auto remLeftPtHeadIt = std::find_if(leftDivPointOutEdges.cbegin(), + leftDivPointOutEdges.cend(), + [=](const OpNode::Edge& edge) -> bool { + return edge.first != centerDiversionPoint0 && + edge.first->signName() == "OP_POINTWISE:MUL"; + }); + assert(remLeftPtHeadIt != leftDivPointOutEdges.cend()); + + currentNode = remLeftPtHeadIt->first; + prevOutputTensor = dynamic_cast(*leftDiversionPoint0).getY(); + auto& leftMul5 = dynamic_cast(*currentNode); + if(leftMul5.getX() != prevOutputTensor) + { + addMapping(miopenTensorMhaScaleS, leftMul5.getX()); + } + else + { + addMapping(miopenTensorMhaScaleS, leftMul5.getB()); + } + + currentNode = graph.findOutNeighByName(currentNode, "OP_RESHAPE"); + assert(currentNode); + + currentNode = graph.findOutNeighByName(currentNode, "OP_MATMUL"); + assert(currentNode); + prevOutputTensor = dynamic_cast(*currentNode).getC(); + + currentNode = graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(currentNode); + auto& leftMul6 = dynamic_cast(*currentNode); + if(leftMul6.getX() != prevOutputTensor) + { + addMapping(miopenTensorMhaDescaleS, leftMul6.getX()); + } + else + { + addMapping(miopenTensorMhaDescaleS, leftMul6.getB()); + } + + OpNode* leftDiversionPoint1 = graph.findOutNeighByName(currentNode, "OP_POINTWISE:MUL"); + assert(leftDiversionPoint1); + prevOutputTensor = dynamic_cast(*leftDiversionPoint1).getY(); + + currentNode = graph.findOutNeighByName(leftDiversionPoint1, "OP_REDUCTION:MAX"); + assert(currentNode); + addMapping(miopenTensorMhaAmaxDV, dynamic_cast(*currentNode).getY()); + + currentNode = graph.findOutNeighByName(leftDiversionPoint1, "OP_POINTWISE:MUL"); + assert(currentNode); + auto& leftMul7 = dynamic_cast(*currentNode); + if(leftMul7.getX() != prevOutputTensor) + { + addMapping(miopenTensorMhaScaleDV, leftMul7.getX()); + } + else + { + addMapping(miopenTensorMhaScaleDV, leftMul7.getB()); + } + addMapping(miopenTensorMhaDV, leftMul7.getY()); + + return tensorMap; + } + public: - static std::unique_ptr Make() + static std::unique_ptr Make() + { + return std::make_unique(); + } + + std::string_view name() const final { - return std::make_unique(); + static const char* n = "mha_bwd_f8"; + return n; + } + + bool matches(const OpGraph* graph_ptr) const final + { + assert(graph_ptr); + return isIsomorphic(*graph_ptr, getPatternGraph()); + } + + std::vector getEngines(OpGraph* graphPtr) const override + { + assert(graphPtr); + assert(matches(graphPtr)); + auto& graph = *graphPtr; + + MhaDescriptor mhaDesc; + + float attnScale = std::numeric_limits::quiet_NaN(); + std::shared_ptr tensorMap = extractFind20Tensors(graph, &attnScale); + assert(attnScale != std::numeric_limits::quiet_NaN()); + + mhaDesc.SetParams(attnScale); + + miopenProblem_t mhaProblem; + + auto s = miopenCreateMhaProblem(&mhaProblem, &mhaDesc, miopenProblemDirectionBackward); + MIOPEN_THROW_IF(s != miopenStatusSuccess, "failed while creating problem for mha bwd"); + + // Ensure miopenDestroyProblem() will be called even if an exception occurs + scope_exit finallyForProblem([=]() { miopenDestroyProblem(mhaProblem); }); + + for(auto& [k, v] : *tensorMap) + { + s = miopenSetProblemTensorDescriptor(mhaProblem, v.mEnumId, v.mGraphTensor); + MIOPEN_THROW_IF(s != miopenStatusSuccess, + "failed while setting tensor descriptor for mha bwd"); + } + + std::vector solutions(10); + size_t numFound = 0; + s = miopenFindSolutions( + graph.getHandle(), mhaProblem, nullptr, solutions.data(), &numFound, solutions.size()); + MIOPEN_THROW_IF(s != miopenStatusSuccess, "failed while finding solutions for mha bwd"); + + solutions.resize(numFound); + + // Ensure miopenDestroySolution() will be called even if an exception occurs + scope_exit finallyForSolutions([&]() { + for(miopenSolution_t sol : solutions) + { + miopenDestroySolution(sol); + } + }); + + std::vector engines; + engines.reserve(numFound); + + size_t i = 0; + std::transform(solutions.cbegin(), + solutions.cend(), + std::back_inserter(engines), + [&i, tensorMap, graphPtr](miopenSolution_t sol) -> Engine { + return EngineBuilder() + .setGraph(graphPtr) + .setExecutor(std::make_shared( + std::move(deref(sol)), tensorMap)) + .setGlobalIndex(i++) + .build(); + }); + + return engines; } }; -*/ std::vector findEngines(OpGraph* graph) { @@ -323,6 +1041,8 @@ std::vector findEngines(OpGraph* graph) std::vector> patterns; patterns.emplace_back(MHA_Fwd_F8_Pattern::Make()); + patterns.emplace_back(MHA_Bwd_F8_Pattern::Make()); + patterns.emplace_back(ConvBiasResAddActive_Fwd_Pattern::Make()); for(const auto& p : patterns) { diff --git a/src/graphapi/tensor.cpp b/src/graphapi/tensor.cpp index 6fa5da8f15..4b9367a7b1 100644 --- a/src/graphapi/tensor.cpp +++ b/src/graphapi/tensor.cpp @@ -27,6 +27,7 @@ #include #include #include +#include namespace miopen { @@ -303,6 +304,20 @@ void BackendTensorDescriptor::getAttribute(miopenBackendAttributeName_t attribut } } +void to_json(nlohmann::json& json, const Tensor& tensor) +{ + json = dynamic_cast(tensor); + json[Tensor::JsonFields::Id] = tensor.mId; + json[Tensor::JsonFields::IsVirtual] = tensor.mVirtual; +} + +void from_json(const nlohmann::json& json, Tensor& tensor) +{ + json.get_to(tensor); + json.at(Tensor::JsonFields::Id).get_to(tensor.mId); + json.at(Tensor::JsonFields::IsVirtual).get_to(tensor.mVirtual); +} + } // namespace graphapi } // namespace miopen diff --git a/src/hip/handlehip.cpp b/src/hip/handlehip.cpp index 956556af17..a76a8ef470 100644 --- a/src/hip/handlehip.cpp +++ b/src/hip/handlehip.cpp @@ -453,7 +453,7 @@ void Handle::Copy(ConstData_t src, Data_t dest, std::size_t size) const { MIOPEN_HANDLE_LOCK this->impl->set_ctx(); - auto status = hipMemcpy(dest, src, size, hipMemcpyDeviceToDevice); + auto status = hipMemcpyWithStream(dest, src, size, hipMemcpyDeviceToDevice, this->GetStream()); if(status != hipSuccess) MIOPEN_THROW_HIP_STATUS(status, "Hip error copying buffer: "); } @@ -518,8 +518,8 @@ void Handle::ClearKernels(const std::string& algorithm, const std::string& netwo this->impl->cache.ClearKernels(algorithm, network_config); } -const std::vector& Handle::GetKernelsImpl(const std::string& algorithm, - const std::string& network_config) const +std::vector Handle::GetKernelsImpl(const std::string& algorithm, + const std::string& network_config) const { return this->impl->cache.GetKernels(algorithm, network_config); } diff --git a/src/include/miopen/activ.hpp b/src/include/miopen/activ.hpp index 20c5f2ba2b..26ce91112f 100644 --- a/src/include/miopen/activ.hpp +++ b/src/include/miopen/activ.hpp @@ -58,7 +58,7 @@ struct MIOPEN_INTERNALS_EXPORT ActivationDescriptor : miopenActivationDescriptor const TensorDescriptor& yDesc, Data_t y, size_t xOffset = 0, - size_t yOffset = 0); + size_t yOffset = 0) const; miopenStatus_t Backward(Handle& handle, const void* alpha, @@ -74,7 +74,7 @@ struct MIOPEN_INTERNALS_EXPORT ActivationDescriptor : miopenActivationDescriptor size_t yOffset = 0, size_t dyOffset = 0, size_t xOffset = 0, - size_t dxOffset = 0); + size_t dxOffset = 0) const; friend std::ostream& operator<<(std::ostream& stream, const ActivationDescriptor& x); diff --git a/src/include/miopen/any_solver.hpp b/src/include/miopen/any_solver.hpp index 0908c2823f..6305893fd9 100644 --- a/src/include/miopen/any_solver.hpp +++ b/src/include/miopen/any_solver.hpp @@ -31,6 +31,7 @@ #include #include #include +#include #include diff --git a/src/include/miopen/batch_norm.hpp b/src/include/miopen/batch_norm.hpp index 50c309550c..92444f039b 100644 --- a/src/include/miopen/batch_norm.hpp +++ b/src/include/miopen/batch_norm.hpp @@ -163,40 +163,44 @@ void bnFwdTrainSelectMulti(const Handle& handle, void profileSequence(const Handle& handle, unsigned char select, float* ctime); -MIOPEN_INTERNALS_EXPORT void -BatchNormForwardInference(Handle& handle, - miopenBatchNormMode_t bn_mode, - const void* alpha, - const void* beta, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& yDesc, - Data_t y, - const TensorDescriptor& bnScaleBiasMeanVarDesc, - ConstData_t bnScale, - ConstData_t bnBias, - ConstData_t estimatedMean, - ConstData_t estimatedVariance, - double epsilon); - -MIOPEN_INTERNALS_EXPORT void -BatchNormForwardTraining(Handle& handle, - miopenBatchNormMode_t bn_mode, - const void* alpha, /* these don't seem to be used in conv */ - const void* beta, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& yDesc, - Data_t y, - const TensorDescriptor& bnScaleBiasMeanVarDesc, - ConstData_t bnScale, - ConstData_t bnBias, - double expAvgFactor, - Data_t resultRunningMean, - Data_t resultRunningVariance, - double epsilon, - Data_t resultSaveMean, - Data_t resultSaveInvVariance); +MIOPEN_INTERNALS_EXPORT void BatchNormForwardInference(Handle& handle, + miopenBatchNormMode_t bn_mode, + const void* alpha, + const void* beta, + const TensorDescriptor& xDesc, + ConstData_t x, + const TensorDescriptor& yDesc, + Data_t y, + const TensorDescriptor& scaleDesc, + const TensorDescriptor& BiasDesc, + const TensorDescriptor& estMeanDesc, + const TensorDescriptor& estVarianceDesc, + ConstData_t bnScale, + ConstData_t bnBias, + ConstData_t estimatedMean, + ConstData_t estimatedVariance, + double epsilon); + +MIOPEN_INTERNALS_EXPORT void BatchNormForwardTraining(Handle& handle, + miopenBatchNormMode_t bn_mode, + const void* alpha, + const void* beta, + const TensorDescriptor& xDesc, + ConstData_t x, + const TensorDescriptor& yDesc, + Data_t y, + const TensorDescriptor& scaleDesc, + const TensorDescriptor& biasDesc, + const TensorDescriptor& savedMeanDesc, + const TensorDescriptor& savedVarianceDesc, + ConstData_t bnScale, + ConstData_t bnBias, + double expAvgFactor, + Data_t resultRunningMean, + Data_t resultRunningVariance, + double epsilon, + Data_t resultSaveMean, + Data_t resultSaveInvVariance); MIOPEN_INTERNALS_EXPORT void BatchNormBackward(Handle& handle, miopenBatchNormMode_t bn_mode, @@ -210,7 +214,10 @@ MIOPEN_INTERNALS_EXPORT void BatchNormBackward(Handle& handle, ConstData_t dy, const TensorDescriptor& dxDesc, Data_t dx, - const TensorDescriptor& bnScaleBiasDiffDesc, + const TensorDescriptor& scaleDesc, + const TensorDescriptor& BiasDesc, + const TensorDescriptor& savedMeanDesc, + const TensorDescriptor& savedVarianceDesc, ConstData_t bnScale, Data_t resultBnScaleDiff, Data_t resultBnBiasDiff, diff --git a/src/include/miopen/batchnorm/invoke_params.hpp b/src/include/miopen/batchnorm/invoke_params.hpp index efab2f3ddb..5c3df3cbcf 100644 --- a/src/include/miopen/batchnorm/invoke_params.hpp +++ b/src/include/miopen/batchnorm/invoke_params.hpp @@ -32,9 +32,9 @@ namespace miopen { namespace batchnorm { -struct InvokeParams : public miopen::InvokeParams +struct FwdTrainInvokeParams : public miopen::InvokeParams { - InvokeParams() = default; + FwdTrainInvokeParams() = default; ConstData_t x = nullptr; Data_t y = nullptr; diff --git a/src/include/miopen/batchnorm/problem_description.hpp b/src/include/miopen/batchnorm/problem_description.hpp index b00473dfc0..4fed9033a6 100644 --- a/src/include/miopen/batchnorm/problem_description.hpp +++ b/src/include/miopen/batchnorm/problem_description.hpp @@ -52,13 +52,24 @@ struct ProblemDescriptionTag { }; -struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase, ProblemDescriptionTag +bool is_fp16_or_bfp16(miopenDataType_t type); +bool is_fp32_or_fp64(miopenDataType_t type); + +struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase, + ProblemDescriptionTag +#if MIOPEN_ENABLE_SQLITE + , + SQLiteSerializable +#endif { // Forward Training ProblemDescription(miopenBatchNormMode_t bn_mode_, const TensorDescriptor& xDesc_, const TensorDescriptor& yDesc_, - const TensorDescriptor& bnScaleBiasMeanVarDesc_, + const TensorDescriptor& scaleDesc_, + const TensorDescriptor& biasDesc_, + const TensorDescriptor& sMeanDesc_, + const TensorDescriptor& sVarianceDesc_, double expAvgFactor_, double epsilon_, bool resultsave_, @@ -67,31 +78,42 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase, Prob bn_mode(bn_mode_), xDesc(xDesc_), yOrDyDesc(yDesc_), - scaleBiasDesc(bnScaleBiasMeanVarDesc_), + scaleDesc(scaleDesc_), + biasDesc(biasDesc_), + sMeanDesc(sMeanDesc_), + sVarianceDesc(sVarianceDesc_), expAvgFactor(expAvgFactor_), epsilon(epsilon_), resultsave(resultsave_), resultrunning(resultrunning_) { - in_layout = xDesc.GetLayout(xDesc.GetLengths().size() == 4 ? "NCHW" : "NCDHW"); - out_layout = yOrDyDesc.GetLayout(yOrDyDesc.GetLengths().size() == 4 ? "NCHW" : "NCDHW"); + SetSpatialDims(); + in_layout = ComputeInLayout(); + out_layout = ComputeOutLayout(); } - // Forward + // Forward Inference ProblemDescription(miopenBatchNormMode_t bn_mode_, const TensorDescriptor& xDesc_, const TensorDescriptor& yDesc_, - const TensorDescriptor& bnScaleBiasMeanVarDesc_, + const TensorDescriptor& scaleDesc_, + const TensorDescriptor& biasDesc_, + const TensorDescriptor& sMeanDesc_, + const TensorDescriptor& sVarianceDesc_, double epsilon_) : direction(Direction::ForwardInference), bn_mode(bn_mode_), xDesc(xDesc_), yOrDyDesc(yDesc_), - scaleBiasDesc(bnScaleBiasMeanVarDesc_), + scaleDesc(scaleDesc_), + biasDesc(biasDesc_), + sMeanDesc(sMeanDesc_), + sVarianceDesc(sVarianceDesc_), epsilon(epsilon_) { - in_layout = xDesc.GetLayout(xDesc.GetLengths().size() == 4 ? "NCHW" : "NCDHW"); - out_layout = yOrDyDesc.GetLayout(yOrDyDesc.GetLengths().size() == 4 ? "NCHW" : "NCDHW"); + SetSpatialDims(); + in_layout = ComputeInLayout(); + out_layout = ComputeOutLayout(); } // Backward @@ -99,7 +121,10 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase, Prob const TensorDescriptor& xDesc_, const TensorDescriptor& dyDesc_, const TensorDescriptor& dxDesc_, - const TensorDescriptor& bnScaleBiasDiffDesc_, + const TensorDescriptor& scaleDesc_, + const TensorDescriptor& biasDesc_, + const TensorDescriptor& sMeanDesc_, + const TensorDescriptor& sVarianceDesc_, double epsilon_, bool useSaved_) : direction(Direction::Backward), @@ -107,15 +132,28 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase, Prob xDesc(xDesc_), yOrDyDesc(dyDesc_), dxDesc(dxDesc_), - scaleBiasDesc(bnScaleBiasDiffDesc_), + scaleDesc(scaleDesc_), + biasDesc(biasDesc_), + sMeanDesc(sMeanDesc_), + sVarianceDesc(sVarianceDesc_), epsilon(epsilon_), useSaved(useSaved_) { - in_layout = xDesc.GetLayout(xDesc.GetLengths().size() == 4 ? "NCHW" : "NCDHW"); - out_layout = yOrDyDesc.GetLayout(yOrDyDesc.GetLengths().size() == 4 ? "NCHW" : "NCDHW"); - din_layout = dxDesc.GetLayout(dxDesc.GetLengths().size() == 4 ? "NCHW" : "NCDHW"); + SetSpatialDims(); + in_layout = ComputeInLayout(); + out_layout = ComputeOutLayout(); + din_layout = ComputeDinLayout(); } + void SetSpatialDims() + { + if(Is2D()) + spatial_dim = 2; + else if(Is3D()) + spatial_dim = 3; + else + MIOPEN_THROW("Unknown spatial dim!"); + } Direction GetDirection() const { return direction; } miopenBatchNormMode_t GetMode() const { return bn_mode; } const TensorDescriptor& GetXDesc() const { return xDesc; } @@ -138,17 +176,13 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase, Prob return dxDesc; } - const TensorDescriptor& GetBnScaleBiasMeanVarDesc() const - { - assert(direction == Direction::ForwardTraining || direction == Direction::ForwardInference); - return scaleBiasDesc; - } + const TensorDescriptor& GetBnScale() const { return scaleDesc; } - const TensorDescriptor& GetScaleBiasDiffDesc() const - { - assert(direction == Direction::Backward); - return scaleBiasDesc; - } + const TensorDescriptor& GetBnBias() const { return biasDesc; } + + const TensorDescriptor& GetBnSMean() const { return sMeanDesc; } + + const TensorDescriptor& GetBnSVar() const { return sVarianceDesc; } bool GetResultSave() const { @@ -182,15 +216,68 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase, Prob : ((in_layout == "NDHWC") && (out_layout == "NDHWC")); } + bool IsLayoutNCHW() const + { + if(direction == Direction::Backward) + { + return xDesc.GetLengths().size() == 4 + ? ((in_layout == "NCHW") && (out_layout == "NCHW") && (din_layout == "NCHW")) + : ((in_layout == "NCDHW") && (out_layout == "NCDHW") && + (din_layout == "NCDHW")); + } + + return xDesc.GetLengths().size() == 4 ? ((in_layout == "NCHW") && (out_layout == "NCHW")) + : ((in_layout == "NCDHW") && (out_layout == "NCDHW")); + } + bool Is2D() const { return xDesc.GetLengths().size() == 4; } + bool Is3D() const { return xDesc.GetLengths().size() == 5; } bool IsFp64() const { return xDesc.GetType() == miopenDouble; } bool IsFp32() const { return xDesc.GetType() == miopenFloat; } bool IsFp16() const { return xDesc.GetType() == miopenHalf; } - bool IsBfp16() const { return xDesc.GetType() == miopenBFloat16; } + bool IsBFp16() const { return xDesc.GetType() == miopenBFloat16; } + bool IsMix() const { return (IsFp16() || IsBFp16()) && sMeanDesc.GetType() == miopenFloat; } + bool IsScaleFp16() const { return scaleDesc.GetType() == miopenHalf; } + bool IsScaleFp32() const { return scaleDesc.GetType() == miopenFloat; } + + void Serialize(std::ostream& stream) const { stream << MakeNetworkConfig().ToString(); } NetworkConfig MakeNetworkConfig() const override; + template + static void Visit(Self&& self, std::function f) + { + // The column names match the driver command line argument names + f(self.spatial_dim, "spatial_dim"); + f(self.GetBatchSize(), "batchsize"); + f(self.GetChannel(), "in_channels"); + f(self.GetHeight(), "in_h"); + f(self.GetWidth(), "in_w"); + f(self.GetDepth(), "in_d"); + + f(self.resultsave, "resultsave"); + f(self.resultrunning, "resultrunning"); + f(self.useSaved, "useSaved"); + } + + template + static void Visit(Self&& self, std::function f) + { + f(self.ComputeInLayout(), "layout"); + f(self.GetDirectionStr(), "direction"); + f(GetDataTypeName(self.xDesc.GetType()), "data_type"); + f(self.GetModeStr(), "mode"); + } + + template + static void VisitAll(Self&& self, const Visitor& f) + { + Visit(std::forward(self), [&](int64_t value, std::string name) { f(value, name); }); + Visit(std::forward(self), + [&](std::string value, std::string name) { f(value, name); }); + } + // This declaration marks batchnorm as a primitive with tuning enabled. // Any tunable solver would be able pick it and fetch a db instance in ExecutePrimitive. // It has to be discoverable via ADL from problem description. @@ -199,10 +286,14 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase, Prob private: Direction direction; miopenBatchNormMode_t bn_mode; - TensorDescriptor xDesc; - TensorDescriptor yOrDyDesc; + TensorDescriptor xDesc; // input + TensorDescriptor yOrDyDesc; // output TensorDescriptor dxDesc; - TensorDescriptor scaleBiasDesc; + + TensorDescriptor scaleDesc; // scale + TensorDescriptor biasDesc; // bias (shift) + TensorDescriptor sMeanDesc; + TensorDescriptor sVarianceDesc; #ifdef __clang__ #pragma clang diagnostic push @@ -216,18 +307,66 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase, Prob #pragma clang diagnostic pop #endif - bool resultsave = false; - bool resultrunning = false; - bool useSaved = false; - std::string in_layout = "NCHW"; - std::string out_layout = "NCHW"; - std::string din_layout = "NCHW"; + bool resultsave = false; + bool resultrunning = false; + bool useSaved = false; + std::string in_layout = "NCHW"; + std::string out_layout = "NCHW"; + std::string din_layout = "NCHW"; + std::size_t spatial_dim = 2; NetworkConfig MakeForwardTrainingNetworkConfig() const; NetworkConfig MakeForwardInferenceNetworkConfig() const; NetworkConfig MakeBackwardNetworkConfig() const; + + std::string ComputeLayout(const TensorDescriptor& td) const { return td.GetLayout_str(); } + std::string ComputeInLayout() const { return ComputeLayout(xDesc); } + std::string ComputeOutLayout() const { return ComputeLayout(yOrDyDesc); } + std::string ComputeDinLayout() const { return ComputeLayout(dxDesc); } + + size_t GetSpatialDims() const { return spatial_dim; } + + std::size_t GetBatchSize() const { return GetN5(GetSpatialDims(), xDesc.GetLengths()); } + std::size_t GetChannel() const { return GetC5(GetSpatialDims(), xDesc.GetLengths()); } + std::size_t GetHeight() const { return GetH5(GetSpatialDims(), xDesc.GetLengths()); } + std::size_t GetWidth() const { return GetW5(GetSpatialDims(), xDesc.GetLengths()); } + std::size_t GetDepth() const { return GetD5(GetSpatialDims(), xDesc.GetLengths()); } + + std::string GetDirectionStr() const + { + std::string s; + + switch(direction) + { + case Direction::ForwardInference: return "Inf"; ; + case Direction::ForwardTraining: return "Trn"; + case Direction::Backward: return "Bwd"; + default: MIOPEN_THROW(miopenStatusInvalidValue, "Wrong Batchnorm Direction provided"); + } + + return s; + } + + std::string GetModeStr() const + { + switch(bn_mode) + { + case miopenBNPerActivation: return "0"; + case miopenBNSpatial: return "1"; + default: MIOPEN_THROW(miopenStatusInvalidValue, "Wrong Batchnorm Mode provided"); + } + } }; +bool IsOCLInferTypeValid(const ProblemDescription& bn_problem); +bool IsCKInferTypeValid(const ProblemDescription& bn_problem); + +bool IsOCLFwdTrainTypeValid(const ProblemDescription& bn_problem); +bool IsCKFwdTrainTypeValid(const ProblemDescription& bn_problem); + +bool IsOCLBwdTypeValid(const ProblemDescription& bn_problem); +bool IsCKBwdTypeValid(const ProblemDescription& bn_problem); + } // namespace batchnorm } // namespace miopen diff --git a/src/include/miopen/batchnorm/solvers.hpp b/src/include/miopen/batchnorm/solvers.hpp index 7edba36b49..7b8696b95d 100644 --- a/src/include/miopen/batchnorm/solvers.hpp +++ b/src/include/miopen/batchnorm/solvers.hpp @@ -44,6 +44,11 @@ namespace batchnorm { using BatchnormSolver = NonTunableSolverBase; +template +using BatchNormTunableSolver = + TunableSolverMixin; +; + struct BnFwdTrainingSpatialSingle final : BatchnormSolver { const std::string& SolverDbId() const override @@ -128,38 +133,212 @@ struct BnFwdInference final : BatchnormSolver bool IsApplicable(const ExecutionContext& context, const miopen::batchnorm::ProblemDescription& problem) const override; + bool IsDynamic() const override { return true; } ConvSolution GetSolution(const ExecutionContext& context, const miopen::batchnorm::ProblemDescription& problem) const override; }; -struct BnCKFwdInference final : BatchnormSolver +struct PerformanceConfigBnCKFwdInference : PerfConfigBaseCK +{ + int index; + std::string kernel_id; + std::vector valid_kernels; + MIOPEN_INTERNALS_EXPORT PerformanceConfigBnCKFwdInference(int idx, std::string kernl_id) + : index(idx), kernel_id(kernl_id) + { + } + PerformanceConfigBnCKFwdInference() : PerformanceConfigBnCKFwdInference(0, "") {} + PerformanceConfigBnCKFwdInference(bool) : PerformanceConfigBnCKFwdInference(0, "") {} + MIOPEN_INTERNALS_EXPORT void + HeuristicInit(const miopen::batchnorm::ProblemDescription& problem); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::batchnorm::ProblemDescription& problem); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool + IsValid(const ExecutionContext&, const miopen::batchnorm::ProblemDescription& problem) const; + + template + static void Visit(Self&& s, F f) + { + f(s.kernel_id, "kernel_id"); + } + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigBnCKFwdInference& other) const; + +private: + template + void Init(const miopen::batchnorm::ProblemDescription&); + template + bool CheckIsSupportCKArgs(const miopen::batchnorm::ProblemDescription&) const; +}; + +struct BnCKFwdInference final : BatchNormTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - bool IsApplicable(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem) const override; - ConvSolution GetSolution(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT PerformanceConfigBnCKFwdInference GetDefaultPerformanceConfig( + const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem, + const PerformanceConfigBnCKFwdInference& config) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigBnCKFwdInference + Search(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem, + const PerformanceConfigBnCKFwdInference& config) const override; +}; + +struct PerformanceConfigBnCKBwdBackward : PerfConfigBaseCK +{ + int index; + std::string kernel_id; + std::vector valid_kernels; + MIOPEN_INTERNALS_EXPORT PerformanceConfigBnCKBwdBackward(int idx, std::string kernl_id) + : index(idx), kernel_id(kernl_id) + { + } + PerformanceConfigBnCKBwdBackward() : PerformanceConfigBnCKBwdBackward(0, "") {} + PerformanceConfigBnCKBwdBackward(bool) : PerformanceConfigBnCKBwdBackward(0, "") {} + MIOPEN_INTERNALS_EXPORT void + HeuristicInit(const miopen::batchnorm::ProblemDescription& problem); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::batchnorm::ProblemDescription& problem); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool + IsValid(const ExecutionContext&, const miopen::batchnorm::ProblemDescription& problem) const; + + template + static void Visit(Self&& s, F f) + { + f(s.kernel_id, "kernel_id"); + } + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigBnCKBwdBackward& other) const; + +private: + template + void Init(const miopen::batchnorm::ProblemDescription&); + template + bool CheckIsSupportCKArgs(const miopen::batchnorm::ProblemDescription&) const; }; -struct BnCKBwdBackward final : BatchnormSolver +struct BnCKBwdBackward final : BatchNormTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - bool IsApplicable(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem) const override; - ConvSolution GetSolution(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT PerformanceConfigBnCKBwdBackward GetDefaultPerformanceConfig( + const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem, + const PerformanceConfigBnCKBwdBackward& config) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigBnCKBwdBackward + Search(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem, + const PerformanceConfigBnCKBwdBackward& config) const override; +}; + +struct PerformanceConfigBnCKFwdTraining : PerfConfigBaseCK +{ + int index; + std::string kernel_id; + std::vector valid_kernels; + MIOPEN_INTERNALS_EXPORT PerformanceConfigBnCKFwdTraining(int idx, std::string kernl_id) + : index(idx), kernel_id(kernl_id) + { + } + PerformanceConfigBnCKFwdTraining() : PerformanceConfigBnCKFwdTraining(0, "") {} + PerformanceConfigBnCKFwdTraining(bool) : PerformanceConfigBnCKFwdTraining(0, "") {} + MIOPEN_INTERNALS_EXPORT void + HeuristicInit(const miopen::batchnorm::ProblemDescription& problem); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::batchnorm::ProblemDescription& problem); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool + IsValid(const ExecutionContext&, const miopen::batchnorm::ProblemDescription& problem) const; + + template + static void Visit(Self&& s, F f) + { + f(s.kernel_id, "kernel_id"); + } + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigBnCKFwdTraining& other) const; + +private: + template + void Init(const miopen::batchnorm::ProblemDescription&); + template + bool CheckIsSupportCKArgs(const miopen::batchnorm::ProblemDescription&) const; }; -struct BnCKFwdTraining final : BatchnormSolver +struct BnCKFwdTraining final : BatchNormTunableSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } - bool IsApplicable(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem) const override; - ConvSolution GetSolution(const ExecutionContext& context, - const miopen::batchnorm::ProblemDescription& problem) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT PerformanceConfigBnCKFwdTraining GetDefaultPerformanceConfig( + const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem, + const PerformanceConfigBnCKFwdTraining& config) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigBnCKFwdTraining + Search(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext& ctx, + const miopen::batchnorm::ProblemDescription& problem, + const PerformanceConfigBnCKFwdTraining& config) const override; }; } // namespace batchnorm diff --git a/src/include/miopen/conv/heuristics/ai_heuristics.hpp b/src/include/miopen/conv/heuristics/ai_heuristics.hpp index b7d2cdeb23..2d240c0814 100644 --- a/src/include/miopen/conv/heuristics/ai_heuristics.hpp +++ b/src/include/miopen/conv/heuristics/ai_heuristics.hpp @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include diff --git a/src/include/miopen/conv/invokers/impl_gemm_dynamic.hpp b/src/include/miopen/conv/invokers/impl_gemm_dynamic.hpp index 3bdba461c2..3248989c4b 100644 --- a/src/include/miopen/conv/invokers/impl_gemm_dynamic.hpp +++ b/src/include/miopen/conv/invokers/impl_gemm_dynamic.hpp @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/include/miopen/conv/problem_description.hpp b/src/include/miopen/conv/problem_description.hpp index 9447d7a5ca..16d207d5b0 100644 --- a/src/include/miopen/conv/problem_description.hpp +++ b/src/include/miopen/conv/problem_description.hpp @@ -32,6 +32,7 @@ #include #include +#include #include #include @@ -101,45 +102,15 @@ constexpr TElement GetWofCHWN(const std::vector& data) return std::get<2>(GetCHWN(data)); } -template -constexpr TElement GetN5(unsigned spatial_dims, const std::vector& data) -{ - return std::get<0>(GetNCDHW(spatial_dims, data)); -} - -template -constexpr TElement GetC5(unsigned spatial_dims, const std::vector& data) -{ - return std::get<1>(GetNCDHW(spatial_dims, data)); -} - -template -constexpr TElement GetD5(unsigned spatial_dims, const std::vector& data) -{ - return std::get<2>(GetNCDHW(spatial_dims, data)); -} - -template -constexpr TElement GetH5(unsigned spatial_dims, const std::vector& data) -{ - return std::get<3>(GetNCDHW(spatial_dims, data)); -} - -template -constexpr TElement GetW5(unsigned spatial_dims, const std::vector& data) -{ - return std::get<4>(GetNCDHW(spatial_dims, data)); -} - namespace conv { MIOPEN_INTERNALS_EXPORT miopenAlphaBetaCase_t ClassifyAlphaBeta(const Scalar& alpha, const Scalar& beta); -struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase #if MIOPEN_ENABLE_SQLITE - , - SQLiteSerializable +struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionWeightsBase +#else +struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionWeightsBase, SQLiteSerializable #endif { ProblemDescription() = default; @@ -153,13 +124,8 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase int bias_ = 0, const Scalar& alpha_ = Scalar(1.0), const Scalar& beta_ = Scalar(0.0)) - : in(in_), - weights(weights_), - out(out_), + : ProblemDescriptionWeightsBase(in_, weights_, out_), conv(conv_), - in_layout(ComputeInLayout()), - weights_layout(ComputeWeightsLayout()), - out_layout(ComputeOutLayout()), direction(direction_), bias(bias_), alpha(alpha_), @@ -224,14 +190,14 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase std::size_t GetWeightsDepth() const { return GetD5(GetSpatialDims(), weights.GetLengths()); } std::size_t GetWeightsHeight() const { - if(weights.GetLayout_str() == "CHWNc") + if(weights_layout == "CHWNc") return GetHofCHWN(weights.GetLengths()); else return GetH5(GetSpatialDims(), weights.GetLengths()); } std::size_t GetWeightsWidth() const { - if(weights.GetLayout_str() == "CHWNc") + if(weights_layout == "CHWNc") return GetWofCHWN(weights.GetLengths()); else return GetW5(GetSpatialDims(), weights.GetLengths()); @@ -309,7 +275,8 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase bool IsInt8() const { return GetInDataType() == miopenInt8 && GetWeightsDataType() == miopenInt8 && - (GetOutDataType() == miopenInt32 || GetOutDataType() == miopenFloat); + (GetOutDataType() == miopenInt32 || GetOutDataType() == miopenInt8 || + GetOutDataType() == miopenFloat); } bool IsFp8() const { @@ -391,10 +358,6 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase return os; } -#if MIOPEN_ENABLE_SQLITE - static std::string table_name() { return "config"; } -#endif - template static void Visit(Self&& self, std::function f) { @@ -443,49 +406,18 @@ struct MIOPEN_INTERNALS_EXPORT ProblemDescription : ProblemDescriptionBase void SetupFloats(ExecutionContext& ctx) const; private: - std::string ComputeInLayout() const - { - if(GetSpatialDims() == 2) - { - return in.GetLayout(in.GetLayout_str()); - } - else - { - return in.GetLayout("NCDHW"); - } - } - - std::string ComputeOutLayout() const - { - if(GetSpatialDims() == 2) - { - return out.GetLayout(out.GetLayout_str()); - } - else - { - return out.GetLayout("NCDHW"); - } - } - - std::string ComputeWeightsLayout() const - { - if(GetSpatialDims() == 2) - { - return weights.GetLayout(weights.GetLayout_str()); - } - else - { - return weights.GetLayout("NCDHW"); - } - } +<<<<<<< HEAD +======= + std::string ComputeLayout(const TensorDescriptor& td) const; + std::string ComputeInLayout() const; + std::string ComputeOutLayout() const; + std::string ComputeWeightsLayout() const; TensorDescriptor in; TensorDescriptor weights; TensorDescriptor out; +>>>>>>> develop ConvolutionDescriptor conv; - std::string in_layout; - std::string weights_layout; - std::string out_layout; Direction direction = Direction::Forward; int bias = 0; Scalar alpha = Scalar(1.0); diff --git a/src/include/miopen/conv/solvers.hpp b/src/include/miopen/conv/solvers.hpp new file mode 100644 index 0000000000..1861adc5eb --- /dev/null +++ b/src/include/miopen/conv/solvers.hpp @@ -0,0 +1,4916 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace miopen { + +namespace debug { + +/// If set to true, then always enable ConvDirectNaive* solver, regardless of environment value +/// MIOPEN_DEBUG_CONV_DIRECT_NAIVE_CONV_* that control enable/disable of these solvers. +/// Currently used during driver using naive kernel as gpu reference. +MIOPEN_EXPORT extern bool + AlwaysEnableConvDirectNaive; // NOLINT (cppcoreguidelines-avoid-non-const-global-variables) + +/// WORKAROUND_SWDEV_271887 disables ConvOclDirectFwd1x1 solver on gfx10 due to precision issues. +/// However we still want to check that the solver is not broken and therefore use +/// disable_wa_swdev_271887 = true to enable it. +// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables) +MIOPEN_INTERNALS_EXPORT extern bool disable_wa_swdev_271887; + +} // namespace debug + +struct AnyInvokeParams; + +namespace solver { +/// \todo Move wave_size into abstraction wich represent GPU information +const int wave_size = 64; + +namespace conv { + +/// Common interface for convolution tunable and non-tunable solvers +using ConvSolverInterface = SolverInterface; + +/// Common interface for convolution non-tunable solvers +using ConvSolverInterfaceNonTunable = + SolverInterfaceNonTunable; + +/// Common interface for convolution tunable solvers +using ConvSolverInterfaceTunable = + SolverInterfaceTunable; + +/// Typedef for convolution non-tunable solvers +using ConvSolver = SolverBaseNonTunable; + +/// Typedef for convolution tunable solvers +template +using ConvTunableSolver = + SolverBaseTunable; + +struct PerformanceConfigConvAsm3x3U : PerfConfigBase +{ + int limit_wave_cnt; // [0..9] + int filters_per_wave; // [1..8] + int output_lines_per_wave; // [1..8] + + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm3x3U(int lwc, int fpw, int olpw); + PerformanceConfigConvAsm3x3U() : PerformanceConfigConvAsm3x3U(-1, -1, -1) {} + PerformanceConfigConvAsm3x3U(bool) : PerformanceConfigConvAsm3x3U(0, 1, 1) {} + + template + static void Visit(Self&& self, F f) + { + f(self.limit_wave_cnt, "limit_wave_cnt"); + f(self.filters_per_wave, "filters_per_wave"); + f(self.output_lines_per_wave, "output_lines_per_wave"); + } + + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvAsm3x3U& other) const; +}; + +struct ConvAsm3x3U final : ConvTunableSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm3x3U GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvAsm3x3U&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm3x3U + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvAsm3x3U&) const override; +}; + +struct PerformanceConfigConvAsm1x1U : PerfConfigBase +{ + // ----------------- // Full set Optimized Spare + // ---------------------------------------------------------------------------- + int read_size; // [1..4] + int k_mult; // 1,[4,8,12..32] 2^n[8..32] 1,4 + int chunks_per_wave; // [1..16] [1..8] + int chunk_size; // 2^n[1..64] 2^n[16..64] 1,4 + int n_mult; // [1..8] [1..4] + int c_mult; // 2^n[1..32] 2^n[1..4] + int waves_c_in_group; // [1..8] [1..4] + int waves_k_in_group; // 1,[2,4,8] 1,[2,4,8] + bool use_spare_set; + + MIOPEN_INTERNALS_EXPORT + PerformanceConfigConvAsm1x1U(int, int, int, int, int, int, int, int, bool); + PerformanceConfigConvAsm1x1U() + : PerformanceConfigConvAsm1x1U(-1, -1, -1, -1, -1, -1, -1, -1, false) + { + } + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1U(bool spare); + + template + static void Visit(Self&& self, F f) + { + f(self.read_size, "read_size"); + f(self.k_mult, "k_mult"); + f(self.chunks_per_wave, "chunks_per_wave"); + f(self.chunk_size, "chunk_size"); + f(self.n_mult, "n_mult"); + f(self.c_mult, "c_mult"); + f(self.waves_c_in_group, "waves_c_in_group"); + f(self.waves_k_in_group, "waves_k_in_group"); + } + + int GetReadSize() const { return read_size; } + int GetKMult() const { return k_mult; } + int GetChunksPerWave() const { return chunks_per_wave; } + int GetChunkSize() const { return chunk_size; } + int GetNMult() const { return n_mult; } + int GetCMult() const { return c_mult; } + int GetWavesCInGroup() const { return waves_c_in_group; } + int GetWavesKInGroup() const { return waves_k_in_group; } + int GetNPerGpr() const + { + assert(chunk_size); + return 64 / chunk_size; + } + + MIOPEN_INTERNALS_EXPORT void StaticHeuristic(const miopen::conv::ProblemDescription& problem); + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool + IsModelApplicable(const ExecutionContext& ctx, + const miopen::conv::ProblemDescription& problem) const; + bool IsValidValue() const { return IsValidValueImpl(8); } + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + bool IsValid(const miopen::conv::ProblemDescription& problem) const + { + return IsValidImpl(problem, 8); + } + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvAsm1x1U& other) const; + +private: +#if MIOPEN_ENABLE_AI_KERNEL_TUNING + bool IsPartiallyValid(const miopen::conv::ProblemDescription& problem, + int sequence_length) const + { + return IsValidImpl(problem, sequence_length); + } + bool IsPartiallyValidValue(int sequence_length) const + { + return IsValidValueImpl(sequence_length); + } + bool RunParameterPredictionModel(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + bool ModelApplyToken(int index, std::string value, const miopen::conv::ProblemDescription&); +#endif + bool IsValidImpl(const miopen::conv::ProblemDescription& problem, int sequence_length) const; + bool IsValidValueImpl(int sequence_length) const; +}; + +struct ConvAsm1x1U final : ConvTunableSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1U GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvAsm1x1U&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1U + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvAsm1x1U&) const override; +}; + +struct PerformanceConfigConvAsm1x1UV2 : PerfConfigBase +{ + // ----------------- // Full set Optimized Spare + // ---------------------------------------------------------------------------- + int chunk_size; // 2^n[1..64] 2^n[16..64] + int dwords_per_ld; // [1..4] 1,2,3 + int k_mult; // [1..32] 8,16 1,2,3,4 + int c_mult; // [1..32] 2^n[1..4] + int n_mult; // [1..32] 1,2 + int w_mult; // [1..32] 1,2 + int h_mult; // [1..32] 1,2 + int h_per_chunk; // 2^n[1..64] [2,4,8] + int waves_k_in_group; // [1..8] 2,4 + int waves_c_in_group; // [1..8] 1,2 + bool use_spare_set; + + MIOPEN_INTERNALS_EXPORT + PerformanceConfigConvAsm1x1UV2(int, int, int, int, int, int, int, int, int, int, bool); + PerformanceConfigConvAsm1x1UV2() + : PerformanceConfigConvAsm1x1UV2(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, false) + { + } + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1UV2(bool spare); + + template + static void Visit(Self&& self, F f) + { + f(self.chunk_size, "chunk_size"); + f(self.dwords_per_ld, "dwords_per_ld"); + f(self.k_mult, "k_mult"); + f(self.c_mult, "c_mult"); + f(self.n_mult, "n_mult"); + f(self.w_mult, "w_mult"); + f(self.h_mult, "h_mult"); + f(self.h_per_chunk, "h_per_chunk"); + f(self.waves_k_in_group, "waves_k_in_group"); + f(self.waves_c_in_group, "waves_c_in_group"); + } + + int GetChunkSize() const { return chunk_size; } + int GetDwordsPerLd() const { return dwords_per_ld; } + int GetCMult() const { return c_mult; } + int GetKMult() const { return k_mult; } + int GetNMult() const { return n_mult; } + int GetWMult() const { return w_mult; } + int GetHMult() const { return h_mult; } + int GetHPerChunk() const { return h_per_chunk; } + int GetWavesCInGroup() const { return waves_c_in_group; } + int GetWavesKInGroup() const { return waves_k_in_group; } + int GetNPerGpr() const + { + assert(chunk_size); + return 64 / chunk_size; + } + + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvAsm1x1UV2& other) const; +}; + +struct ConvAsm1x1UV2 final : ConvTunableSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1UV2 GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvAsm1x1UV2&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1UV2 + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvAsm1x1UV2&) const override; +}; + +struct ConvAsm5x10u2v2f1 final : ConvSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct ConvAsm5x10u2v2b1 final : ConvSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct ConvAsm7x7c3h224w224k64u2v2p3q3f1 final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct ConvOclDirectFwd11x11 final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct ConvOclDirectFwdGen final : ConvSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct PerformanceImplicitGemm : PerfConfigBase +{ + int BPerBlock; // 2^n[8..16] + int KPerBlock; // 2^n[32..128] + int EPerBlock; // 2^n[4..16] + + int GemmNRepeat; // == 2 + + int GemmMPerThreadSubC; // 2^n[2..4] + int GemmNPerThreadSubC; // 2^n[2..4] + + int GemmMLevel0Cluster; // 2^n[1..4] + int GemmNLevel0Cluster; // 2^n[1..4] + int GemmMLevel1Cluster; // 2^n[1..4] + int GemmNLevel1Cluster; // 2^n[1..4] + + int InBlockCopyClusterLengths_E; // 2^n[4..16] + int InBlockCopyClusterLengths_B; // 2^n[8..16] + int InBlockCopyClusterLengths_N1; // 2^n[1..2] + int InBlockCopyClusterLengths_N2; // 2^n[1..4] + + int WeiBlockCopyClusterLengths_E; // 2^n[1..4] + int WeiBlockCopyClusterLengths_K; // 2^n[16..128] + + bool use_spare_set; + + PerformanceImplicitGemm( + int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, bool); + + PerformanceImplicitGemm() + : PerformanceImplicitGemm( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, false) + { + } + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemm(bool spare); + + template + static void Visit(Self&& self, F f) + { + f(self.BPerBlock, "BPerBlock"); + f(self.KPerBlock, "KPerBlock"); + f(self.EPerBlock, "EPerBlock"); + f(self.GemmNRepeat, "GemmNRepeat"); + f(self.GemmMPerThreadSubC, "GemmMPerThreadSubC"); + f(self.GemmNPerThreadSubC, "GemmNPerThreadSubC"); + f(self.GemmMLevel0Cluster, "GemmMLevel0Cluster"); + f(self.GemmNLevel0Cluster, "GemmNLevel0Cluster"); + f(self.GemmMLevel1Cluster, "GemmMLevel1Cluster"); + f(self.GemmNLevel1Cluster, "GemmNLevel1Cluster"); + f(self.InBlockCopyClusterLengths_E, "InBlockCopyClusterLengths_E"); + f(self.InBlockCopyClusterLengths_N1, "InBlockCopyClusterLengths_N1"); + f(self.InBlockCopyClusterLengths_B, "InBlockCopyClusterLengths_B"); + f(self.InBlockCopyClusterLengths_N2, "InBlockCopyClusterLengths_N2"); + f(self.WeiBlockCopyClusterLengths_E, "WeiBlockCopyClusterLengths_E"); + f(self.WeiBlockCopyClusterLengths_K, "WeiBlockCopyClusterLengths_K"); + } + + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemm& other) const; +}; + +struct PerformanceImplicitGemmV4R1 : public PerformanceImplicitGemm +{ + PerformanceImplicitGemmV4R1(int a, + int b, + int c, + int d, + int e, + int f, + int g, + int h, + int i, + int j, + int k, + int l, + int m, + int n, + int o, + int p, + bool q) + : PerformanceImplicitGemm(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) + { + } + + PerformanceImplicitGemmV4R1() + : PerformanceImplicitGemmV4R1( + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, false) + { + } + + PerformanceImplicitGemmV4R1(bool spare) : PerformanceImplicitGemm(spare) {} + + MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; +}; + +struct PerformanceImplicitGemmV4R4Fwd : PerfConfigBase +{ + int BlockSize; + + int GemmMPerBlock; + int GemmNPerBlock; + int GemmKPerBlock; + + int GemmMPerThread; + int GemmNPerThread; + + bool use_spare_set; + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4Fwd(int, int, int, int, int, int, bool); + + PerformanceImplicitGemmV4R4Fwd(int a, int b, int c, int d, int e, int f) + : PerformanceImplicitGemmV4R4Fwd(a, b, c, d, e, f, false) + { + } + + PerformanceImplicitGemmV4R4Fwd() : PerformanceImplicitGemmV4R4Fwd(-1, -1, -1, -1, -1, -1, false) + { + } + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4Fwd(bool spare); + + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemmV4R4Fwd& other) const; + + template + static void Visit(Self&& self, F f) + { + f(self.BlockSize, "BlockSize"); + f(self.GemmMPerBlock, "GemmMPerBlock"); + f(self.GemmNPerBlock, "GemmNPerBlock"); + f(self.GemmKPerBlock, "GemmKPerBlock"); + f(self.GemmMPerThread, "GemmMPerThread"); + f(self.GemmNPerThread, "GemmNPerThread"); + } + + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGridSize(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateBlockGemmPerformanceParameters() const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmABlockCopyPerformanceParameters() const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); +}; + +struct PerformanceImplicitGemmV4R4WrW : PerfConfigBase +{ + int BlockSize; + + int GemmMPerBlock; + int GemmNPerBlock; + int GemmKPerBlock; + + int GemmMPerThread; + int GemmNPerThread; + + bool use_spare_set; + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4WrW(int, int, int, int, int, int, bool); + + PerformanceImplicitGemmV4R4WrW(int a, int b, int c, int d, int e, int f) + : PerformanceImplicitGemmV4R4WrW(a, b, c, d, e, f, false) + { + } + + PerformanceImplicitGemmV4R4WrW() : PerformanceImplicitGemmV4R4WrW(-1, -1, -1, -1, -1, -1, false) + { + } + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4WrW(bool spare); + + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemmV4R4WrW& other) const; + + template + static void Visit(Self&& self, F f) + { + f(self.BlockSize, "BlockSize"); + f(self.GemmMPerBlock, "GemmMPerBlock"); + f(self.GemmNPerBlock, "GemmNPerBlock"); + f(self.GemmKPerBlock, "GemmKPerBlock"); + f(self.GemmMPerThread, "GemmMPerThread"); + f(self.GemmNPerThread, "GemmNPerThread"); + } + + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGridSize(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateBlockGemmPerformanceParameters() const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmABlockCopyPerformanceParameters() const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); +}; + +struct PerformanceImplicitGemmBwdDataV1R1 : PerfConfigBase +{ + int BlockSize; + + int GemmMPerBlock; + int GemmNPerBlock; + int GemmKPerBlock; + + int GemmMPerThread; + int GemmNPerThread; + + bool use_spare_set; + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV1R1(int, int, int, int, int, int, bool); + + PerformanceImplicitGemmBwdDataV1R1() + : PerformanceImplicitGemmBwdDataV1R1(-1, -1, -1, -1, -1, -1, false) + { + } + + PerformanceImplicitGemmBwdDataV1R1(int a, int b, int c, int d, int e, int f) + : PerformanceImplicitGemmBwdDataV1R1(a, b, c, d, e, f, false) + { + } + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV1R1(bool spare); + + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemmBwdDataV1R1& other) const; + + template + static void Visit(Self&& self, F f) + { + f(self.BlockSize, "BlockSize"); + f(self.GemmMPerBlock, "GemmMPerBlock"); + f(self.GemmNPerBlock, "GemmNPerBlock"); + f(self.GemmKPerBlock, "GemmKPerBlock"); + f(self.GemmMPerThread, "GemmMPerThread"); + f(self.GemmNPerThread, "GemmNPerThread"); + } + + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGridSize(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateBlockGemmPerformanceParameters() const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmABlockCopyPerformanceParameters(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmBBlockCopyPerformanceParameters(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateLdsNumberOfByte(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); +}; + +struct PerformanceImplicitGemmBwdDataV4R1 : PerfConfigBase +{ + int BlockSize; + + int GemmMPerBlock; + int GemmNPerBlock; + int GemmKPerBlock; + + int GemmMPerThread; + int GemmNPerThread; + + bool use_spare_set; + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1(int, int, int, int, int, int, bool); + + PerformanceImplicitGemmBwdDataV4R1() + : PerformanceImplicitGemmBwdDataV4R1(-1, -1, -1, -1, -1, -1, false) + { + } + + PerformanceImplicitGemmBwdDataV4R1(int a, int b, int c, int d, int e, int f) + : PerformanceImplicitGemmBwdDataV4R1(a, b, c, d, e, f, false) + { + } + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1(bool spare); + + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemmBwdDataV4R1& other) const; + + template + static void Visit(Self&& self, F f) + { + f(self.BlockSize, "BlockSize"); + f(self.GemmMPerBlock, "GemmMPerBlock"); + f(self.GemmNPerBlock, "GemmNPerBlock"); + f(self.GemmKPerBlock, "GemmKPerBlock"); + f(self.GemmMPerThread, "GemmMPerThread"); + f(self.GemmNPerThread, "GemmNPerThread"); + } + + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGridSize(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateBlockGemmPerformanceParameters() const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple MIOPEN_INTERNALS_EXPORT + CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); +}; + +struct PerformanceImplicitGemmBwdDataV4R1Xdlops + : PerfConfigBase +{ + int GemmNPerBlock; // 2^n[8..16] + int GemmMPerBlock; // 2^n[32..128] + int GemmKPerBlock; // 2^n[4..16] + + int GemmKPACKSize; // 2^[1..4] + + int GemmMPerWave; + int GemmNPerWave; + + // GemmAThreadCopyMoreGemmK is currently a fix value, is untunable + bool GemmAThreadCopyMoreGemmK; + bool GemmBThreadCopyMoreGemmKPack; + + bool use_spare_set; + MIOPEN_INTERNALS_EXPORT + PerformanceImplicitGemmBwdDataV4R1Xdlops(int, int, int, int, int, int, bool, bool, bool); + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops(); + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops(bool spare); + PerformanceImplicitGemmBwdDataV4R1Xdlops( + int a, int b, int c, int d, int e, int f, bool g, bool h) + : PerformanceImplicitGemmBwdDataV4R1Xdlops(a, b, c, d, e, f, g, h, false) + { + } + + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceImplicitGemmBwdDataV4R1Xdlops& other) const; + + template + static void Visit(Self&& self, F f) + { + f(self.GemmNPerBlock, "GemmNPerBlock"); + f(self.GemmMPerBlock, "GemmMPerBlock"); + f(self.GemmKPerBlock, "GemmKPerBlock"); + f(self.GemmKPACKSize, "GemmKPACKSize"); + f(self.GemmMPerWave, "GemmMPerWave"); + f(self.GemmNPerWave, "GemmNPerWave"); + f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); + f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); + } + + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGridSize(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool + IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); +}; + +struct ConvHipImplicitGemmV4R1Fwd final : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmV4R1&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmV4R1&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; +}; + +struct ConvHipImplicitGemmV4R4Fwd final : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4Fwd GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmV4R4Fwd&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4Fwd + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmV4R4Fwd&) const override; + +private: + static std::tuple CalculateGemmSize(const miopen::conv::ProblemDescription&); + + friend struct PerformanceImplicitGemmV4R4Fwd; +}; + +struct PerformanceConvMlirIgemm : PerfConfigBase +{ + int BlockSize; + int GemmMPerBlock; + int GemmNPerBlock; + int GemmKPerBlock; + int GemmMPerThread; + int GemmNPerThread; + bool use_spare_set; + + /// \ref https://github.com/ROCm/MIOpen/issues/1154 + static PerformanceConvMlirIgemm& MlirHeuristicInitRequest() + { + static PerformanceConvMlirIgemm heur; + heur.SetMlirHeuristicInitRequest(); + return heur; + } + + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm(int, int, int, int, int, int, bool); + + PerformanceConvMlirIgemm(int a, int b, int c, int d, int e, int f) + : PerformanceConvMlirIgemm(a, b, c, d, e, f, false) + { + } + + PerformanceConvMlirIgemm() : PerformanceConvMlirIgemm(-1, -1, -1, -1, -1, -1, false) {} + + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm(bool spare); + + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConvMlirIgemm& other) const; + + template + static void Visit(Self&& self, F f) + { + f(self.BlockSize, "BlockSize"); + f(self.GemmMPerBlock, "GemmMPerBlock"); + f(self.GemmNPerBlock, "GemmNPerBlock"); + f(self.GemmKPerBlock, "GemmKPerBlock"); + f(self.GemmMPerThread, "GemmMPerThread"); + f(self.GemmNPerThread, "GemmNPerThread"); + } + + MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + +private: + void SetMlirHeuristicInitRequest(); +}; + +struct ConvMlirIgemmFwd final : ConvTunableSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemm&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemm&) const override; +}; + +struct PerformanceConvMlirIgemmXdlops : PerfConfigBase +{ + int GemmMPerBlock; // 2^n[32..128] + int GemmNPerBlock; // 2^n[8..16] + int GemmKPerBlock; // 2^n[4..16] + int GemmMPerWave; + int GemmNPerWave; + int GemmKPACKSize; // 2^[1..4] + + // GemmAThreadCopyMoreGemmK is currently a fix value, is untunable + bool GemmAThreadCopyMoreGemmK; + bool GemmBThreadCopyMoreGemmKPack; + + bool use_spare_set; + + /// \ref https://github.com/ROCm/MIOpen/issues/1154 + static PerformanceConvMlirIgemmXdlops& MlirHeuristicInitRequest() + { + static PerformanceConvMlirIgemmXdlops heur; + heur.SetMlirHeuristicInitRequest(); + return heur; + } + + MIOPEN_INTERNALS_EXPORT + PerformanceConvMlirIgemmXdlops(int, int, int, int, int, int, bool, bool, bool); + + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops(); + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops(bool spare); + PerformanceConvMlirIgemmXdlops(int a, int b, int c, int d, int e, int f, bool g, bool h) + : PerformanceConvMlirIgemmXdlops(a, b, c, d, e, f, g, h, false) + { + } + + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConvMlirIgemmXdlops& other) const; + + template + static void Visit(Self&& self, F f) + { + f(self.GemmNPerBlock, "GemmNPerBlock"); + f(self.GemmMPerBlock, "GemmMPerBlock"); + f(self.GemmKPerBlock, "GemmKPerBlock"); + f(self.GemmMPerWave, "GemmMPerWave"); + f(self.GemmNPerWave, "GemmNPerWave"); + f(self.GemmKPACKSize, "GemmKPACKSize"); + f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); + f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); + } + + MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + +private: + void SetMlirHeuristicInitRequest(); +}; + +struct ConvMlirIgemmFwdXdlops final : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemmXdlops&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemmXdlops&) const override; +}; + +struct ConvHipImplicitGemmV4R4WrW final : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4WrW GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmV4R4WrW&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4WrW + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmV4R4WrW&) const override; + +private: + static std::tuple CalculateGemmSize(const miopen::conv::ProblemDescription&); + + friend struct PerformanceImplicitGemmV4R4WrW; +}; + +struct ConvMlirIgemmWrW final : ConvTunableSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemm&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemm&) const override; +}; + +struct ConvMlirIgemmWrWXdlops final : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemmXdlops&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool MayNeedWorkspace() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemmXdlops&) const override; +}; + +struct PerformanceImplicitGemmForwardV4R4Xdlops + : PerfConfigBase +{ + int GemmMPerBlock; + int GemmNPerBlock; + int GemmKPerBlock; + int GemmMPerWave; + int GemmNPerWave; + int GemmKPack; + bool GemmAThreadCopyMoreGemmK; + bool GemmBThreadCopyMoreGemmKPack; + int GemmBThreadDataPerRead_GemmN; + + MIOPEN_INTERNALS_EXPORT + PerformanceImplicitGemmForwardV4R4Xdlops(int, int, int, int, int, int, bool, bool, int); + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops(); + PerformanceImplicitGemmForwardV4R4Xdlops(bool) : PerformanceImplicitGemmForwardV4R4Xdlops() {} + + template + static void Visit(Self&& self, F f) + { + f(self.GemmMPerBlock, "GemmMPerBlock"); + f(self.GemmNPerBlock, "GemmNPerBlock"); + f(self.GemmKPerBlock, "GemmKPerBlock"); + f(self.GemmMPerWave, "GemmMPerWave"); + f(self.GemmNPerWave, "GemmNPerWave"); + f(self.GemmKPack, "GemmKPack"); + f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); + f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); + f(self.GemmBThreadDataPerRead_GemmN, "GemmBThreadDataPerRead_GemmN"); + } + + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceImplicitGemmForwardV4R4Xdlops& other) const; + + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool + IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + + MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGridSize(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; +}; + +struct PerformanceImplicitGemmForwardV4R5Xdlops + : PerfConfigBase +{ + int GemmMPerBlock; + int GemmNPerBlock; + int GemmKPerBlock; + int GemmMPerWave; + int GemmNPerWave; + int GemmKPack; + bool GemmAThreadCopyMoreGemmK; + bool GemmBThreadCopyMoreGemmKPack; + int GemmBThreadDataPerRead_GemmN; + + bool use_spare_set; + + MIOPEN_INTERNALS_EXPORT + PerformanceImplicitGemmForwardV4R5Xdlops(int, int, int, int, int, int, bool, bool, int, bool); + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops(); + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops(bool spare); + + PerformanceImplicitGemmForwardV4R5Xdlops( + int a, int b, int c, int d, int e, int f, bool g, bool h, int i) + : PerformanceImplicitGemmForwardV4R5Xdlops(a, b, c, d, e, f, g, h, i, false) + { + } + + template + static void Visit(Self&& self, F f) + { + f(self.GemmMPerBlock, "GemmMPerBlock"); + f(self.GemmNPerBlock, "GemmNPerBlock"); + f(self.GemmKPerBlock, "GemmKPerBlock"); + f(self.GemmMPerWave, "GemmMPerWave"); + f(self.GemmNPerWave, "GemmNPerWave"); + f(self.GemmKPack, "GemmKPack"); + f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); + f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); + f(self.GemmBThreadDataPerRead_GemmN, "GemmBThreadDataPerRead_GemmN"); + } + + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceImplicitGemmForwardV4R5Xdlops& other) const; + + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool + IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + + MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGridSize(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; +}; + +struct PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm + : PerfConfigBase +{ + int GemmMPerBlock; + int GemmNPerBlock; + int GemmKPerBlock; + int GemmMPerWave; + int GemmNPerWave; + int GemmKPack; + int GemmMFactor; + int GemmNFactor; + int GemmKFactor; + bool GemmAThreadCopyMoreGemmK; + bool GemmBThreadCopyMoreGemmKPack; + int GemmBThreadDataPerRead_GemmN; + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm( + int, int, int, int, int, int, int, int, int, bool, bool, int); + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm(); + PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm(bool) + : PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm() + { + } + + template + static void Visit(Self&& self, F f) + { + f(self.GemmMPerBlock, "GemmMPerBlock"); + f(self.GemmNPerBlock, "GemmNPerBlock"); + f(self.GemmKPerBlock, "GemmKPerBlock"); + f(self.GemmMPerWave, "GemmMPerWave"); + f(self.GemmNPerWave, "GemmNPerWave"); + f(self.GemmKPack, "GemmKPack"); + f(self.GemmMFactor, "GemmMFactor"); + f(self.GemmNFactor, "GemmNFactor"); + f(self.GemmKFactor, "GemmKFactor"); + f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); + f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); + f(self.GemmBThreadDataPerRead_GemmN, "GemmBThreadDataPerRead_GemmN"); + } + + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm& other) const; + + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool + IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + + MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGridSize(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; +}; + +struct PerformanceImplicitGemmBwdV1R1Xdlops : PerfConfigBase +{ + int GemmMPerBlock; + int GemmNPerBlock; + int GemmKPerBlock; + int GemmMPerWave; + int GemmNPerWave; + int GemmKPack; + bool GemmAThreadCopyMoreGemmK; + bool GemmBThreadCopyMoreGemmKPack; + + MIOPEN_INTERNALS_EXPORT + PerformanceImplicitGemmBwdV1R1Xdlops(int, int, int, int, int, int, bool, bool); + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdV1R1Xdlops(); + PerformanceImplicitGemmBwdV1R1Xdlops(bool) : PerformanceImplicitGemmBwdV1R1Xdlops() {} + + template + static void Visit(Self&& self, F f) + { + f(self.GemmMPerBlock, "GemmMPerBlock"); + f(self.GemmNPerBlock, "GemmNPerBlock"); + f(self.GemmKPerBlock, "GemmKPerBlock"); + f(self.GemmMPerWave, "GemmMPerWave"); + f(self.GemmNPerWave, "GemmNPerWave"); + f(self.GemmKPack, "GemmKPack"); + f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); + f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); + } + + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceImplicitGemmBwdV1R1Xdlops& other) const; + + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool + IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + + MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGridSize(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; +}; + +struct ConvHipImplicitGemmForwardV4R4Xdlops final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmForwardV4R4Xdlops&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmForwardV4R4Xdlops&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + +private: + static std::tuple + CalculateGemmSize(const miopen::conv::ProblemDescription&); + + friend struct PerformanceImplicitGemmForwardV4R4Xdlops; +}; + +struct ConvHipImplicitGemmForwardV4R4Xdlops_Padded_Gemm final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + +private: + static std::tuple CalculateGemmSize( + const miopen::conv::ProblemDescription&, int GemmMFactor, int GemmNFactor, int GemmKFactor); + + friend struct PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm; +}; + +struct ConvHipImplicitGemmForwardV4R5Xdlops final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmForwardV4R5Xdlops&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmForwardV4R5Xdlops&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; +}; + +struct ConvHipImplicitGemmV4R1WrW final : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmV4R1&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmV4R1&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; +}; + +struct ConvHipImplicitGemmBwdDataV1R1 final : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV1R1 GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmBwdDataV1R1&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV1R1 + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmBwdDataV1R1&) const override; + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool MayNeedWorkspace() const override { return true; } + +private: + static std::tuple CalculateGemmSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + + friend struct PerformanceImplicitGemmBwdDataV1R1; +}; + +struct ConvMlirIgemmBwd final : ConvTunableSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemm&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemm&) const override; +}; + +struct ConvMlirIgemmBwdXdlops final : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemmXdlops&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvMlirIgemmXdlops&) const override; +}; + +struct ConvHipImplicitGemmBwdDataV4R1 final : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1 GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmBwdDataV4R1&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1 + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmBwdDataV4R1&) const override; + +private: + static int CalculateNumberOfGemm(const miopen::conv::ProblemDescription&); + static std::tuple CalculateGemmSize(const miopen::conv::ProblemDescription&, + int gemm_id); + + friend struct PerformanceImplicitGemmBwdDataV4R1; +}; + +struct ConvHipImplicitGemmBwdDataV4R1Xdlops final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmBwdDataV4R1Xdlops&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmBwdDataV4R1Xdlops&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + +private: + static int CalculateNumberOfGemm(const miopen::conv::ProblemDescription&); + static std::tuple CalculateGemmSize(const miopen::conv::ProblemDescription&, + int gemm_id); + + friend struct PerformanceImplicitGemmBwdDataV4R1Xdlops; +}; + +struct ConvHipImplicitGemmBwdDataV1R1Xdlops final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdV1R1Xdlops GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmBwdV1R1Xdlops&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdV1R1Xdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmBwdV1R1Xdlops&) const override; + +private: + static std::tuple + CalculateGemmSize(const miopen::conv::ProblemDescription&); + + friend struct PerformanceImplicitGemmBwdV1R1Xdlops; +}; + +struct ConvAsmImplicitGemmV4R1DynamicFwd final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + bool IsDynamic() const override { return true; } + + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct ConvAsmImplicitGemmV4R1DynamicFwd_1x1 final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + bool IsDynamic() const override { return true; } + + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct ConvAsmImplicitGemmV4R1DynamicWrw final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + bool IsDynamic() const override { return true; } + + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + bool MayNeedWorkspace() const override { return true; } + + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct ConvAsmImplicitGemmGTCDynamicWrwXdlops final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + bool IsDynamic() const override { return true; } + + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + bool MayNeedWorkspace() const override { return true; } + + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct ConvAsmImplicitGemmV4R1DynamicBwd final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + bool IsDynamic() const override { return true; } + + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct ConvAsmImplicitGemmGTCDynamicFwdXdlops final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + bool IsDynamic() const override { return true; } + + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct ConvAsmImplicitGemmGTCDynamicBwdXdlops final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + bool IsDynamic() const override { return true; } + + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +/// Holds common member functions for the Solvers which share the same +/// "legacy exhaustive search" machinery. +struct ConvOclDirectFwdLegacyExhaustiveSearch : ConvTunableSolver +{ + LegacyPerformanceConfig + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + LegacyPerformanceConfig Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + +private: + template + LegacyPerformanceConfig SearchImpl(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const; +}; + +struct MIOPEN_INTERNALS_EXPORT ConvOclDirectFwd final : ConvOclDirectFwdLegacyExhaustiveSearch +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + static ConvSolution BaseGetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const LegacyPerformanceConfig&); + + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const LegacyPerformanceConfig&) const override; + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const LegacyPerformanceConfig&) const override; +}; + +struct MIOPEN_INTERNALS_EXPORT ConvOclDirectFwd1x1 final : ConvOclDirectFwdLegacyExhaustiveSearch +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const LegacyPerformanceConfig&) const override; + + bool IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const LegacyPerformanceConfig&) const override + { + return true; + } +}; + +struct ConvBinWinograd3x3U final : ConvSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + bool IsDynamic() const override { return true; } + + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct ConvBinWinogradRxS final : ConvSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + bool IsDynamic() const override { return true; } + + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct PerformanceConfigConvBinWinogradRxS : PerfConfigBase +{ + int n_groups; + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvBinWinogradRxS(int n_groups_); + PerformanceConfigConvBinWinogradRxS() : PerformanceConfigConvBinWinogradRxS(-1) {} + PerformanceConfigConvBinWinogradRxS(bool) : PerformanceConfigConvBinWinogradRxS(1) {} + + template + static void Visit(Self&& self, F f) + { + f(self.n_groups, "n_groups"); + } + int GetNGroups() const { return n_groups; } + + template + void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValid(const ExecutionContext& ctx, const miopen::conv::ProblemDescription&) const + { + return IsValid(ctx); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&) const; + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvBinWinogradRxS& other) const; +}; + +template +struct ConvBinWinoRxS final : ConvTunableSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + static const std::string& GetSolverDbId() + { + static const std::string dbId = std::string("ConvBinWinogradRxSf") + .append(std::to_string(Winodata)) + .append("x") + .append(std::to_string(Winofilter)); + return dbId; + } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvBinWinogradRxS GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvBinWinogradRxS&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvBinWinogradRxS + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvBinWinogradRxS&) const override; + +private: + static size_t GetNGroups(const size_t group_conv, const size_t grid_group_size) + { + assert(group_conv != 0); + return grid_group_size / group_conv; + } +}; + +// Suppress misleading clang warnings +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wweak-template-vtables" +#endif + +extern template struct ConvBinWinoRxS<2, 3>; +extern template struct ConvBinWinoRxS<3, 2>; + +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + +struct ConvBinWinogradRxSf2x3g1 final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT float GetWti(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +template +struct ConvMPBidirectWinograd final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId< + ConvMPBidirectWinograd>(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + // kernel_file_name for solver identification + static fs::path GetSolverFileNames(int id) + { + static const fs::path names[3] = {"xform_bidirect_winograd_data.s", + "xform_bidirect_winograd_filter.s", + "xform_bidirect_winograd_out.s"}; + return names[id]; + } + + static std::string GetSolverKernelNames(int id) + { + static const std::string name_suffix = + '_' + std::to_string(WinoDataH) + '_' + std::to_string(WinoDataW) + '_' + + std::to_string(WinoFilterH) + '_' + std::to_string(WinoFilterW); + static const std::string names[3] = { + "miopenGcnAsmMPBidirectWinogradXformData" + name_suffix, + "miopenGcnAsmMPBidirectWinogradXformFilter" + name_suffix, + "miopenGcnAsmMPBidirectWinogradXformOut" + name_suffix}; + return names[id]; + } + + static int GetSolverWinoXformHWSize() { return WinoDataH + WinoFilterH - 1; } +}; + +// To suppress misleading clang warnings +#if defined(__clang__) && defined(CONV_MP_BIDIRECTIONAL_WINOGRAD_CPP) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wweak-template-vtables" +#endif + +extern template struct ConvMPBidirectWinograd<2, 3>; +extern template struct ConvMPBidirectWinograd<3, 3>; +extern template struct ConvMPBidirectWinograd<4, 3>; +extern template struct ConvMPBidirectWinograd<5, 3>; +extern template struct ConvMPBidirectWinograd<6, 3>; + +#if defined(__clang__) && defined(CONV_MP_BIDIRECTIONAL_WINOGRAD_CPP) +#pragma clang diagnostic pop +#endif + +template +struct ConvMPBidirectWinograd_xdlops final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId< + ConvMPBidirectWinograd_xdlops>(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + bool IsDynamic() const override + { + return ConvHipImplicitGemmForwardV4R4Xdlops{}.IsDynamic() && + ConvMPBidirectWinograd{} + .IsDynamic() && + IsThisSolverDynamic(); + } + + PerformanceImplicitGemmForwardV4R4Xdlops + GetDefaultPerformanceConfig(const ExecutionContext& ctx, + const miopen::conv::ProblemDescription& problem) const override + { + const auto xdlops_problem = GetTransformedProblem(problem); + const auto xdlops_ctx = GetTransformedConvContext(ctx, xdlops_problem); + + return ConvHipImplicitGemmForwardV4R4Xdlops{}.GetDefaultPerformanceConfig(xdlops_ctx, + xdlops_problem); + } + + bool + IsValidPerformanceConfig(const ExecutionContext& ctx, + const miopen::conv::ProblemDescription& problem, + const PerformanceImplicitGemmForwardV4R4Xdlops& config) const override + { + const auto xdlops_problem = GetTransformedProblem(problem); + const auto xdlops_ctx = GetTransformedConvContext(ctx, xdlops_problem); + + return ConvHipImplicitGemmForwardV4R4Xdlops{}.IsValidPerformanceConfig( + xdlops_ctx, xdlops_problem, config); + } + + size_t GetWorkspaceSize(const ExecutionContext& ctx, + const miopen::conv::ProblemDescription& problem) const override + { + const auto xdlops_problem = GetTransformedProblem(problem); + const auto xdlops_ctx = GetTransformedConvContext(ctx, xdlops_problem); + + return ConvMPBidirectWinograd() + .GetWorkspaceSize(ctx, problem) + + ConvHipImplicitGemmForwardV4R4Xdlops{}.GetWorkspaceSize(xdlops_ctx, xdlops_problem); + } + + bool MayNeedWorkspace() const override { return true; } + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmForwardV4R4Xdlops&) const override; + +private: + ExecutionContext + GetTransformedConvContext(const ExecutionContext& ctx, + const miopen::conv::ProblemDescription& transformed_problem) const; + miopen::conv::ProblemDescription + GetTransformedProblem(const miopen::conv::ProblemDescription& problem) const; + + // kernel_file_name for solver identification + static fs::path GetSolverFileNames(int id) + { + return ConvMPBidirectWinograd:: + GetSolverFileNames(id); + } + + static std::string GetSolverKernelNames(int id) + { + return ConvMPBidirectWinograd:: + GetSolverKernelNames(id); + } + + static int GetSolverWinoXformHWSize() + { + return ConvMPBidirectWinograd:: + GetSolverWinoXformHWSize(); + } + + bool IsThisSolverDynamic() const { return true; } +}; + +// To suppress misleading clang warnings +#if defined(__clang__) && defined(CONV_MP_BIDIRECTIONAL_WINOGRAD_CPP) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wweak-template-vtables" +#endif + +extern template struct ConvMPBidirectWinograd_xdlops<2, 3>; +extern template struct ConvMPBidirectWinograd_xdlops<3, 3>; +extern template struct ConvMPBidirectWinograd_xdlops<4, 3>; +extern template struct ConvMPBidirectWinograd_xdlops<5, 3>; +extern template struct ConvMPBidirectWinograd_xdlops<6, 3>; + +#if defined(__clang__) && defined(CONV_MP_BIDIRECTIONAL_WINOGRAD_CPP) +#pragma clang diagnostic pop +#endif + +template +struct ConvWinograd3x3MultipassWrW final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId< + ConvWinograd3x3MultipassWrW>(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + bool IsDynamic() const override { return true; } + + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + bool MayNeedWorkspace() const override { return true; } + + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + // kernel_file_name for solver identification + static fs::path GetSolverFileNames(int id) + { + static const fs::path names[3] = {"xform_data.s", "xform_filter.s", "xform_out.s"}; + return names[id]; + } + + static std::string GetSolverKernelNames(int id) + { + static const std::string name_suffix = + '_' + std::to_string(WinoDataH) + '_' + std::to_string(WinoDataW) + '_' + + std::to_string(WinoFilterH) + '_' + std::to_string(WinoFilterW); + static const std::string names[3] = {"miopenGcnAsmWinogradXformData" + name_suffix, + "miopenGcnAsmWinogradXformFilter" + name_suffix, + "miopenGcnAsmWinogradXformOut" + name_suffix}; + + return names[id]; + } + + static int GetGroupCountMult() { return 4; } + + static int GetSolverWinoXformHWSize(const miopen::conv::ProblemDescription& problem, int id) + { + if(id == 0) + { + return WinoDataH + + (WinoFilterH - 1) * (WinoDataH == 7 ? 2 : problem.GetKernelStrideH()); + } + else + { + return WinoDataW + + (WinoFilterW - 1) * (WinoDataW == 7 ? 2 : problem.GetKernelStrideW()); + } + } + +private: + InvokerFactory PrepareInvokerFactory(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + std::size_t ws_sz) const; +}; + +// To suppress misleading clang warnings +#if defined(__clang__) && defined(CONV_MULTIPASS_WINO3X3WRW_CPP) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wweak-template-vtables" +#endif + +extern template struct ConvWinograd3x3MultipassWrW<3, 2>; +extern template struct ConvWinograd3x3MultipassWrW<3, 3>; +extern template struct ConvWinograd3x3MultipassWrW<3, 4>; +extern template struct ConvWinograd3x3MultipassWrW<3, 5>; +extern template struct ConvWinograd3x3MultipassWrW<3, 6>; +extern template struct ConvWinograd3x3MultipassWrW<7, 2>; +extern template struct ConvWinograd3x3MultipassWrW<7, 3>; +extern template struct ConvWinograd3x3MultipassWrW<1, 1, 7, 2>; +extern template struct ConvWinograd3x3MultipassWrW<1, 1, 7, 3>; +extern template struct ConvWinograd3x3MultipassWrW<7, 2, 1, 1>; +extern template struct ConvWinograd3x3MultipassWrW<7, 3, 1, 1>; +extern template struct ConvWinograd3x3MultipassWrW<5, 3>; +extern template struct ConvWinograd3x3MultipassWrW<5, 4>; + +#if defined(__clang__) && defined(CONV_MULTIPASS_WINO3X3WRW_CPP) +#pragma clang diagnostic pop +#endif + +struct PerformanceConfigAsmDirect3x3WrW : PerfConfigBase +{ + int limit_wave_cnt; // [0..9] + int reverse_inout; // [0..1], 1 is allowed for stride=1x1 only. + int chunk_size; // {16,8}, Smaller values increase register pressure. + int k_per_wave; // {1,2,4,8} && ((chunk_size * k_per_wave) <= 64). + // Higher values increase register pressure. + int pipe_lines_depth; // [1..16] && (pipe_lines_depth <= img_h). + // Higher values increase register pressure. + int n_per_group; // [1..8] && (n_per_group <= batch_size). + + PerformanceConfigAsmDirect3x3WrW(int lwc, int rio, int csz, int kpw, int pld, int npg); + PerformanceConfigAsmDirect3x3WrW() : PerformanceConfigAsmDirect3x3WrW(-1, -1, -1, -1, -1, -1) {} + PerformanceConfigAsmDirect3x3WrW(bool) : PerformanceConfigAsmDirect3x3WrW(0, 0, 8, 1, 1, 1) {} + + template + static void Visit(Self&& self, F f) + { + f(self.limit_wave_cnt, "limit_wave_cnt"); + f(self.reverse_inout, "reverse_inout"); + f(self.chunk_size, "chunk_size"); + f(self.k_per_wave, "k_per_wave"); + f(self.pipe_lines_depth, "pipe_lines_depth"); + f(self.n_per_group, "n_per_group"); + } + + int GetLimitWaveCnt() const { return limit_wave_cnt; } + int GetReverseInout() const { return reverse_inout; } + int GetChunkSize() const { return chunk_size; } + int GetKPerWave() const { return k_per_wave; } + int GetPipeLinesDepth() const { return pipe_lines_depth; } + int GetNPerGroup() const { return n_per_group; } + int GetCPerWave() const + { + assert(chunk_size); + return 64 / chunk_size; + } + + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigAsmDirect3x3WrW& other) const; +}; + +struct ConvAsmBwdWrW3x3 final : ConvTunableSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmDirect3x3WrW GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigAsmDirect3x3WrW&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmDirect3x3WrW + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigAsmDirect3x3WrW& config) const override; +}; + +template +struct ConvWinoFuryRxS final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId>(); + } + + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; +}; + +#ifndef CONV_WINO_FURY_RXS_CPP +extern template struct ConvWinoFuryRxS<2, 3>; +// extern template struct ConvWinoFuryRxS<3, 2>; +#endif + +struct PerformanceConfigConvAsmBwdWrW1x1 : PerfConfigBase +{ + + int chunk_size; // {1,2,4,8,16} + int c_per_gpr; // {1,2,4,8,16} + int c_mult; // {1,2,4,8,16} + int k_per_gpr; // {1,2,4,8,16} + int k_mult; // {1,2,4,8,16} + int n_per_gpr; // {1,2,4} + int n_part_cnt; // [1..8] + int read_size; // [1..4] + int short_store; // {0,1} + int data_prefetch; // [0..4] + bool use_spare_set; + + /// The following conditions must be met. + /// + /// Shader design-related constraints: + /// - (A) (chunk_size * c_per_gpr) == 16 + /// - (B) k_per_gpr <= c_per_gpr + /// - (C) (c_mult > 1 || k_mult > 1) + /// ? ((fwd_C % (c_per_gpr * c_mult) == 0) && (fwd_K % (k_per_gpr * k_mult) == 0)) + /// : (true) + /// + /// Resource-related constraints: + /// - (D) c_mult * k_mult * k_per_gpr + 9 + (c_mult + k_mult) * read_size * pipe_depth <= 256 + /// + /// Where: + /// - fwd_C := Num input channels for forward convolution (-c). + /// For backward, this is actually n_outputs. + /// - fwd_K := Num output channels for forward convolution (-k). + /// For backward, this is actually n_inputs. + + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsmBwdWrW1x1(int chunk_size_, + int c_per_gpr_, + int c_mult_, + int k_per_gpr_, + int k_mult_, + int n_per_gpr_, + int n_part_cnt_, + int read_size_, + int short_store_, + int data_prefetch_, + bool); + PerformanceConfigConvAsmBwdWrW1x1() + : PerformanceConfigConvAsmBwdWrW1x1(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, false) + { + } + PerformanceConfigConvAsmBwdWrW1x1(bool spare) + : PerformanceConfigConvAsmBwdWrW1x1(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, spare) + { + } + + template + static void Visit(Self&& self, F f) + { + f(self.chunk_size, "chunk_size"); + f(self.c_per_gpr, "c_per_gpr"); + f(self.c_mult, "c_mult"); + f(self.k_per_gpr, "k_per_gpr"); + f(self.k_mult, "k_mult"); + f(self.n_per_gpr, "n_per_gpr"); + f(self.n_part_cnt, "n_part_cnt"); + f(self.read_size, "read_size"); + f(self.short_store, "short_store"); + f(self.data_prefetch, "data_prefetch"); + } + + int GetChunkSize() const { return chunk_size; } + int GetCPerGpr() const { return c_per_gpr; } + int GetCMult() const { return c_mult; } + int GetKPerGpr() const { return k_per_gpr; } + int GetKMult() const { return k_mult; } + int GetNPerGpr() const { return n_per_gpr; } + int GetNPartCnt() const { return n_part_cnt; } + // "hw" stands for "height-and-width". + int GetHWPerGpr() const + { + assert(c_per_gpr); + assert(n_per_gpr); + assert(chunk_size); + return wave_size / (c_per_gpr * n_per_gpr * chunk_size); + } + int GetReadSize() const { return read_size; } + int GetShortStore() const { return short_store; } + int GetDataPrefetch() const { return data_prefetch; } + + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvAsmBwdWrW1x1& other) const; +}; + +struct ConvAsmBwdWrW1x1 final : ConvTunableSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsmBwdWrW1x1 MIOPEN_INTERNALS_EXPORT + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvAsmBwdWrW1x1&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsmBwdWrW1x1 + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvAsmBwdWrW1x1&) const override; +}; + +/// N_BATCH_LOOPS - {1,2,4,8,16} Num batches processed in single workitem. +/// Required workspace size depends on it. However there is a restriction in the internal +/// Solver API that this shouldn't be so. Therefore the family of Solvers created. +/// Each Solver in the family has constant value of this parameter. +template +struct PerformanceConfigConvOclBwdWrw2 + : PerfConfigBase> +{ + // Num waves involved a workgroup. + int n_waves = -1; // {1,2,4,8} + // Num values to read in a workitem (read_unit). + int read_size = -1; // [6..12] + // Num of output channels (top/bottom layer in forward/backward direction) + // that share the same input channel in single workgroup. + // Also represents number of output channels in single tile. + int n_out_channels_per_tile = -1; // {1,2,4,8} + // How many tiles of output channels are processed in a single workgroup? + // n_out_channels_in_lcl * n_out_channels_tiles = total number of + // output channels processed in single workgroup. + int n_out_channels_tiles = -1; // {1,2,4,8} + // Num of output rows processed in a single iteration of loop in a workitem + // (N_ALIGNED_OUT_SCAN_BLK). + int n_out_rows_in_lcl = -1; // [2..11] + + PerformanceConfigConvOclBwdWrw2(int nw, int rs, int nocpt, int noct, int noril) + : n_waves(nw), + read_size(rs), + n_out_channels_per_tile(nocpt), + n_out_channels_tiles(noct), + n_out_rows_in_lcl(noril) + { + } + PerformanceConfigConvOclBwdWrw2() {} + PerformanceConfigConvOclBwdWrw2(bool) : PerformanceConfigConvOclBwdWrw2(1, 6, 1, 1, 2) {} + // spare_set is not used in this solver. + + template + static void Visit(Self&& self, F f) + { + f(self.n_waves, "n_waves"); + f(self.read_size, "read_size"); + f(self.n_out_channels_per_tile, "n_out_channels_per_tile"); + f(self.n_out_channels_tiles, "n_out_channels_tiles"); + f(self.n_out_rows_in_lcl, "n_out_rows_in_lcl"); + } + + int GetNumWaves() const { return n_waves; } + int GetReadSize() const { return read_size; } + int GetNumOutChannelsPerTile() const { return n_out_channels_per_tile; } + int GetNumOutChannelTiles() const { return n_out_channels_tiles; } + int GetNumOutRowsPerIterPerWork() const { return n_out_rows_in_lcl; } + + void HeuristicInit(const miopen::conv::ProblemDescription&); + bool IsValidValue() const; + bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + bool operator==(const PerformanceConfigConvOclBwdWrw2& other) const; +}; + +template +struct ConvOclBwdWrW2 final : ConvTunableSolver> +{ + MIOPEN_INTERNALS_EXPORT const std::string& SolverDbId() const override + { + return this->template GetSolverDbId>(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvOclBwdWrw2 + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvOclBwdWrw2&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigConvOclBwdWrw2 + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool MayNeedWorkspace() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigConvOclBwdWrw2&) const override; + +protected: + bool IsApplicableBase(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + + friend struct ConvOclBwdWrW2NonTunable; +}; + +// To suppress misleading clang warnings +#if defined(__clang__) && defined(CONV_OCL_DIR2D_BWDWRW_2_CPP) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wweak-template-vtables" +#endif + +extern template struct PerformanceConfigConvOclBwdWrw2<1>; +extern template struct PerformanceConfigConvOclBwdWrw2<2>; +extern template struct PerformanceConfigConvOclBwdWrw2<4>; +extern template struct PerformanceConfigConvOclBwdWrw2<8>; +extern template struct PerformanceConfigConvOclBwdWrw2<16>; + +extern template struct ConvOclBwdWrW2<1>; +extern template struct ConvOclBwdWrW2<2>; +extern template struct ConvOclBwdWrW2<4>; +extern template struct ConvOclBwdWrW2<8>; +extern template struct ConvOclBwdWrW2<16>; + +#if defined(__clang__) && defined(CONV_OCL_DIR2D_BWDWRW_2_CPP) +#pragma clang diagnostic pop +#endif + +/// A separate solver from ConvOclBwdWrW2 to disable auto-tuning for certain configs. +/// Basically, this is *hack* for non-group 3x3 and 1x1 cases. +/// It is assumed that Solutions provided by the ConvOclBwdWrW2 solver +/// would never beat 3x3 and 1x1 assembly WrW kernels, even after tuning. +struct MIOPEN_INTERNALS_EXPORT ConvOclBwdWrW2NonTunable final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; +}; + +struct ConvOclBwdWrW53 final : ConvSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct ConvOclBwdWrW1x1 final : ConvSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct fft final : ConvSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + + bool MayNeedWorkspace() const override { return true; } + + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct PerformanceImplicitGemmWrwV4R4Xdlops : PerfConfigBase +{ + int GemmMPerBlock; + int GemmNPerBlock; + int GemmKPerBlock; + int GemmMPerWave; + int GemmNPerWave; + int GemmKPack; + bool GemmAThreadCopyMoreGemmK; + bool GemmBThreadCopyMoreGemmK; + bool use_spare_set; + + MIOPEN_INTERNALS_EXPORT + PerformanceImplicitGemmWrwV4R4Xdlops(int, int, int, int, int, int, bool, bool, bool); + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops(); + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops(bool spare); + PerformanceImplicitGemmWrwV4R4Xdlops(int a, int b, int c, int d, int e, int f, bool g, bool h) + : PerformanceImplicitGemmWrwV4R4Xdlops(a, b, c, d, e, f, g, h, false) + { + } + + template + static void Visit(Self&& self, F f) + { + f(self.GemmMPerBlock, "GemmMPerBlock"); + f(self.GemmNPerBlock, "GemmNPerBlock"); + f(self.GemmKPerBlock, "GemmKPerBlock"); + f(self.GemmMPerWave, "GemmMPerWave"); + f(self.GemmNPerWave, "GemmNPerWave"); + f(self.GemmKPack, "GemmKPack"); + f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); + f(self.GemmBThreadCopyMoreGemmK, "GemmBThreadCopyMoreGemmK"); + } + + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceImplicitGemmWrwV4R4Xdlops& other) const; + + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool + IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmSizeAndGemmKBlock(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGridSize(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; +}; + +struct ConvHipImplicitGemmWrwV4R4Xdlops final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmWrwV4R4Xdlops&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmWrwV4R4Xdlops&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; +}; + +struct PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm + : PerfConfigBase +{ + int GemmMPerBlock; + int GemmNPerBlock; + int GemmKPerBlock; + int GemmMPerWave; + int GemmNPerWave; + int GemmKPack; + int GemmMFactor; + int GemmNFactor; + int GemmKTotalFactor; + bool GemmAThreadCopyMoreGemmK; + bool GemmBThreadCopyMoreGemmK; + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm( + int, int, int, int, int, int, int, int, int, bool, bool); + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm(); + PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm(bool) + : PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm() + { + } + + template + static void Visit(Self&& self, F f) + { + f(self.GemmMPerBlock, "GemmMPerBlock"); + f(self.GemmNPerBlock, "GemmNPerBlock"); + f(self.GemmKPerBlock, "GemmKPerBlock"); + f(self.GemmMPerWave, "GemmMPerWave"); + f(self.GemmNPerWave, "GemmNPerWave"); + f(self.GemmKPack, "GemmKPack"); + f(self.GemmMFactor, "GemmMFactor"); + f(self.GemmNFactor, "GemmNFactor"); + f(self.GemmKTotalFactor, "GemmKTotalFactor"); + f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); + f(self.GemmBThreadCopyMoreGemmK, "GemmBThreadCopyMoreGemmK"); + } + + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm& other) const; + + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool + IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; + + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmSizeAndGemmKBlock(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; + std::tuple CalculateGridSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT std::tuple + CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; +}; + +struct ConvHipImplicitGemmWrwV4R4Xdlops_Padded_Gemm final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; +}; + +struct PerformanceConvCkIgemmFwdV6r1DlopsNchw + : PerfConfigBase +{ + int ck_tunable_list_id; + + PerformanceConvCkIgemmFwdV6r1DlopsNchw(int a) : ck_tunable_list_id(a) {} + + PerformanceConvCkIgemmFwdV6r1DlopsNchw() : PerformanceConvCkIgemmFwdV6r1DlopsNchw(-1) {} + + PerformanceConvCkIgemmFwdV6r1DlopsNchw(bool) : PerformanceConvCkIgemmFwdV6r1DlopsNchw(0) {} + + template + static void Visit(Self&& self, F f) + { + f(self.ck_tunable_list_id, "ck_tunable_list_id"); + } + + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + bool operator==(const PerformanceConvCkIgemmFwdV6r1DlopsNchw& config) const + { + return ck_tunable_list_id == config.ck_tunable_list_id; + } +}; + +struct ConvCkIgemmFwdV6r1DlopsNchw final : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + MIOPEN_INTERNALS_EXPORT PerformanceConvCkIgemmFwdV6r1DlopsNchw GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvCkIgemmFwdV6r1DlopsNchw&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConvCkIgemmFwdV6r1DlopsNchw + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConvCkIgemmFwdV6r1DlopsNchw&) const override; +}; + +struct ConvDirectNaiveConvFwd final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + /// Use very small fixed value enough to backup GEMM for cases when + /// GEMM is disabled. + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override + { + return 0.01f; + } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct ConvDirectNaiveConvBwd final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + /// Use very small fixed value enough to backup GEMM for cases when + /// GEMM is disabled. + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override + { + return 0.01f; + } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct ConvDirectNaiveConvWrw final : ConvSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + /// Use very small fixed value enough to backup GEMM for cases when + /// GEMM is disabled. + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override + { + return 0.01f; + } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; +}; + +struct GemmFwdBase : ConvSolver +{ + bool IsDynamic() const override { return true; } + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + +private: + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + friend struct GemmFwd1x1_0_2; + friend struct GemmFwd1x1_0_1_int8; + friend struct GemmFwd1x1_0_1; + friend struct GemmFwdRest; +}; + +struct MIOPEN_INTERNALS_EXPORT GemmFwd1x1_0_2 final : GemmFwdBase +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + bool MayNeedWorkspace() const override { return true; } + + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + friend struct GemmFwdRest; +}; + +struct MIOPEN_INTERNALS_EXPORT GemmFwd1x1_0_1_int8 final : GemmFwdBase +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + bool MayNeedWorkspace() const override { return true; } + + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + friend struct GemmFwdRest; +}; + +struct MIOPEN_INTERNALS_EXPORT GemmFwd1x1_0_1 final : GemmFwdBase +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + bool MayNeedWorkspace() const override { return true; } + + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + friend struct GemmFwdRest; +}; + +struct MIOPEN_INTERNALS_EXPORT GemmFwdRest final : GemmFwdBase +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + bool MayNeedWorkspace() const override { return true; } + + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; +}; + +struct GemmBwdBase : ConvSolver +{ + bool IsDynamic() const override { return true; } + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + +private: + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + friend struct GemmBwd1x1_stride2; + friend struct GemmBwd1x1_stride1; + friend struct GemmBwdRest; +}; + +struct MIOPEN_INTERNALS_EXPORT GemmBwd1x1_stride2 final : GemmBwdBase +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + bool MayNeedWorkspace() const override { return true; } + + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + friend struct GemmBwdRest; +}; + +struct MIOPEN_INTERNALS_EXPORT GemmBwd1x1_stride1 final : GemmBwdBase +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + bool MayNeedWorkspace() const override { return true; } + + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription& problem) const override; + + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription& problem) const override; + + friend struct GemmBwdRest; +}; + +struct MIOPEN_INTERNALS_EXPORT GemmBwdRest final : GemmBwdBase +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + bool MayNeedWorkspace() const override { return true; } + + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; +}; + +struct GemmWrwBase : ConvSolver +{ + bool IsDynamic() const override { return true; } + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + +private: + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + friend struct GemmWrw1x1_stride1; + friend struct GemmWrwUniversal; +}; + +struct MIOPEN_INTERNALS_EXPORT GemmWrw1x1_stride1 final : GemmWrwBase +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + friend struct GemmWrwUniversal; +}; + +struct MIOPEN_INTERNALS_EXPORT GemmWrwUniversal final : GemmWrwBase +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + size_t GetWorkspaceSize(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + bool MayNeedWorkspace() const override { return true; } + + bool IsApplicable(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + + ConvSolution GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; +}; + +struct PerformanceConfigAsmImplicitGemmGTC : PerfConfigBase +{ + std::string direction; + std::string tensor_layout; + std::string precision; + int nxb; + int nxe; + + int gemm_m_per_block; + int gemm_n_per_block; + int gemm_k_per_block; + + int wave_tile_m; + int wave_tile_n; + int wave_tile_k; + int wave_step_m; + int wave_step_n; + int wave_repeat_m; + int wave_repeat_n; + + int multihead; + int vector_store; + int gemm_k_global_split; + int merge_e; + int tensor_a_pass_through; + + std::vector tensor_a_thread_lengths; + std::vector tensor_a_cluster_lengths; + std::vector tensor_b_thread_lengths; + std::vector tensor_b_cluster_lengths; + + bool use_spare_set; + int index; + + MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTC(std::string dir, + std::string layout, + std::string prec, + int b, + int e, + int mpb, + int npb, + int kpb, + int wtm, + int wtn, + int wtk, + int wsm, + int wsn, + int wrm, + int wrn, + int mh, + int vs, + int gks, + int me, + int pta, + std::initializer_list ta_t, + std::initializer_list ta_c, + std::initializer_list tb_t, + std::initializer_list tb_c, + bool spare = false); + MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTC(std::string dir, + std::string layout, + miopenDataType_t prec, + int b, + int e, + int mpb, + int npb, + int kpb, + int wtm, + int wtn, + int wtk, + int wsm, + int wsn, + int wrm, + int wrn, + int mh, + int vs, + int gks, + int me, + int pta, + std::initializer_list ta_t, + std::initializer_list ta_c, + std::initializer_list tb_t, + std::initializer_list tb_c, + bool spare = false); + PerformanceConfigAsmImplicitGemmGTC() + : PerformanceConfigAsmImplicitGemmGTC("fwd", + "nchw", + "fp32", + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + false) + { + } + PerformanceConfigAsmImplicitGemmGTC(bool spare) + : PerformanceConfigAsmImplicitGemmGTC("fwd", + "nchw", + "fp32", + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + spare) + { + } + + template + static void Visit(Self&& self, F f) + { + f(self.direction, "dir"); + f(self.tensor_layout, "lyt"); + f(self.precision, "pre"); + f(self.nxb, "nxb"); + f(self.nxe, "nxe"); + f(self.gemm_m_per_block, "mpb"); + f(self.gemm_n_per_block, "npb"); + f(self.gemm_k_per_block, "kpb"); + + f(self.wave_tile_m, "wtm"); + f(self.wave_tile_n, "wtn"); + f(self.wave_tile_k, "wtk"); + f(self.wave_step_m, "wsm"); + f(self.wave_step_n, "wsn"); + f(self.wave_repeat_m, "wrm"); + f(self.wave_repeat_n, "wrn"); + + f(self.multihead, "mh"); + f(self.vector_store, "vs"); + f(self.gemm_k_global_split, "gks"); + f(self.merge_e, "me"); + f(self.tensor_a_pass_through, "pta"); + + f(self.tensor_a_thread_lengths[0], "ta0"); + f(self.tensor_a_thread_lengths[1], "ta1"); + f(self.tensor_a_thread_lengths[2], "ta2"); + f(self.tensor_a_thread_lengths[3], "ta3"); + + f(self.tensor_a_cluster_lengths[0], "ca0"); + f(self.tensor_a_cluster_lengths[1], "ca1"); + f(self.tensor_a_cluster_lengths[2], "ca2"); + f(self.tensor_a_cluster_lengths[3], "ca3"); + + f(self.tensor_b_thread_lengths[0], "tb0"); + f(self.tensor_b_thread_lengths[1], "tb1"); + f(self.tensor_b_thread_lengths[2], "tb2"); + f(self.tensor_b_thread_lengths[3], "tb3"); + + f(self.tensor_b_cluster_lengths[0], "cb0"); + f(self.tensor_b_cluster_lengths[1], "cb1"); + f(self.tensor_b_cluster_lengths[2], "cb2"); + f(self.tensor_b_cluster_lengths[3], "cb3"); + f(self.index, "index"); + } + + // Chilrden must provide support for ComputedContainer. + void HeuristicInit(const ExecutionContext&) = delete; + bool SetNextValue(const miopen::conv::ProblemDescription&) = delete; + bool IsValidValue() const = delete; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const = delete; + + MIOPEN_INTERNALS_EXPORT bool IsDefaultConstructed() const; + MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigAsmImplicitGemmGTC& other) const; + MIOPEN_INTERNALS_EXPORT void CopyParameters(const PerformanceConfigAsmImplicitGemmGTC& other); + MIOPEN_INTERNALS_EXPORT std::string ToString() const override; + MIOPEN_INTERNALS_EXPORT std::string ToKernelName(const ExecutionContext&) const; + MIOPEN_INTERNALS_EXPORT int BlockSize() const; +}; + +struct PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC : PerformanceConfigAsmImplicitGemmGTC +{ + PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC(std::string dir, + std::string layout, + std::string prec, + int b, + int e, + int mpb, + int npb, + int kpb, + int wtm, + int wtn, + int wtk, + int wsm, + int wsn, + int wrm, + int wrn, + int mh, + int vs, + int gks, + int me, + int pta, + std::initializer_list ta_t, + std::initializer_list ta_c, + std::initializer_list tb_t, + std::initializer_list tb_c, + bool spare = false) + : PerformanceConfigAsmImplicitGemmGTC(dir, + layout, + prec, + b, + e, + mpb, + npb, + kpb, + wtm, + wtn, + wtk, + wsm, + wsn, + wrm, + wrn, + mh, + vs, + gks, + me, + pta, + ta_t, + ta_c, + tb_t, + tb_c, + spare) + { + } + PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC(std::string dir, + std::string layout, + miopenDataType_t prec, + int b, + int e, + int mpb, + int npb, + int kpb, + int wtm, + int wtn, + int wtk, + int wsm, + int wsn, + int wrm, + int wrn, + int mh, + int vs, + int gks, + int me, + int pta, + std::initializer_list ta_t, + std::initializer_list ta_c, + std::initializer_list tb_t, + std::initializer_list tb_c, + bool spare = false) + : PerformanceConfigAsmImplicitGemmGTC(dir, + layout, + prec, + b, + e, + mpb, + npb, + kpb, + wtm, + wtn, + wtk, + wsm, + wsn, + wrm, + wrn, + mh, + vs, + gks, + me, + pta, + ta_t, + ta_c, + tb_t, + tb_c, + spare) + { + } + PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC() + : PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC("fwd", + "nchw", + "fp32", + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + false) + { + } + PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC(bool spare) + : PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC("fwd", + "nchw", + "fp32", + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + spare) + { + } + + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription& config); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; +}; + +struct ConvAsmImplicitGemmGTCDynamicFwdXdlopsNHWC final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC&) const override; +}; + +struct PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC : PerformanceConfigAsmImplicitGemmGTC +{ + PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC(std::string dir, + std::string layout, + std::string prec, + int b, + int e, + int mpb, + int npb, + int kpb, + int wtm, + int wtn, + int wtk, + int wsm, + int wsn, + int wrm, + int wrn, + int mh, + int vs, + int gks, + int me, + int pta, + std::initializer_list ta_t, + std::initializer_list ta_c, + std::initializer_list tb_t, + std::initializer_list tb_c, + bool spare = false) + : PerformanceConfigAsmImplicitGemmGTC(dir, + layout, + prec, + b, + e, + mpb, + npb, + kpb, + wtm, + wtn, + wtk, + wsm, + wsn, + wrm, + wrn, + mh, + vs, + gks, + me, + pta, + ta_t, + ta_c, + tb_t, + tb_c, + spare) + { + } + PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC(std::string dir, + std::string layout, + miopenDataType_t prec, + int b, + int e, + int mpb, + int npb, + int kpb, + int wtm, + int wtn, + int wtk, + int wsm, + int wsn, + int wrm, + int wrn, + int mh, + int vs, + int gks, + int me, + int pta, + std::initializer_list ta_t, + std::initializer_list ta_c, + std::initializer_list tb_t, + std::initializer_list tb_c, + bool spare = false) + : PerformanceConfigAsmImplicitGemmGTC(dir, + layout, + prec, + b, + e, + mpb, + npb, + kpb, + wtm, + wtn, + wtk, + wsm, + wsn, + wrm, + wrn, + mh, + vs, + gks, + me, + pta, + ta_t, + ta_c, + tb_t, + tb_c, + spare) + { + } + PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC() + : PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC("fwd", + "nchw", + "fp32", + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + false) + { + } + PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC(bool spare) + : PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC("fwd", + "nchw", + "fp32", + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + spare) + { + } + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; +}; + +struct ConvAsmImplicitGemmGTCDynamicBwdXdlopsNHWC final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC&) const override; +}; + +struct PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC : PerformanceConfigAsmImplicitGemmGTC +{ + PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC(std::string dir, + std::string layout, + std::string prec, + int b, + int e, + int mpb, + int npb, + int kpb, + int wtm, + int wtn, + int wtk, + int wsm, + int wsn, + int wrm, + int wrn, + int mh, + int vs, + int gks, + int me, + int pta, + std::initializer_list ta_t, + std::initializer_list ta_c, + std::initializer_list tb_t, + std::initializer_list tb_c, + bool spare = false) + : PerformanceConfigAsmImplicitGemmGTC(dir, + layout, + prec, + b, + e, + mpb, + npb, + kpb, + wtm, + wtn, + wtk, + wsm, + wsn, + wrm, + wrn, + mh, + vs, + gks, + me, + pta, + ta_t, + ta_c, + tb_t, + tb_c, + spare) + { + } + PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC(std::string dir, + std::string layout, + miopenDataType_t prec, + int b, + int e, + int mpb, + int npb, + int kpb, + int wtm, + int wtn, + int wtk, + int wsm, + int wsn, + int wrm, + int wrn, + int mh, + int vs, + int gks, + int me, + int pta, + std::initializer_list ta_t, + std::initializer_list ta_c, + std::initializer_list tb_t, + std::initializer_list tb_c, + bool spare = false) + : PerformanceConfigAsmImplicitGemmGTC(dir, + layout, + prec, + b, + e, + mpb, + npb, + kpb, + wtm, + wtn, + wtk, + wsm, + wsn, + wrm, + wrn, + mh, + vs, + gks, + me, + pta, + ta_t, + ta_c, + tb_t, + tb_c, + spare) + { + } + PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC() + : PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC("fwd", + "nchw", + "fp32", + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + false) + { + } + PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC(bool spare) + : PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC("fwd", + "nchw", + "fp32", + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + spare) + { + } + + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT size_t ComputeKernelOccupancy() const; + +private: + void SetParamsForKSplit(const miopen::conv::ProblemDescription& problem, + const size_t& occupancy); +}; + +struct ConvAsmImplicitGemmGTCDynamicWrwXdlopsNHWC final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC&) const override; +}; + +struct PerformanceConfigAsmImplicitGemmGTCvector + : PerfConfigBase +{ + std::string direction; + std::string tensor_layout; + std::string precision; + int nxb; + int nxe; + + int gemm_m_per_block; + int gemm_n_per_block; + int gemm_k_per_block; + + int lanegroup_tile_m; + int lanegroup_tile_n; + int lanegroup_wave_m; + int lanegroup_wave_n; + int lanegroup_repeat_m; + int lanegroup_repeat_n; + + int vector_c; + + std::vector tensor_a_thread_lengths; + std::vector tensor_a_cluster_lengths; + std::vector tensor_b_thread_lengths; + std::vector tensor_b_cluster_lengths; + + bool use_spare_set; + int index; + + MIOPEN_INTERNALS_EXPORT + PerformanceConfigAsmImplicitGemmGTCvector(std::string dir, + std::string layout, + std::string prec, + int b, + int e, + int mpb, + int npb, + int kpb, + int lgtm, + int lgtn, + int lgpwm, + int lgpwn, + int lgrm, + int lgrn, + int vec_c, + std::initializer_list ta_t, + std::initializer_list ta_c, + std::initializer_list tb_t, + std::initializer_list tb_c, + bool spare = false); + + MIOPEN_INTERNALS_EXPORT + PerformanceConfigAsmImplicitGemmGTCvector(std::string dir, + std::string layout, + miopenDataType_t prec, + int b, + int e, + int mpb, + int npb, + int kpb, + int lgtm, + int lgtn, + int lgpwm, + int lgpwn, + int lgrm, + int lgrn, + int vec_c, + std::initializer_list ta_t, + std::initializer_list ta_c, + std::initializer_list tb_t, + std::initializer_list tb_c, + bool spare = false); + + PerformanceConfigAsmImplicitGemmGTCvector() + : PerformanceConfigAsmImplicitGemmGTCvector("fwd", + "nchwc_kcyxc", + "Half", + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + false) + { + } + PerformanceConfigAsmImplicitGemmGTCvector(bool spare) + : PerformanceConfigAsmImplicitGemmGTCvector("fwd", + "nchwc_kcyxc", + "Half", + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + spare) + { + } + + template + static void Visit(Self&& self, F f) + { + f(self.direction, "dir"); + f(self.tensor_layout, "lyt"); + f(self.precision, "pre"); + f(self.nxb, "nxb"); + f(self.nxe, "nxe"); + f(self.gemm_m_per_block, "mpb"); + f(self.gemm_n_per_block, "npb"); + f(self.gemm_k_per_block, "kpb"); + + f(self.lanegroup_tile_m, "lgtm"); + f(self.lanegroup_tile_n, "lgtn"); + f(self.lanegroup_wave_m, "lgpwm"); + f(self.lanegroup_wave_n, "lgpwn"); + f(self.lanegroup_repeat_m, "lgrm"); + f(self.lanegroup_repeat_n, "lgrn"); + + f(self.vector_c, "vec_c"); + + f(self.tensor_a_thread_lengths[0], "ta0"); + f(self.tensor_a_thread_lengths[1], "ta1"); + f(self.tensor_a_thread_lengths[2], "ta2"); + f(self.tensor_a_thread_lengths[3], "ta3"); + + f(self.tensor_a_cluster_lengths[0], "ca0"); + f(self.tensor_a_cluster_lengths[1], "ca1"); + f(self.tensor_a_cluster_lengths[2], "ca2"); + f(self.tensor_a_cluster_lengths[3], "ca3"); + + f(self.tensor_b_thread_lengths[0], "tb0"); + f(self.tensor_b_thread_lengths[1], "tb1"); + f(self.tensor_b_thread_lengths[2], "tb2"); + f(self.tensor_b_thread_lengths[3], "tb3"); + + f(self.tensor_b_cluster_lengths[0], "cb0"); + f(self.tensor_b_cluster_lengths[1], "cb1"); + f(self.tensor_b_cluster_lengths[2], "cb2"); + f(self.tensor_b_cluster_lengths[3], "cb3"); + f(self.index, "index"); + } + + // Chilrden must provide support for ComputedContainer. + void HeuristicInit(const ExecutionContext&) = delete; + bool SetNextValue(const miopen::conv::ProblemDescription&) = delete; + bool IsValidValue() const = delete; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const = delete; + + MIOPEN_INTERNALS_EXPORT bool IsDefaultConstructed() const; + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceConfigAsmImplicitGemmGTCvector& other) const; + MIOPEN_INTERNALS_EXPORT void + CopyParameters(const PerformanceConfigAsmImplicitGemmGTCvector& other); + MIOPEN_INTERNALS_EXPORT std::string ToString() const override; + MIOPEN_INTERNALS_EXPORT std::string ToKernelName(const ExecutionContext&) const; + MIOPEN_INTERNALS_EXPORT int BlockSize() const; +}; +struct PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC : PerformanceConfigAsmImplicitGemmGTCvector +{ + + PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC(std::string dir, + std::string layout, + std::string prec, + int b, + int e, + int mpb, + int npb, + int kpb, + int lgtm, + int lgtn, + int lgpwm, + int lgpwn, + int lgrm, + int lgrn, + int vec_c, + std::initializer_list ta_t, + std::initializer_list ta_c, + std::initializer_list tb_t, + std::initializer_list tb_c, + bool spare = false) + : PerformanceConfigAsmImplicitGemmGTCvector(dir, + layout, + prec, + b, + e, + mpb, + npb, + kpb, + lgtm, + lgtn, + lgpwm, + lgpwn, + lgrm, + lgrn, + vec_c, + ta_t, + ta_c, + tb_t, + tb_c, + spare) + { + } + + PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC(std::string dir, + std::string layout, + miopenDataType_t prec, + int b, + int e, + int mpb, + int npb, + int kpb, + int lgtm, + int lgtn, + int lgpwm, + int lgpwn, + int lgrm, + int lgrn, + int vec_c, + std::initializer_list ta_t, + std::initializer_list ta_c, + std::initializer_list tb_t, + std::initializer_list tb_c, + bool spare = false) + : PerformanceConfigAsmImplicitGemmGTCvector(dir, + layout, + prec, + b, + e, + mpb, + npb, + kpb, + lgtm, + lgtn, + lgpwm, + lgpwn, + lgrm, + lgrn, + vec_c, + ta_t, + ta_c, + tb_t, + tb_c, + spare) + { + } + + PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC() + : PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC("fwd", + "nchwc_kcyxc", + "Half", + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + false) + { + } + PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC(bool spare) + : PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC("fwd", + "nchwc_kcyxc", + "Half", + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + {1, 1, 1, 1}, + spare) + { + } + + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; +}; + +struct ConvAsmImplicitGemmGTCDynamicFwdDlopsNCHWC final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC&) const override; +}; + +struct PerformanceConfigHipImplicitGemmFwdXdlops + : PerfConfigBaseCK +{ + int index = 0; + std::string kernel_id = ""; + std::vector valid_kernels; + + PerformanceConfigHipImplicitGemmFwdXdlops(int idx, std::string kernl_id) + : index(idx), kernel_id(kernl_id) + { + } + + PerformanceConfigHipImplicitGemmFwdXdlops() = default; + + explicit PerformanceConfigHipImplicitGemmFwdXdlops(bool) + : PerformanceConfigHipImplicitGemmFwdXdlops(0, "") + { + } + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceConfigHipImplicitGemmFwdXdlops& other) const; + +private: + template + void Init(const miopen::conv::ProblemDescription&); + template + bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; +}; + +struct ConvHipImplicitGemmFwdXdlops final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmFwdXdlops GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmFwdXdlops&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmFwdXdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmFwdXdlops&) const override; + /// \anchor igemm_get_wti_magic_number + // Magic Number Alert: + // Naive convolutions have GetWti() that return very small value (0.01f). + // This allows MIOpen to use Naive Solvers if no other applicable Solvers + // have known WTIs. Right now this means that in case of find-db miss, + // the library will try to use Winograd or GEMM (whatever is faster according + // to their GetWti's), but if both are not applicable, the library will + // use Naive Solver + // Since we would like to us CK before naive, and use it instead (because + // we do expect that CK is faster than Naive), therefore we use a + // value bigger than 0.01f, e.g. 0.02f. + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override + { + return 0.02f; + }; + +private: + template + bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; +}; + +struct PerformanceConfigHipImplicitGemmBwdXdlops + : PerfConfigBaseCK +{ + int index = 0; + std::string kernel_id = ""; + std::vector valid_kernels; + + PerformanceConfigHipImplicitGemmBwdXdlops(int idx, std::string kernl_id) + : index(idx), kernel_id(kernl_id) + { + } + + PerformanceConfigHipImplicitGemmBwdXdlops() = default; + + explicit PerformanceConfigHipImplicitGemmBwdXdlops(bool) + : PerformanceConfigHipImplicitGemmBwdXdlops(0, "") + { + } + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceConfigHipImplicitGemmBwdXdlops& other) const; + +private: + template + void Init(const miopen::conv::ProblemDescription&); + template + bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; +}; + +struct ConvHipImplicitGemmBwdXdlops final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmBwdXdlops GetDefaultPerformanceConfig( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmBwdXdlops&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmBwdXdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmBwdXdlops&) const override; + /// \ref igemm_get_wti_magic_number + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override + { + return 0.02f; + }; + +private: + template + bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; +}; + +struct PerformanceConfigHipImplicitGemmGroupFwdXdlops + : PerfConfigBaseCK +{ + int index = 0; + std::string kernel_id = ""; + std::vector valid_kernels; + + PerformanceConfigHipImplicitGemmGroupFwdXdlops(int idx, std::string kernl_id) + : index(idx), kernel_id(kernl_id) + { + } + + PerformanceConfigHipImplicitGemmGroupFwdXdlops() = default; + + explicit PerformanceConfigHipImplicitGemmGroupFwdXdlops(bool) + : PerformanceConfigHipImplicitGemmGroupFwdXdlops(0, "") + { + } + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceConfigHipImplicitGemmGroupFwdXdlops& other) const; + MIOPEN_INTERNALS_EXPORT bool + IsModelApplicable(const ExecutionContext& ctx, + const miopen::conv::ProblemDescription& problem) const; + +private: +#if MIOPEN_ENABLE_AI_KERNEL_TUNING + std::vector heuristic_indexes; + std::unordered_map> heuristic_kernels; + template + bool RunParameterPredictionModel(const ExecutionContext& ctx, + const miopen::conv::ProblemDescription& problem); + void InitHeuristicKernelIDs(const std::string& type); + bool ModelApplyToken(int idx, std::string value, const std::string& arch); +#endif + template + void Init(const miopen::conv::ProblemDescription&); + template + bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; +}; + +struct ConvHipImplicitGemmGroupFwdXdlops final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupFwdXdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmGroupFwdXdlops&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupFwdXdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmGroupFwdXdlops&) const override; + /// \ref igemm_get_wti_magic_number + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override + { + return 0.02f; + }; + + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + +private: + template + bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; +}; + +struct PerformanceConfigHipImplicitGemm3DGroupFwdXdlops + : PerfConfigBaseCK +{ + int index = 0; + std::string kernel_id = ""; + std::vector valid_kernels; + + PerformanceConfigHipImplicitGemm3DGroupFwdXdlops(int idx, std::string kernl_id) + : index(idx), kernel_id(kernl_id) + { + } + + PerformanceConfigHipImplicitGemm3DGroupFwdXdlops() = default; + + explicit PerformanceConfigHipImplicitGemm3DGroupFwdXdlops(bool) + : PerformanceConfigHipImplicitGemm3DGroupFwdXdlops(0, "") + { + } + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceConfigHipImplicitGemm3DGroupFwdXdlops& other) const; + +private: + template + void Init(const miopen::conv::ProblemDescription&); + template + bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; +}; + +struct ConvHipImplicitGemm3DGroupFwdXdlops final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupFwdXdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemm3DGroupFwdXdlops&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupFwdXdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemm3DGroupFwdXdlops&) const override; + /// \ref igemm_get_wti_magic_number + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override + { + return 0.02f; + }; + + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + +private: + template + bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; +}; + +struct PerformanceConfigHipImplicitGemm3DGroupWrwXdlops + : PerfConfigBaseCK +{ + int index; + int split_k; + std::string kernel_id; + std::vector valid_kernels; + PerformanceConfigHipImplicitGemm3DGroupWrwXdlops(int idx, std::string kernl_id) + : index(idx), kernel_id(kernl_id) + { + } + PerformanceConfigHipImplicitGemm3DGroupWrwXdlops() + : PerformanceConfigHipImplicitGemm3DGroupWrwXdlops(0, "") + { + } + PerformanceConfigHipImplicitGemm3DGroupWrwXdlops(bool) + : PerformanceConfigHipImplicitGemm3DGroupWrwXdlops(0, "") + { + } + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceConfigHipImplicitGemm3DGroupWrwXdlops& other) const; + +private: + template + void Init(const miopen::conv::ProblemDescription&); + template + bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; +}; + +struct ConvHipImplicitGemm3DGroupWrwXdlops final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupWrwXdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemm3DGroupWrwXdlops&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupWrwXdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemm3DGroupWrwXdlops&) const override; + /// \ref igemm_get_wti_magic_number + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override + { + return 0.02f; + }; + + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + +private: + template + bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; +}; + +struct PerformanceConfigHipImplicitGemm3DGroupBwdXdlops + : PerfConfigBaseCK +{ + int index; + std::string kernel_id; + std::vector valid_kernels; + PerformanceConfigHipImplicitGemm3DGroupBwdXdlops(int idx, std::string kernl_id) + : index(idx), kernel_id(kernl_id) + { + } + PerformanceConfigHipImplicitGemm3DGroupBwdXdlops() + : PerformanceConfigHipImplicitGemm3DGroupBwdXdlops(0, "") + { + } + PerformanceConfigHipImplicitGemm3DGroupBwdXdlops(bool) + : PerformanceConfigHipImplicitGemm3DGroupBwdXdlops(0, "") + { + } + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceConfigHipImplicitGemm3DGroupBwdXdlops& other) const; + +private: + template + void Init(const miopen::conv::ProblemDescription&); + template + bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; +}; + +struct ConvHipImplicitGemm3DGroupBwdXdlops final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupBwdXdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( + const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemm3DGroupBwdXdlops&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupBwdXdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemm3DGroupBwdXdlops&) const override; + /// \ref igemm_get_wti_magic_number + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override + { + return 0.02f; + }; + + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + +private: + template + bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; +}; + +struct PerformanceConfigHipImplicitGemmGroupBwdXdlops + : PerfConfigBaseCK +{ + int index; + std::string kernel_id; + std::vector valid_kernels; + PerformanceConfigHipImplicitGemmGroupBwdXdlops(int idx, std::string kernl_id) + : index(idx), kernel_id(kernl_id) + { + } + PerformanceConfigHipImplicitGemmGroupBwdXdlops() + : PerformanceConfigHipImplicitGemmGroupBwdXdlops(0, "") + { + } + PerformanceConfigHipImplicitGemmGroupBwdXdlops(bool) + : PerformanceConfigHipImplicitGemmGroupBwdXdlops(0, "") + { + } + + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceConfigHipImplicitGemmGroupBwdXdlops& other) const; + MIOPEN_INTERNALS_EXPORT bool + IsModelApplicable(const ExecutionContext& ctx, + const miopen::conv::ProblemDescription& problem) const; + +private: +#if MIOPEN_ENABLE_AI_KERNEL_TUNING + std::vector heuristic_indexes; + std::unordered_map> heuristic_kernels; + template + bool RunParameterPredictionModel(const ExecutionContext& ctx, + const miopen::conv::ProblemDescription& problem); + void InitHeuristicKernelIDs(); + bool ModelApplyToken(int idx, std::string value); +#endif + template + void Init(const miopen::conv::ProblemDescription&); + template + bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; +}; + +struct ConvHipImplicitGemmGroupBwdXdlops final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupBwdXdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmGroupBwdXdlops&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupBwdXdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmGroupBwdXdlops&) const override; + /// \ref igemm_get_wti_magic_number + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override + { + return 0.02f; + }; + + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + +private: + template + bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; +}; + +struct PerformanceConfigHipImplicitGemmGroupWrwXdlops + : PerfConfigBaseCK +{ + int index; + int split_k; + std::string kernel_id; + std::vector valid_kernels; + PerformanceConfigHipImplicitGemmGroupWrwXdlops(int idx, std::string kernl_id) + : index(idx), kernel_id(kernl_id) + { + } + PerformanceConfigHipImplicitGemmGroupWrwXdlops() + : PerformanceConfigHipImplicitGemmGroupWrwXdlops(0, "") + { + } + PerformanceConfigHipImplicitGemmGroupWrwXdlops(bool) + : PerformanceConfigHipImplicitGemmGroupWrwXdlops(0, "") + { + } + MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, + const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); + MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; + bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const + { + return IsValid(problem); + } + MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; + MIOPEN_INTERNALS_EXPORT bool + operator==(const PerformanceConfigHipImplicitGemmGroupWrwXdlops& other) const; + MIOPEN_INTERNALS_EXPORT bool + IsModelApplicable(const ExecutionContext& ctx, + const miopen::conv::ProblemDescription& problem) const; + +private: +#if MIOPEN_ENABLE_AI_KERNEL_TUNING + std::vector heuristic_indexes; + std::unordered_map> heuristic_kernels; + template + bool RunParameterPredictionModel(const ExecutionContext& ctx, + const miopen::conv::ProblemDescription& problem); + void InitHeuristicKernelIDs(const std::string& type); + bool ModelApplyToken(int idx, + std::string value, + const std::string& arch, + const miopen::conv::ProblemDescription& problem); +#endif + template + void Init(const miopen::conv::ProblemDescription&); + template + bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; +}; + +struct ConvHipImplicitGemmGroupWrwXdlops final + : ConvTunableSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupWrwXdlops + GetDefaultPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&) const override; + MIOPEN_INTERNALS_EXPORT bool + IsValidPerformanceConfig(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmGroupWrwXdlops&) const override; + MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupWrwXdlops + Search(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const AnyInvokeParams& invoke_ctx) const override; + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool IsDynamic() const override { return true; } + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext&, + const miopen::conv::ProblemDescription&, + const PerformanceConfigHipImplicitGemmGroupWrwXdlops&) const override; + /// \ref igemm_get_wti_magic_number + float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override + { + return 0.02f; + }; + + MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( + const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; + bool MayNeedWorkspace() const override { return true; } + +private: + template + bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; +}; + +} // namespace conv +} // namespace solver +} // namespace miopen diff --git a/src/include/miopen/conv_solution.hpp b/src/include/miopen/conv_solution.hpp index ac791cd559..897a55e9bf 100644 --- a/src/include/miopen/conv_solution.hpp +++ b/src/include/miopen/conv_solution.hpp @@ -65,7 +65,6 @@ struct ConvSolution int n_out_pix_tiles; // # output pixel tiles per wk-item (ALU) int n_in_data_tiles; // # of blocks of different inputs in LDS int n_stacks; // # of diff stacks (part of batch). - float weight = 0.0f; ConvSolution(miopenStatus_t status_ = miopenStatusSuccess) : status(status_), diff --git a/src/include/miopen/driver_arguments.hpp b/src/include/miopen/driver_arguments.hpp index da4064b7f0..f28a772e3c 100644 --- a/src/include/miopen/driver_arguments.hpp +++ b/src/include/miopen/driver_arguments.hpp @@ -67,6 +67,10 @@ std::string ConvArgsForMIOpenDriver(const miopen::TensorDescriptor& xDesc, bool print_for_conv_driver = true); std::string BnormArgsForMIOpenDriver(miopenTensorDescriptor_t xDesc, + miopenTensorDescriptor_t yDesc, + miopenTensorDescriptor_t scaleDesc, + miopenTensorDescriptor_t biasDesc, + miopenTensorDescriptor_t saveMeanDesc, miopenBatchNormMode_t bn_mode, const void* resultRunningMean, const void* resultRunningVariance, diff --git a/src/include/miopen/dropout.hpp b/src/include/miopen/dropout.hpp index 985972edb2..b332f87690 100644 --- a/src/include/miopen/dropout.hpp +++ b/src/include/miopen/dropout.hpp @@ -73,29 +73,18 @@ struct MIOPEN_INTERNALS_EXPORT DropoutDescriptor : miopenDropoutDescriptor size_t prng_stateSizeInBytes, unsigned long long prng_seed) const; - void DropoutForward(const Handle& handle, - const TensorDescriptor& noise_shape, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& yDesc, - Data_t y, - Data_t reserveSpace, - size_t reserveSpaceSizeInBytes, - size_t in_offset = 0, - size_t out_offset = 0, - size_t rsvsp_offset = 0) const; - - void DropoutBackward(const Handle& handle, - const TensorDescriptor& noise_shape, - const TensorDescriptor& dyDesc, - ConstData_t dy, - const TensorDescriptor& dxDesc, - Data_t dx, - Data_t reserveSpace, - size_t reserveSpaceSizeInBytes, - size_t in_offset = 0, - size_t out_offset = 0, - size_t rsvsp_offset = 0) const; + void Dropout(const Handle& handle, + const TensorDescriptor& noise_shape, + const TensorDescriptor& xDesc, + ConstData_t x, + const TensorDescriptor& yDesc, + Data_t y, + Data_t reserveSpace, + size_t reserveSpaceSizeInBytes, + size_t in_offset = 0, + size_t out_offset = 0, + size_t rsvsp_offset = 0, + bool is_backward = false) const; }; std::ostream& operator<<(std::ostream& stream, const DropoutDescriptor& x); diff --git a/src/include/miopen/each_args.hpp b/src/include/miopen/each_args.hpp index 6a7c6f0d4e..e078153dc9 100644 --- a/src/include/miopen/each_args.hpp +++ b/src/include/miopen/each_args.hpp @@ -33,46 +33,17 @@ namespace miopen { namespace detail { -template -struct seq -{ - using type = seq; -}; - -template -struct merge_seq; - -template -struct merge_seq, seq> : seq -{ -}; - -template -struct gens : merge_seq::type, typename gens::type> -{ -}; - -template <> -struct gens<0> : seq<> -{ -}; -template <> -struct gens<1> : seq<0> -{ -}; - template -void each_args_i_impl(F f, seq, Ts&&... xs) +void each_args_i_impl(F f, std::index_sequence, Ts&&... xs) { (void)std::initializer_list{ (f(std::integral_constant{}, std::forward(xs)), 0)...}; } template -auto unpack_impl(F f, seq, T&& x) +auto unpack_impl(F f, std::index_sequence, T&& x) { - using std::get; - return f(get(x)...); + return f(std::get(x)...); } } // namespace detail @@ -80,8 +51,7 @@ auto unpack_impl(F f, seq, T&& x) template void each_args_i(F f, Ts&&... xs) { - detail::each_args_i_impl( - f, typename detail::gens::type{}, std::forward(xs)...); + detail::each_args_i_impl(f, std::make_index_sequence(), std::forward(xs)...); } template @@ -101,7 +71,7 @@ auto unpack(F f, T&& x) { using type = typename std::remove_cv::type>::type; return detail::unpack_impl( - f, typename detail::gens::value>::type{}, std::forward(x)); + f, std::make_index_sequence::value>(), std::forward(x)); } } // namespace miopen diff --git a/src/include/miopen/env.hpp b/src/include/miopen/env.hpp index a37414d3b4..4233a1c50c 100644 --- a/src/include/miopen/env.hpp +++ b/src/include/miopen/env.hpp @@ -170,6 +170,7 @@ struct EnvVar } \ operator ::miopen::env::detail::EnvVar<_type>&() const { return ref(); } \ operator bool() const { return ref().exist(); } \ + constexpr std::string_view GetName() const { return #_name; } \ } _name; #define MIOPEN_DECLARE_ENV_VAR_BOOL(name, ...) MIOPEN_DECLARE_ENV_VAR(name, bool, __VA_ARGS__) diff --git a/src/include/miopen/filesystem.hpp b/src/include/miopen/filesystem.hpp index 9d65a2630e..eab2a71cb3 100644 --- a/src/include/miopen/filesystem.hpp +++ b/src/include/miopen/filesystem.hpp @@ -27,35 +27,49 @@ #ifndef GUARD_MIOPEN_FILESYSTEM_HPP_ #define GUARD_MIOPEN_FILESYSTEM_HPP_ +// See CMakeLists.txt in addkernels +#if !defined(MIOPEN_HACK_DO_NOT_INCLUDE_CONFIG_H) +#include +#endif + #include #include +// clang-format off #if defined(CPPCHECK) -#define MIOPEN_HAS_FILESYSTEM 1 -#define MIOPEN_HAS_FILESYSTEM_TS 1 + #define MIOPEN_HAS_FILESYSTEM 1 + #define MIOPEN_HAS_FILESYSTEM_TS 1 #elif defined(_WIN32) -#if _MSC_VER >= 1920 -#define MIOPEN_HAS_FILESYSTEM 1 -#define MIOPEN_HAS_FILESYSTEM_TS 0 -#elif _MSC_VER >= 1900 -#define MIOPEN_HAS_FILESYSTEM 0 -#define MIOPEN_HAS_FILESYSTEM_TS 1 -#else -#define MIOPEN_HAS_FILESYSTEM 0 -#define MIOPEN_HAS_FILESYSTEM_TS 0 -#endif + #if _MSC_VER >= 1920 + #define MIOPEN_HAS_FILESYSTEM 1 + #define MIOPEN_HAS_FILESYSTEM_TS 0 + #elif _MSC_VER >= 1900 + #define MIOPEN_HAS_FILESYSTEM 0 + #define MIOPEN_HAS_FILESYSTEM_TS 1 + #else + #define MIOPEN_HAS_FILESYSTEM 0 + #define MIOPEN_HAS_FILESYSTEM_TS 0 + #endif #elif defined(__has_include) -#if __has_include() && __cplusplus >= 201703L -#define MIOPEN_HAS_FILESYSTEM 1 + #if __has_include() && __cplusplus >= 201703L + #define MIOPEN_HAS_FILESYSTEM 1 + #else + #define MIOPEN_HAS_FILESYSTEM 0 + #endif + #if __has_include() && __cplusplus >= 201103L + #define MIOPEN_HAS_FILESYSTEM_TS 1 + #else + #define MIOPEN_HAS_FILESYSTEM_TS 0 + #endif #else -#define MIOPEN_HAS_FILESYSTEM 0 + #define MIOPEN_HAS_FILESYSTEM 0 + #define MIOPEN_HAS_FILESYSTEM_TS 0 #endif -#if __has_include() && __cplusplus >= 201103L -#define MIOPEN_HAS_FILESYSTEM_TS 1 -#else -#define MIOPEN_HAS_FILESYSTEM_TS 0 -#endif -#else +// clang-format on + +#if MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM +#undef MIOPEN_HAS_FILESYSTEM +#undef MIOPEN_HAS_FILESYSTEM_TS #define MIOPEN_HAS_FILESYSTEM 0 #define MIOPEN_HAS_FILESYSTEM_TS 0 #endif @@ -64,6 +78,28 @@ #include #elif MIOPEN_HAS_FILESYSTEM_TS #include +#elif MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM +/// Explicit inclusion of is a workaround for the Boost 1.83 issue. +/// +/// Boost doc is saying: +/// When writing template classes, you might not want to include the main hash.hpp header as it's +/// quite an expensive include that brings in a lot of other headers, so instead you can include +/// the header which forward declares boost::hash, +/// boost::hash_combine, boost::hash_range, and boost::hash_unordered_range. You'll need to +/// include the main header before instantiating boost::hash. +/// +/// \ref https://www.boost.org/doc/libs/1_83_0/libs/container_hash/doc/html/hash.html#combine +/// +/// It seems like boost::filesystem uses boost::hash_range but misses to include hash.hpp, so we +/// have to include it explicitly. Otherwise an error like this happens at runtime (!): +/// +/// \code +/// ...symbol lookup error: .../libMIOpen.so.1: undefined symbol: +/// _ZN5boost10hash_rangeIN9__gnu_cxx17__normal_iteratorIPKcNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEEEEEmT_SC_ +/// \endcode +#include +#define BOOST_FILESYSTEM_NO_DEPRECATED 1 +#include #else #error "No filesystem include available" #endif @@ -74,20 +110,37 @@ namespace miopen { namespace fs = ::std::filesystem; #elif MIOPEN_HAS_FILESYSTEM_TS namespace fs = ::std::experimental::filesystem; +#elif MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM +namespace fs = ::boost::filesystem; #endif } // namespace miopen inline std::string operator+(const std::string_view s, const miopen::fs::path& path) { +#if MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM + return std::string(path.native()).insert(0, s); +#else return path.string().insert(0, s); +#endif } inline std::string operator+(const miopen::fs::path& path, const std::string_view s) { +#if MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM + return std::string(path.native()).append(s); +#else return path.string().append(s); +#endif } +// Hack to minimize changes for boost fs. +#if MIOPEN_WORKAROUND_USE_BOOST_FILESYSTEM +#define FS_ENUM_PERMS_ALL fs::perms::all_all +#else +#define FS_ENUM_PERMS_ALL fs::perms::all +#endif + #if MIOPEN_HAS_FILESYSTEM_TS #ifdef __linux__ #include diff --git a/src/include/miopen/fin/fin_interface.hpp b/src/include/miopen/fin/fin_interface.hpp new file mode 100644 index 0000000000..b816a3038e --- /dev/null +++ b/src/include/miopen/fin/fin_interface.hpp @@ -0,0 +1,258 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace miopen { + +struct AnyInvokeParams; +struct ExecutionContext; + +namespace conv { +struct ProblemDescription; +} // namespace conv + +namespace batchnorm { +struct ProblemDescription; +} // namespace batchnorm + +namespace solver { +struct SolverBase; +} // namespace solver + +namespace fin_interface { + +// ================== Interface for Fin ================== + +// Base classes for solvers. +class MIOPEN_INTERNALS_EXPORT Solver +{ +public: + // GetId(), IsDynamic() and IsTunable() throw miopenStatusNotInitialized if the solver is not + // valid. + + // Returns false if the solver could not be found by its name. + bool IsValid() const; + + uint64_t GetId() const; + // Returns the name even if the solver is not valid (returns the requested name). + const std::string& GetName() const; + + bool IsTunable() const; + bool IsDynamic() const; + +protected: + Solver(const miopen::solver::SolverBase* solver_base, uint64_t solver_id); + Solver(const std::string& requested_name); + + const miopen::solver::SolverBase* const sbase = nullptr; + const std::string rname; + uint64_t id; +}; + +template +class SolverMixin : public Solver +{ +public: + // All the methods throw miopenStatusNotInitialized if the solver is not valid. + + bool IsApplicable(const Context& ctx, const Problem& problem) const; + + size_t GetWorkspaceSize(const Context& ctx, const Problem& problem) const; + + miopen::solver::ConvSolution FindSolution(const Context& ctx, + const Problem& problem, + miopen::PerformanceDb& db, + const miopen::AnyInvokeParams& invoke_ctx, + const std::string& perf_cfg = "") const; + + std::vector GetAllSolutions(const Context& ctx, + const Problem& problem) const; + + std::string + GetPerfCfgParams(const Context& ctx, const Problem& problem, miopen::PerformanceDb& db) const; + + bool + TestPerfCfgParams(const Context& ctx, const Problem& problem, const std::string& params) const; + +protected: + SolverMixin(const miopen::solver::SolverBase* solver_base, uint64_t solver_id); + using Solver::Solver; + + std::any asolver; + + template + friend const std::vector& GetAllSolvers(); + template + friend Solver GetSolver(const std::string&); +}; + +extern template class SolverMixin; +extern template class SolverMixin; + +// Convolution solver +class MIOPEN_INTERNALS_EXPORT ConvSolver final + : public SolverMixin +{ +public: + ConvSolver(const miopen::solver::SolverBase* solver_base, uint64_t solver_id) = delete; + std::string GetAlgo(miopen::conv::Direction dir) const; + +protected: + ConvSolver(const miopen::solver::SolverBase* solver_base, + uint64_t solver_id, + miopenConvAlgorithm_t algo_); + using SolverMixin::SolverMixin; + + miopenConvAlgorithm_t algo; + + template + friend const std::vector& GetAllSolvers(); + template + friend Solver GetSolver(const std::string&); +}; + +// Batch normalization solver +class MIOPEN_INTERNALS_EXPORT BatchNormSolver final + : public SolverMixin +{ +protected: + using SolverMixin::SolverMixin; +}; + +// GetAll*Solvers() - returns all solvers for a particular primitive. All solvers are always valid. +// +// Get*Solvers(names) - returns all solvers with the names specified in the list for a particular +// primitive. May return dummies ​​for solvers that do not exist. +// +// Get*Solver(name) - returns single solver by its name for a particular primitive. May return a +// dummy if a solver with specified name does not exist. + +// Convolution +MIOPEN_INTERNALS_EXPORT const std::vector& GetAllConvSolvers(); +MIOPEN_INTERNALS_EXPORT std::vector +GetConvSolvers(const std::vector& names); +MIOPEN_INTERNALS_EXPORT ConvSolver GetConvSolver(const std::string& name); + +// Batch normalization +MIOPEN_INTERNALS_EXPORT const std::vector& GetAllBatchNormSolvers(); +MIOPEN_INTERNALS_EXPORT std::vector +GetBatchNormSolvers(const std::vector& names); +MIOPEN_INTERNALS_EXPORT BatchNormSolver GetBatchNormSolver(const std::string& name); + +// Examples: +// +// Convolution solvers: +// +// 1a (Old version): +// +// const auto& solver_id_list = +// miopen::solver::GetSolversByPrimitive(miopen::solver::Primitive::Convolution); +// for(const auto& id : solver_id_list) +// { +// std::unordered_map solver_info; +// solver_info["name"] = id.ToString(); +// if(!id.IsValid()) +// continue; +// solver_info["id"] = std::to_string(id.Value()); +// solver_info["algo"] = id.GetAlgo(miopen::conv::Direction::Forward); +// const auto solver = id.GetSolver(); +// if(solver.IsEmpty()) +// continue; +// solver_info["tunable"] = solver.IsTunable() ? "1" : "0"; +// solver_info["dynamic"] = solver.IsDynamic() ? "1" : "0"; +// } +// +// 1b (Nev version): +// +// const auto& solver_list = miopen::fin_interface::GetAllConvSolvers(); +// for(const auto& solver : solver_list) +// { +// std::unordered_map solver_info; +// solver_info["name"] = solver.GetName(); +// if(!solver.IsValid()) +// continue; +// solver_info["id"] = std::to_string(solver.GetId()); +// solver_info["algo"] = solver.GetAlgo(miopen::conv::Direction::Forward); +// solver_info["tunable"] = solver.IsTunable() ? "1" : "0"; +// solver_info["dynamic"] = solver.IsDynamic() ? "1" : "0"; +// } +// +// 2a (Old version): +// +// std::string solver_name = "ConvBiasActivAsm1x1U"; +// const auto id = miopen::solver::Id{solver_name}; +// std::unordered_map solver_info; +// solver_info["name"] = id.ToString(); +// if(id.IsValid()) +// { +// solver_info["id"] = std::to_string(id.Value()); +// solver_info["algo"] = id.GetAlgo(miopen::conv::Direction::Forward); +// const auto solver = id.GetSolver(); +// if(!solver.IsEmpty()) +// { +// solver_info["tunable"] = solver.IsTunable() ? "1" : "0"; +// solver_info["dynamic"] = solver.IsDynamic() ? "1" : "0"; +// } +// } +// +// 2b (Nev version): +// +// std::string solver_name = "ConvBiasActivAsm1x1U"; +// const auto solver = miopen::fin_interface::GetConvSolver(solver_name); +// std::unordered_map solver_info; +// solver_info["name"] = solver.GetName(); +// if(solver.IsValid()) +// { +// solver_info["id"] = std::to_string(solver.GetId()); +// solver_info["algo"] = solver.GetAlgo(miopen::conv::Direction::Forward); +// solver_info["tunable"] = solver.IsTunable() ? "1" : "0"; +// solver_info["dynamic"] = solver.IsDynamic() ? "1" : "0"; +// } +// +// Batch normalization solvers: +// +// ... +// const auto& solver_list = miopen::fin_interface::GetAllBatchNormSolvers(); +// ... +// +// ... +// const auto solver = miopen::fin_interface::GetBatchNormSolver(solver_name); +// ... +// + +} // namespace fin_interface +} // namespace miopen diff --git a/src/include/miopen/find_controls.hpp b/src/include/miopen/find_controls.hpp index 901a849c6c..50d3ec5ee4 100644 --- a/src/include/miopen/find_controls.hpp +++ b/src/include/miopen/find_controls.hpp @@ -137,7 +137,8 @@ class MIOPEN_INTERNALS_EXPORT FindMode } public: - FindMode(); + // Todo: remove default value of primitive + FindMode(solver::Primitive primitive = solver::Primitive::Convolution); Values Get() const { return value; } void Set(Values const v) { value = v; } diff --git a/src/include/miopen/find_solution.hpp b/src/include/miopen/find_solution.hpp index ff6a2a1ca1..fa04b46d20 100644 --- a/src/include/miopen/find_solution.hpp +++ b/src/include/miopen/find_solution.hpp @@ -52,7 +52,7 @@ namespace solver { template auto FindSolutionImpl(rank<1>, - Solver s, + const Solver& s, const Context& context, const Problem& problem, Db&& db, @@ -148,7 +148,7 @@ auto FindSolutionImpl(rank<1>, template auto FindSolutionImpl(rank<0>, - Solver s, + const Solver& s, const Context& context, const Problem& problem, Db&&, @@ -200,7 +200,7 @@ auto GetInvokeFactoryImpl( /// Could take long if an exhaustive search is requested/performed. /// May read/write perfDb. template -ConvSolution FindSolution(Solver s, +ConvSolution FindSolution(const Solver& s, const Context& context, const Problem& problem, Db&& db, @@ -395,25 +395,25 @@ struct SolverContainer } template - std::vector> - GetWorkspaceSizes(const Context& ctx, - const Problem& problem, - std::size_t limit = std::numeric_limits::max()) const + std::vector> GetWorkspaceSizes( + const Context& ctx, const Problem& problem, const bool simple_primitive = false) const { std::vector> res; const auto find_only = GetEnvFindOnlySolver(); - std::size_t count = 0; miopen::each_args( [&](auto solver) { - if(count >= limit) - return; - if(find_only && (std::find(find_only->begin(), find_only->end(), Id{solver.SolverDbId()}) == find_only->end())) { // Do nothing (and keep silence for the sake of Tuna), just skip. } - else if(!solver.MayNeedWorkspace()) + // The following optimization is required to avoid checks + // for solvers that have slow IsApplicable() and do not + // require workspace (like MLIR convolutions). However we + // do not want to use it for simple primitives, for example, + // the ones that ExecutePrimitive() which uses the first applicable + // solver: + else if(!simple_primitive && !solver.MayNeedWorkspace()) { MIOPEN_LOG_I2(solver.SolverDbId() << ": Skipped (no workspace required)"); } @@ -429,7 +429,6 @@ struct SolverContainer } else { - ++count; auto sz = solver.GetWorkspaceSize(ctx, problem); res.push_back(std::make_pair(solver.SolverDbId(), sz)); MIOPEN_LOG_I2(solver.SolverDbId() << ": " << sz); @@ -439,41 +438,6 @@ struct SolverContainer return res; } - // Search for all applicable solutions among many solvers - template - bool IsAnySolverApplicable(const Context& ctx, const Problem& problem) const - { - const auto find_only = GetEnvFindOnlySolver(); - auto found = false; - - miopen::each_args( - [&](auto solver) { - if(found || (find_only && (std::find(find_only->begin(), - find_only->end(), - Id{solver.SolverDbId()}) == find_only->end()))) - return; - - // For better performance, check IsDynamic() first, because - // it is much faster than IsApplicable(). - if(ctx.use_dynamic_solutions_only && !solver.IsDynamic()) - { - MIOPEN_LOG_I2(solver.SolverDbId() << ": Skipped (non-dynamic)"); - return; - } - - if(solver.IsApplicable(ctx, problem)) - { - found = true; - return; - } - - MIOPEN_LOG_I2(solver.SolverDbId() << ": Not applicable"); - }, - Solvers{}...); - - return found; - } - template void ExecutePrimitive(const ExecutionContext& ctx, const Problem& problem, diff --git a/src/include/miopen/functional.hpp b/src/include/miopen/functional.hpp index 179c8da373..02c6e3427e 100644 --- a/src/include/miopen/functional.hpp +++ b/src/include/miopen/functional.hpp @@ -34,7 +34,8 @@ namespace miopen { namespace detail { template -auto each_i_impl(F f, seq) MIOPEN_RETURNS(f(std::integral_constant{}...)); +auto each_i_impl(F f, std::index_sequence) + MIOPEN_RETURNS(f(std::integral_constant{}...)); } // namespace detail template @@ -87,8 +88,7 @@ struct sequence_t F f; template auto operator()(IntegralConstant) const - MIOPEN_RETURNS(detail::each_i_impl(f, - typename detail::gens::type{})); + MIOPEN_RETURNS(detail::each_i_impl(f, std::make_index_sequence())); }; template diff --git a/src/include/miopen/fusion.hpp b/src/include/miopen/fusion.hpp index 70d306deeb..babf192b74 100644 --- a/src/include/miopen/fusion.hpp +++ b/src/include/miopen/fusion.hpp @@ -30,7 +30,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/src/include/miopen/fusion/context.hpp b/src/include/miopen/fusion/context.hpp index 65f7cd7b8c..6f5ea25b2d 100644 --- a/src/include/miopen/fusion/context.hpp +++ b/src/include/miopen/fusion/context.hpp @@ -36,6 +36,7 @@ struct Handle; struct FusionContext : ExecutionContext { explicit FusionContext(Handle& handle) : ExecutionContext(&handle) {} + explicit FusionContext(const ExecutionContext& ctx) : ExecutionContext(ctx) {} ExecutionContext GetConvContext(const conv::ProblemDescription& conv_problem) const { diff --git a/src/include/miopen/fusion/problem_description.hpp b/src/include/miopen/fusion/problem_description.hpp index bcb37878d9..b3d1669cee 100644 --- a/src/include/miopen/fusion/problem_description.hpp +++ b/src/include/miopen/fusion/problem_description.hpp @@ -128,7 +128,14 @@ struct FusionDescription : ProblemDescriptionBase dynamic_cast(*fusion_plan_desc->op_map[idx]); miopen::TensorDescriptor out_desc; bn_op.GetOutputDesc(out_desc); - return {bn_op.mode, bn_op.input_desc, out_desc, bn_op.base_desc, not_used}; + return {bn_op.mode, + bn_op.input_desc, + out_desc, + bn_op.base_desc, + bn_op.base_desc, + bn_op.base_desc, + bn_op.base_desc, + not_used}; } else if(dir == miopen::batchnorm::Direction::ForwardTraining) { @@ -140,6 +147,9 @@ struct FusionDescription : ProblemDescriptionBase bn_op.input_desc, out_desc, bn_op.base_desc, + bn_op.base_desc, + bn_op.base_desc, + bn_op.base_desc, not_used, // expAvgFactor filler not_used, true /* resultSave*/, @@ -156,6 +166,9 @@ struct FusionDescription : ProblemDescriptionBase out_desc, bn_op.input_desc, {} /*bn_op.base_desc*/, + {} /*bn_op.base_desc*/, + {} /*bn_op.base_desc*/, + {} /*bn_op.base_desc*/, not_used, bn_op.useBatchStats /*useSaved*/}; } diff --git a/src/include/miopen/fusion/solvers.hpp b/src/include/miopen/fusion/solvers.hpp index a21cbccdc6..2a30bb8915 100644 --- a/src/include/miopen/fusion/solvers.hpp +++ b/src/include/miopen/fusion/solvers.hpp @@ -27,6 +27,7 @@ #pragma once #include +#include #include #include #include @@ -82,6 +83,8 @@ struct ConvBiasActivAsm1x1U : FusionTunableSolver diff --git a/src/include/miopen/generic_search.hpp b/src/include/miopen/generic_search.hpp index 78acaaac6c..a9cb6e4ae9 100644 --- a/src/include/miopen/generic_search.hpp +++ b/src/include/miopen/generic_search.hpp @@ -36,7 +36,6 @@ #include #include #include -#include #include #include @@ -55,6 +54,7 @@ namespace solver { namespace debug { // This struct is not MT-safe, meaning one should use it before starting threads, thus avoiding // constructing it inside a worker thread. +/// \todo This class should be moved out of the library struct MIOPEN_INTERNALS_EXPORT TuningIterationScopedLimiter { TuningIterationScopedLimiter(std::size_t new_limit); @@ -241,7 +241,6 @@ class HeartBeat /// - Its return type shall be suitable for instantiation of the ComputedContainer. /// * GetSolution shall be implemented. /// * Solution should provide invoker -/// * RunAndMeasureSolution must NOT be implemented. Invoker will be used instead. /// /// clang-format-off /// ----------------------------------------------- @@ -263,17 +262,6 @@ class HeartBeat /// ------------------------------------------------ /// clang-format-on -template -using RunAndMeasure_t = - decltype(std::declval().RunAndMeasureSolution(std::declval(), - std::declval(), - std::declval(), - std::declval(), - std::declval(), - std::declval(), - std::declval(), - std::declval())); - template auto GetAllConfigs(const Solver s, const Context& context, const Problem& problem) -> ComputedContainer decltype(s.GetDefaultPerformanceConfig(context_, problem)) { - static_assert( - !(HasMember{} || - HasMember{}), - "RunAndMeasure is obsolete. Solvers should implement auto-tune evaluation in invoker"); - auto context = context_; context.is_for_generic_search = true; @@ -398,6 +381,7 @@ auto GenericSearch(const Solver s, std::shuffle(all_configs.begin(), all_configs.end(), rng); std::size_t n_runs_total = std::min(all_configs.size(), GetTuningIterationsMax()); all_configs.resize(n_runs_total); + std::size_t patience = env::value(MIOPEN_TUNING_PATIENCE); if(all_configs.empty()) { @@ -443,11 +427,22 @@ auto GenericSearch(const Solver s, if(!env::enabled(MIOPEN_DEBUG_COMPILE_ONLY)) { size_t n_current = 0; + size_t last_imprv = 0; auto threads_remaining = total_threads; while(true) { if(n_current >= n_runs_total) + { + MIOPEN_LOG_I2("Ending Search by total runs: " << n_runs_total); + break; + } + if(last_imprv >= patience) + { + MIOPEN_LOG_I2("Ending Search by patience: " << patience); break; + } + + last_imprv++; MIOPEN_LOG_I2("Waiting for item in queue"); const auto kinder = solution_queue.pop(); auto current_config = std::get<0>(kinder); @@ -508,17 +503,18 @@ auto GenericSearch(const Solver s, if(ret == 0) { // Smooth the jitter of measurements: - // If the 1st probe is NOT too bad (measured time <= 1.05 * best known time), - // then re-run it 4 times more and compute average time, - // and decide using average of all 5 attempts vs. the best. - if(elapsed_time / best_time < 1.05f) + // If the 1st probe is NOT too bad (measured time <= 1.10 * best known time), + // then re-run it 9 times more and compute average time, + // and decide using average of all 10 attempts vs. the best. + constexpr int N_RUNS = 10; + if(elapsed_time / best_time < 1.10f) { MIOPEN_LOG_I2("Finding average for: " << elapsed_time << " / " << best_time << " = " << (elapsed_time / best_time)); try { - for(int i = 0; i < 4; ++i) + for(int i = 1; i < N_RUNS; ++i) { invoker(profile_h, invoke_ctx); elapsed_time += profile_h.GetKernelTime(); @@ -532,7 +528,7 @@ auto GenericSearch(const Solver s, if(ret == 0) { is_passed = true; - elapsed_time /= 5; + elapsed_time /= N_RUNS; if(elapsed_time < best_time) { MIOPEN_LOG_I('#' << n_current << '/' << n_failed << '/' << n_runs_total @@ -541,6 +537,7 @@ auto GenericSearch(const Solver s, best_config = current_config; best_time = elapsed_time; n_best = n_current; + last_imprv = 0; } else { diff --git a/src/include/miopen/generic_search_controls.hpp b/src/include/miopen/generic_search_controls.hpp index 00d5697d14..65cc0ff3dc 100644 --- a/src/include/miopen/generic_search_controls.hpp +++ b/src/include/miopen/generic_search_controls.hpp @@ -35,6 +35,9 @@ MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_DEBUG_TUNING_ITERATIONS_MAX, MIOPEN_DECLARE_ENV_VAR_UINT64( MIOPEN_TUNING_TIME_MS_MAX, std::chrono::duration_cast(std::chrono::hours{2}).count()) +MIOPEN_DECLARE_ENV_VAR_UINT64( + MIOPEN_TUNING_PATIENCE, + std::numeric_limits::max()) // End tuning if no improvement in X iterations #if MIOPEN_USE_COMGR MIOPEN_DECLARE_ENV_VAR_UINT64(MIOPEN_COMPILE_PARALLEL_LEVEL, 1) // COMGR is not parallelizable diff --git a/src/include/miopen/getitem.hpp b/src/include/miopen/getitem.hpp new file mode 100644 index 0000000000..191b1dba97 --- /dev/null +++ b/src/include/miopen/getitem.hpp @@ -0,0 +1,58 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#ifndef MIOPEN_GETITEM_HPP_ +#define MIOPEN_GETITEM_HPP_ + +#include + +namespace miopen { + +struct Handle; +struct TensorDescriptor; + +MIOPEN_INTERNALS_EXPORT std::size_t GetGetitemWorkspaceSize( + Handle& handle, uint32_t indexCount, const TensorDescriptor* const* indexDescs); + +MIOPEN_INTERNALS_EXPORT miopenStatus_t GetitemBackward(Handle& handle, + Data_t workspace, + size_t workspaceSizeInBytes, + const TensorDescriptor& dyDesc, + ConstData_t dy, + uint32_t indexCount, + const TensorDescriptor* const* indexDescs, + ConstData_t* indexs, + const TensorDescriptor& dxDesc, + Data_t dx, + const TensorDescriptor& errorDesc, + Data_t error, + uint32_t dimCount, + const int32_t* dims, + uint32_t sliceCount, + const int32_t* slices, + uint32_t offset); + +} // namespace miopen +#endif // _MIOPEN_GETITEM_HPP_ diff --git a/src/include/miopen/getitem/invoke_params.hpp b/src/include/miopen/getitem/invoke_params.hpp new file mode 100644 index 0000000000..e663482271 --- /dev/null +++ b/src/include/miopen/getitem/invoke_params.hpp @@ -0,0 +1,97 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +namespace miopen { +namespace getitem { + +struct GetitemInvokeParams : public miopen::InvokeParams +{ + + GetitemInvokeParams(Data_t workspace_, + std::size_t workspace_size_, + const TensorDescriptor& dyDesc_, + ConstData_t dy_, + uint32_t indexCount_, + const TensorDescriptor* const* indexDescs_, + ConstData_t* indexs_, + const TensorDescriptor& dxDesc_, + Data_t dx_, + const TensorDescriptor& errorDesc_, + Data_t error_, + uint32_t dimCount_, + const int32_t* dims_, + uint32_t sliceCount_, + const int32_t* slices_, + uint32_t offset_) + : workspace(workspace_), + workspace_size(workspace_size_), + dyDesc(dyDesc_), + dy(dy_), + indexCount(indexCount_), + indexDescs(indexDescs_), + indexs(indexs_), + dxDesc(dxDesc_), + dx(dx_), + errorDesc(errorDesc_), + error(error_), + dimCount(dimCount_), + dims(dims_), + sliceCount(sliceCount_), + slices(slices_), + offset(offset_) + { + } + + Data_t workspace = nullptr; + std::size_t workspace_size = 0; + const TensorDescriptor dyDesc{}; + ConstData_t dy = nullptr; + uint32_t indexCount = 0; + const TensorDescriptor* const* indexDescs = nullptr; + ConstData_t* indexs = nullptr; + const TensorDescriptor dxDesc{}; + Data_t dx = nullptr; + const TensorDescriptor errorDesc{}; + Data_t error = nullptr; + + uint32_t dimCount = 0; + const int32_t* dims = nullptr; + uint32_t sliceCount = 0; + const int32_t* slices = nullptr; + uint32_t offset = 0; + + std::size_t GetWorkspaceSize() const { return workspace_size; } + Data_t GetWorkspace() const { return workspace; } +}; + +} // namespace getitem + +} // namespace miopen diff --git a/src/include/miopen/getitem/problem_description.hpp b/src/include/miopen/getitem/problem_description.hpp new file mode 100644 index 0000000000..fed4e78d22 --- /dev/null +++ b/src/include/miopen/getitem/problem_description.hpp @@ -0,0 +1,181 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include +#include +#include +#include + +namespace miopen { + +struct NetworkConfig; + +namespace getitem { + +struct ProblemDescription : ProblemDescriptionBase +{ + ProblemDescription(const TensorDescriptor& dyDesc_, + uint32_t indexCount_, + const TensorDescriptor* const* indexDescs_, + const TensorDescriptor& dxDesc_, + const TensorDescriptor& errorDesc_, + uint32_t dimCount_, + const int32_t* dims_, + uint32_t sliceCount_, + const int32_t* slices_, + uint32_t offset_) + : dyDesc(dyDesc_), + indexCount(indexCount_), + indexDescs(indexDescs_), + dxDesc(dxDesc_), + errorDesc(errorDesc_), + dimCount(dimCount_), + dims(dims_), + sliceCount(sliceCount_), + slices(slices_), + offset(offset_) + { + IsValidIndexsLength(); + IsValidIndexs(); + IsValidDims(); + IsValidSlices(); + } + + ProblemDescription(const int32_t indexCount_, const TensorDescriptor* const* indexDescs_) + : indexCount(indexCount_), indexDescs(indexDescs_) + { + IsValidIndexsLength(); + IsValidIndexs(); + } + + const TensorDescriptor& GetDYDesc() const { return dyDesc; } + int32_t GetIndexCount() const { return indexCount; } + const TensorDescriptor& GetIndexDesc(int i) const + { + if(i >= indexCount) + { + MIOPEN_THROW(miopenStatusInternalError, "Item: Invalid tensor index."); + } + return (*indexDescs)[i]; + } + const TensorDescriptor& GetDXDesc() const { return dxDesc; } + const TensorDescriptor& GetErrorDesc() const { return errorDesc; } + int32_t GetDimCount() const { return dimCount; } + int32_t GetDim(int i) const + { + if(i >= indexCount) + { + MIOPEN_THROW(miopenStatusInternalError, "Item: Invalid dim index."); + } + return dims[i]; + } + int32_t GetSliceCount() const { return sliceCount; } + int32_t GetSlice(int i) const + { + if(i >= sliceCount) + { + MIOPEN_THROW(miopenStatusInternalError, "Item: Invalid slice index."); + } + return slices[i]; + } + int32_t GetOffset() const { return offset; } + + bool IsValidIndexsLength() const + { + if(indexCount > 0) + { + auto firstlength = (*indexDescs)[0]; + for(int32_t i = 1; i < indexCount; ++i) + { + if(firstlength != (*indexDescs)[i]) + MIOPEN_THROW(miopenStatusBadParm, + "Getitem: Indexs dimension lengths do not match."); + } + } + return true; + } + + bool IsValidIndexs() const + { + if(indexCount > 0) + { + if(indexDescs == nullptr) + MIOPEN_THROW(miopenStatusBadParm, "Getitem: indexDesc is nullptr."); + } + return true; + } + + bool IsValidDims() const + { + if(dimCount > 0) + + if(dims == nullptr) + MIOPEN_THROW(miopenStatusBadParm, "Getitem: dims is nullptr."); + return true; + } + + bool IsValidSlices() const + { + if(sliceCount > 0) + { + if(slices == nullptr) + MIOPEN_THROW(miopenStatusBadParm, "Getitem: slices is nullptr."); + } + return true; + } + + bool IsSameType() const + { + if(dyDesc.GetType() != dxDesc.GetType()) + { + return false; + } + return true; + } + + NetworkConfig MakeNetworkConfig() const override; + +private: + TensorDescriptor dyDesc{}; + uint32_t indexCount = 0; + const TensorDescriptor* const* indexDescs = nullptr; + TensorDescriptor dxDesc{}; + TensorDescriptor errorDesc{}; + + uint32_t dimCount = 0; + const int32_t* dims = nullptr; + uint32_t sliceCount = 0; + const int32_t* slices = nullptr; + uint32_t offset = 0; + + NetworkConfig MakeForwardNetworkConfig() const; +}; + +} // namespace getitem + +} // namespace miopen diff --git a/src/include/miopen/getitem/solvers.hpp b/src/include/miopen/getitem/solvers.hpp new file mode 100644 index 0000000000..f2edcbe437 --- /dev/null +++ b/src/include/miopen/getitem/solvers.hpp @@ -0,0 +1,57 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include +#include + +namespace miopen { + +namespace solver { + +namespace getitem { + +using ItemSolver = NonTunableSolverBase; + +struct GetitemBackward final : ItemSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext& context, + const miopen::getitem::ProblemDescription& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::getitem::ProblemDescription& problem) const override; + std::size_t GetWorkspaceSize(const ExecutionContext& context, + const miopen::getitem::ProblemDescription& problem) const override; + bool MayNeedWorkspace() const override { return true; } +}; + +} // namespace getitem + +} // namespace solver + +} // namespace miopen diff --git a/src/include/miopen/sum.hpp b/src/include/miopen/glu.hpp similarity index 59% rename from src/include/miopen/sum.hpp rename to src/include/miopen/glu.hpp index b1a76e0cc6..448929ef94 100644 --- a/src/include/miopen/sum.hpp +++ b/src/include/miopen/glu.hpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (c) 2023 Advanced Micro Devices, Inc. + * Copyright (c) 2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -23,8 +23,7 @@ * SOFTWARE. * *******************************************************************************/ -#ifndef MIOPEN_SUM_HPP_ -#define MIOPEN_SUM_HPP_ +#pragma once #include @@ -33,20 +32,24 @@ namespace miopen { struct Handle; struct TensorDescriptor; -MIOPEN_INTERNALS_EXPORT std::size_t GetSumWorkspaceSize(Handle& handle, - const TensorDescriptor& xDesc, - const TensorDescriptor& yDesc, - int32_t dim); - -MIOPEN_INTERNALS_EXPORT miopenStatus_t SumForward(Handle& handle, - Data_t workspace, - size_t workspaceSizeInBytes, - const TensorDescriptor& xDesc, - ConstData_t x, - const TensorDescriptor& yDesc, - Data_t y, - miopenSumNanPropagation_t nanPropagation, - int32_t dim); +namespace glu { + +MIOPEN_INTERNALS_EXPORT miopenStatus_t GLUForward(Handle& handle, + const TensorDescriptor& inputDesc, + ConstData_t input, + const TensorDescriptor& outputDesc, + Data_t output, + uint32_t dim); + +MIOPEN_INTERNALS_EXPORT miopenStatus_t GLUBackward(Handle& handle, + const TensorDescriptor& inputDesc, + ConstData_t input, + const TensorDescriptor& outputGradDesc, + ConstData_t outputGrad, + const TensorDescriptor& inputGradDesc, + Data_t inputGrad, + uint32_t dim); + +} // namespace glu } // namespace miopen -#endif // _MIOPEN_SUM_HPP_ diff --git a/src/include/miopen/glu/invoke_params.hpp b/src/include/miopen/glu/invoke_params.hpp new file mode 100644 index 0000000000..0a8bc6fb62 --- /dev/null +++ b/src/include/miopen/glu/invoke_params.hpp @@ -0,0 +1,71 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include + +namespace miopen { +namespace glu { + +struct FwdInvokeParams : public miopen::InvokeParams +{ + FwdInvokeParams() = default; + + const TensorDescriptor* inputDesc = nullptr; + const TensorDescriptor* outputDesc = nullptr; + + ConstData_t input = nullptr; + Data_t output = nullptr; + uint32_t dim = 0; + + std::size_t GetWorkspaceSize() const { return 0; } + Data_t GetWorkspace() const { return nullptr; } +}; + +struct BwdInvokeParams : public miopen::InvokeParams +{ + BwdInvokeParams() = default; + + const TensorDescriptor* inputDesc = nullptr; + const TensorDescriptor* inputGradDesc = nullptr; + const TensorDescriptor* outputGradDesc = nullptr; + + ConstData_t input = nullptr; + ConstData_t outputGrad = nullptr; + Data_t inputGrad = nullptr; + uint32_t dim = 0; + + std::size_t GetWorkspaceSize() const { return 0; } + Data_t GetWorkspace() const { return nullptr; } +}; + +} // namespace glu + +} // namespace miopen diff --git a/src/include/miopen/glu/problem_description.hpp b/src/include/miopen/glu/problem_description.hpp new file mode 100644 index 0000000000..d5a6523d66 --- /dev/null +++ b/src/include/miopen/glu/problem_description.hpp @@ -0,0 +1,83 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include + +namespace miopen { + +struct NetworkConfig; + +namespace glu { + +enum class Direction +{ + Forward, + Backward, +}; + +struct ProblemDescription : ProblemDescriptionBase +{ + // Forward constructor + ProblemDescription(const TensorDescriptor& inputDesc_, + const TensorDescriptor& outputDesc_, + uint32_t dim_); + + // Backward constructor + ProblemDescription(const TensorDescriptor& inputDesc_, + const TensorDescriptor& outputGradDesc_, + const TensorDescriptor& inputGradDesc_, + uint32_t dim_); + + Direction GetDirection() const { return direction; } + const TensorDescriptor& GetInputDesc() const { return inputDesc; } + const TensorDescriptor& GetInputGradDesc() const { return inputGradDesc; } + const TensorDescriptor& GetOutputDesc() const { return outputDesc; } + const TensorDescriptor& GetOutputGradDesc() const { return outputGradDesc; } + uint32_t GetDim() const { return dim; } + + bool IsSameType() const; + + bool IsAllContiguous() const; + + NetworkConfig MakeNetworkConfig() const override; + +private: + Direction direction; + TensorDescriptor inputDesc; + TensorDescriptor outputDesc; + TensorDescriptor outputGradDesc; + TensorDescriptor inputGradDesc; + + uint32_t dim; +}; + +} // namespace glu + +} // namespace miopen diff --git a/src/include/miopen/glu/solvers.hpp b/src/include/miopen/glu/solvers.hpp new file mode 100644 index 0000000000..4531d94008 --- /dev/null +++ b/src/include/miopen/glu/solvers.hpp @@ -0,0 +1,63 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include + +namespace miopen { + +namespace solver { + +namespace glu { + +using GLUSolver = NonTunableSolverBase; + +struct GLUForward final : GLUSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext& context, + const miopen::glu::ProblemDescription& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::glu::ProblemDescription& problem) const override; +}; + +struct GLUBackward final : GLUSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext& context, + const miopen::glu::ProblemDescription& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::glu::ProblemDescription& problem) const override; +}; + +} // namespace glu + +} // namespace solver + +} // namespace miopen diff --git a/src/include/miopen/graphapi/conv_bias_res_add_activ_forward_executor.hpp b/src/include/miopen/graphapi/conv_bias_res_add_activ_forward_executor.hpp new file mode 100644 index 0000000000..5d4fe8cc7f --- /dev/null +++ b/src/include/miopen/graphapi/conv_bias_res_add_activ_forward_executor.hpp @@ -0,0 +1,106 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include + +#include + +namespace miopen { + +namespace graphapi { + +class ConvBiasResAddActivForwardExecutor : public GraphPatternExecutor +{ + // We dont use pointers here as we need deserialization + Tensor mXTensor; + Tensor mWTensor; + Convolution mConvolution; + int mGroupCount; + Tensor mZTensor; + Tensor mBiasTensor; + Tensor mYTensor; + float mAlpha1; + float mAlpha2; + float mActivationAlpha; + +public: + ConvBiasResAddActivForwardExecutor(Tensor* xTensor, + Tensor* wTensor, + Convolution* convolution, + int groupCount, + Tensor* zTensor, + Tensor* biasTensor, + Tensor* yTensor, + float alpha1, + float alpha2, + float activationAlpha) + : GraphPatternExecutor(), + mXTensor(*xTensor), + mWTensor(*wTensor), + mConvolution(*convolution), + mGroupCount(groupCount), + mZTensor(*zTensor), + mBiasTensor(*biasTensor), + mYTensor(*yTensor), + mAlpha1(alpha1), + mAlpha2(alpha2), + mActivationAlpha(activationAlpha) + { + } + + ConvBiasResAddActivForwardExecutor(const nlohmann::json& json); + + void execute(miopenHandle_t handle, const VariantPack& vpk) final; + + size_t getWorkspaceSize() const final { return size_t{0}; } + + nlohmann::json getJson() final; + + static constexpr const char* name = "ConvBiasResAddActivForwardExecutor"; + + struct JsonFields + { + static constexpr const char* XTensor = "x_tensor"; + static constexpr const char* WTensor = "w_tensor"; + static constexpr const char* Convolution = "convolution"; + static constexpr const char* GroupCount = "group_count"; + static constexpr const char* ZTensor = "z_tensor"; + static constexpr const char* BiasTensor = "bias_tensor"; + static constexpr const char* YTensor = "y_tensor"; + static constexpr const char* Alpha1 = "alpha1"; + static constexpr const char* Alpha2 = "alpha2"; + static constexpr const char* ActivationAlpha = "activation_alpha"; + }; +}; + +} // namespace graphapi + +} // namespace miopen diff --git a/src/include/miopen/graphapi/convolution.hpp b/src/include/miopen/graphapi/convolution.hpp index 1e826972ac..d483c0cff7 100644 --- a/src/include/miopen/graphapi/convolution.hpp +++ b/src/include/miopen/graphapi/convolution.hpp @@ -28,6 +28,7 @@ #include #include #include +#include #include @@ -93,6 +94,20 @@ class Convolution const std::vector& getPrePaddings() const noexcept { return mPrePaddings; } const std::vector& getPostPaddings() const noexcept { return mPostPaddings; } + friend void to_json(nlohmann::json& json, const Convolution& conv); + friend void from_json(const nlohmann::json& json, Convolution& conv); + + struct JsonFields + { + static constexpr const char* SpatialDims = "spatial_dims"; + static constexpr const char* Dilations = "dilations"; + static constexpr const char* FilterStrides = "filter_dtrides"; + static constexpr const char* PrePaddings = "pre_paddings"; + static constexpr const char* PostPaddings = "post_paddings"; + static constexpr const char* CompType = "comp_type"; + static constexpr const char* Mode = "mode"; + }; + private: friend class ConvolutionBuilder; }; diff --git a/src/include/miopen/graphapi/engine.hpp b/src/include/miopen/graphapi/engine.hpp index 97590461d0..abceca476f 100644 --- a/src/include/miopen/graphapi/engine.hpp +++ b/src/include/miopen/graphapi/engine.hpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -62,7 +63,6 @@ struct TensorInfo TensorInfo(miopenTensorArgumentId_t enum_id, Tensor* tens_ptr) : mEnumId(enum_id), mGraphTensor(tens_ptr) { - assert(tens_ptr); assert(mEnumId != miopenTensorArgumentIdInvalid); } @@ -78,35 +78,54 @@ using TensorInfoMap = std::unordered_map; class MIOPEN_INTERNALS_EXPORT GraphPatternExecutor { - public: virtual void execute(miopenHandle_t handle, const VariantPack& vpk) = 0; virtual size_t getWorkspaceSize() const = 0; + virtual nlohmann::json getJson() = 0; virtual ~GraphPatternExecutor(); + + struct JsonFields + { + static constexpr const char* Name = "name"; + }; }; // generic executor that uses Find 2.0 Solution class GraphExecutorFind20 : public GraphPatternExecutor { - miopenSolution_t mSolution; + Solution mSolution; std::shared_ptr mTensorInfoMap; public: - GraphExecutorFind20(miopenSolution_t sol, const std::shared_ptr& tmap) + GraphExecutorFind20(const Solution& sol, const std::shared_ptr& tmap) : GraphPatternExecutor(), mSolution(sol), mTensorInfoMap(tmap) { + MIOPEN_THROW_IF(tmap.get() == nullptr, + "TensorInfoMap should be allocated before GraphExecutorFind20 creation"); } + GraphExecutorFind20(Solution&& sol, const std::shared_ptr& tmap) + : GraphPatternExecutor(), mSolution(std::move(sol)), mTensorInfoMap(tmap) + { + MIOPEN_THROW_IF(tmap.get() == nullptr, + "TensorInfoMap should be allocated before GraphExecutorFind20 creation"); + } + + GraphExecutorFind20(const nlohmann::json& json); + void execute(miopenHandle_t handle, const VariantPack& vpk) final; size_t getWorkspaceSize() const final; - static std::unique_ptr make(miopenSolution_t sol, - const std::shared_ptr& tmap) + nlohmann::json getJson() final; + + static constexpr const char* name = "GraphExecutorFind20"; + + struct JsonFields { - GraphPatternExecutor* p = new GraphExecutorFind20(sol, tmap); - return std::unique_ptr(p); - } + static constexpr const char* Solution = "solution"; + static constexpr const char* Id2ArgumentMap = "id_to_argument_map"; + }; }; class Engine @@ -134,6 +153,16 @@ class Engine const OpGraph* getOpGraph() const { return mGraph; } OpGraph* getOpGraph() { return mGraph; } + + friend void to_json(nlohmann::json& json, const Engine& engine); + friend void from_json(const nlohmann::json& json, Engine& engine); + + struct JsonFields + { + static constexpr const char* Executor = "executor"; + static constexpr const char* GlobalIndex = "global_index"; + static constexpr const char* SmCount = "sm_count"; + }; }; class MIOPEN_INTERNALS_EXPORT EngineBuilder diff --git a/src/include/miopen/graphapi/enginecfg.hpp b/src/include/miopen/graphapi/enginecfg.hpp index 34c3483be2..d3704d575a 100644 --- a/src/include/miopen/graphapi/enginecfg.hpp +++ b/src/include/miopen/graphapi/enginecfg.hpp @@ -28,6 +28,7 @@ #include #include +#include namespace miopen { @@ -57,6 +58,16 @@ class EngineCfg const Engine& getEngine() const noexcept { return mEngine; } Engine& getEngine() noexcept { return mEngine; } + + friend void to_json(nlohmann::json& json, const EngineCfg& engineCfg) + { + json = engineCfg.mEngine; + } + + friend void from_json(const nlohmann::json& json, EngineCfg& engineCfg) + { + json.get_to(engineCfg.mEngine); + } }; /* For now we don't support tuning and a builder is not needed, diff --git a/src/include/miopen/graphapi/execution_plan.hpp b/src/include/miopen/graphapi/execution_plan.hpp index 8ad9dbd31e..1e7a989bea 100644 --- a/src/include/miopen/graphapi/execution_plan.hpp +++ b/src/include/miopen/graphapi/execution_plan.hpp @@ -28,6 +28,7 @@ #include #include +#include #include #include @@ -73,13 +74,88 @@ class MIOPEN_INTERNALS_EXPORT ExecutionPlan { return mEngineCfg.getEngine().getExecutor()->getWorkspaceSize(); } + + struct SerializationMetadata + { + uint64_t mMagicNumber; + uint64_t mVersion; + + constexpr static SerializationMetadata current() { return {0xA4C9C1597ED9E1AC, 1}; } + + struct JsonFields + { + static constexpr const char* MagicNumber = "magic_number"; + static constexpr const char* Version = "version"; + }; + + friend void to_json(nlohmann::json& json, const SerializationMetadata& metadata) + { + json = nlohmann::json{ + {JsonFields::MagicNumber, metadata.mMagicNumber}, + {JsonFields::Version, metadata.mVersion}, + }; + } + + friend void from_json(const nlohmann::json& json, SerializationMetadata& metadata) + { + json.at(JsonFields::MagicNumber).get_to(metadata.mMagicNumber); + json.at(JsonFields::Version).get_to(metadata.mVersion); + } + }; + + struct JsonFields + { + static constexpr const char* Metadata = "header"; + static constexpr const char* EngineCfg = "engine_cfg"; + static constexpr const char* IntermediateIds = "intermediate_ids"; + }; + + friend void to_json(nlohmann::json& json, const ExecutionPlan& executionPlan) + { + json = nlohmann::json{ + {JsonFields::Metadata, SerializationMetadata::current()}, + {JsonFields::EngineCfg, executionPlan.mEngineCfg}, + {JsonFields::IntermediateIds, executionPlan.mIntermediateIds}, + }; + } + + friend void from_json(const nlohmann::json& json, ExecutionPlan& executionPlan) + { + constexpr auto supportedMetadata = SerializationMetadata::current(); + + auto metadataIterator = json.find(JsonFields::Metadata); + if(metadataIterator == json.end()) + { + MIOPEN_THROW(miopenStatusInvalidValue, + "Unsupported JSON has been passed as an ExecutionPlan"); + } + + const auto metadata = metadataIterator->get(); + + if(metadata.mMagicNumber != supportedMetadata.mMagicNumber) + { + MIOPEN_THROW(miopenStatusInvalidValue, + "JSON with unsupported magic number has been passed as an ExecutionPlan"); + } + + if(metadata.mVersion != supportedMetadata.mVersion) + { + MIOPEN_THROW(miopenStatusVersionMismatch, + "JSON of an invalid version has been passed as an ExecutionPlan"); + } + + json.at(JsonFields::EngineCfg).get_to(executionPlan.mEngineCfg); + json.at(JsonFields::IntermediateIds).get_to(executionPlan.mIntermediateIds); + executionPlan.mHandle = nullptr; + } }; class MIOPEN_INTERNALS_EXPORT ExecutionPlanBuilder { private: ExecutionPlan mExecutionPlan; - bool mEngineCfgSet = false; + bool mEngineCfgSet = false; + bool mJsonRepresentationSet = false; public: ExecutionPlanBuilder& setHandle(miopenHandle_t handle) &; diff --git a/src/include/miopen/graphapi/opgraph.hpp b/src/include/miopen/graphapi/opgraph.hpp index 4a25352297..b224885f6d 100644 --- a/src/include/miopen/graphapi/opgraph.hpp +++ b/src/include/miopen/graphapi/opgraph.hpp @@ -181,8 +181,10 @@ class MIOPEN_INTERNALS_EXPORT SourceOpNode : public OpNode void addOutTensor(Tensor* tens_ptr) { - assert(!hasOutTensor(tens_ptr)); - mOutTensors.emplace_back(tens_ptr); + if(!hasOutTensor(tens_ptr)) + { + mOutTensors.emplace_back(tens_ptr); + } } }; @@ -206,8 +208,10 @@ class MIOPEN_INTERNALS_EXPORT SinkOpNode : public OpNode void addInTensor(Tensor* tens_ptr) { - assert(!hasInTensor(tens_ptr)); - mInTensors.emplace_back(tens_ptr); + if(!hasInTensor(tens_ptr)) + { + mInTensors.emplace_back(tens_ptr); + } } }; diff --git a/src/include/miopen/graphapi/tensor.hpp b/src/include/miopen/graphapi/tensor.hpp index fe6b53cbf1..4425a4f138 100644 --- a/src/include/miopen/graphapi/tensor.hpp +++ b/src/include/miopen/graphapi/tensor.hpp @@ -27,6 +27,7 @@ #include #include +#include #include #include @@ -41,6 +42,9 @@ class Tensor : public TensorDescriptor int64_t mId = 0; bool mVirtual = false; + // Deprecated + using TensorDescriptor::GetLayout_t; + public: Tensor() noexcept = default; Tensor(const Tensor&) = default; @@ -60,9 +64,7 @@ class Tensor : public TensorDescriptor const std::vector& strides, int64_t id, bool isVirtual) - : TensorDescriptor(dataType, getLayout(strides), dimensions, strides), - mId(id), - mVirtual(isVirtual) + : TensorDescriptor(dataType, dimensions, strides), mId(id), mVirtual(isVirtual) { } Tensor(miopenDataType_t dataType, @@ -70,7 +72,7 @@ class Tensor : public TensorDescriptor std::vector&& strides, int64_t id, bool isVirtual) noexcept - : TensorDescriptor(dataType, getLayout(strides), std::move(dimensions), std::move(strides)), + : TensorDescriptor(dataType, std::move(dimensions), std::move(strides)), mId(id), mVirtual(isVirtual) { @@ -79,31 +81,14 @@ class Tensor : public TensorDescriptor int64_t getId() const noexcept { return mId; } bool isVirtual() const noexcept { return mVirtual; } -private: - static miopenTensorLayout_t getLayout(const std::vector& strides) + friend void to_json(nlohmann::json& json, const Tensor& tensor); + friend void from_json(const nlohmann::json& json, Tensor& tensor); + + struct JsonFields { - if(strides.size() >= 4) - { - int stride_c = strides[1]; - - // If channels have the smallest stride, or are tied for smallest stride, then we are - // assuming NHWC format. Otherwise, assume NCHW format. - if(std::all_of(strides.cbegin(), strides.cend(), [stride_c](std::size_t x) { - return x >= stride_c; - })) - { - return strides.size() == 4 ? miopenTensorLayout_t::miopenTensorNHWC - : miopenTensorLayout_t::miopenTensorNDHWC; - } - else - { - return strides.size() == 4 ? miopenTensorLayout_t::miopenTensorNCHW - : miopenTensorLayout_t::miopenTensorNCDHW; - } - } - - return GetDefaultLayout(); - } + static constexpr const char* Id = "id"; + static constexpr const char* IsVirtual = "is_virtual"; + }; }; class MIOPEN_INTERNALS_EXPORT TensorBuilder diff --git a/src/include/miopen/graphapi/util.hpp b/src/include/miopen/graphapi/util.hpp index f9415a3630..303cefaeca 100644 --- a/src/include/miopen/graphapi/util.hpp +++ b/src/include/miopen/graphapi/util.hpp @@ -36,13 +36,12 @@ namespace miopen { namespace graphapi { -inline std::string tensorIdAsStr(const Tensor* tens_ptr) +inline std::string tensorIdAsStr(int64_t tens_id) { - int64_t id = tens_ptr->getId(); - char* b = reinterpret_cast(&id); + char* b = reinterpret_cast(&tens_id); - return {b, sizeof(id)}; + return {b, sizeof(tens_id)}; } template @@ -261,6 +260,8 @@ inline std::string_view tensorEnumIdToStr(miopenTensorArgumentId_t id) ENUM_CASE(miopenTensorMhaAmaxDK) ENUM_CASE(miopenTensorMhaAmaxDV) ENUM_CASE(miopenTensorMhaAmaxDS) + ENUM_CASE(miopenTensorMhaBias) + ENUM_CASE(miopenTensorMhaMask) default: MIOPEN_THROW(miopenStatusInternalError, "unknown tensor enum id"); } #undef ENUM_CASE diff --git a/src/include/miopen/graphapi/variant_pack.hpp b/src/include/miopen/graphapi/variant_pack.hpp index 6239b4d2f3..9d42d5fa2f 100644 --- a/src/include/miopen/graphapi/variant_pack.hpp +++ b/src/include/miopen/graphapi/variant_pack.hpp @@ -106,6 +106,7 @@ class VariantPackBuilder VariantPack mVariantPack; bool mTensorIdsSet = false; bool mDataPointersSet = false; + bool mWorkspaceSet = false; public: VariantPackBuilder& setTensorIds(const std::vector& tensorIds) & @@ -156,12 +157,8 @@ class VariantPackBuilder } VariantPackBuilder& setWorkspace(void* workspace) & { - if(workspace == nullptr) - { - MIOPEN_THROW(miopenStatusBadParm); - } - mVariantPack.mWorkspace = workspace; + mWorkspaceSet = true; return *this; } @@ -202,7 +199,7 @@ class VariantPackBuilder private: bool validate() const { - return mTensorIdsSet && mDataPointersSet && mVariantPack.mWorkspace != nullptr && + return mTensorIdsSet && mDataPointersSet && mWorkspaceSet && mVariantPack.mTensorIds.size() == mVariantPack.mDataPointers.size() && std::find(mVariantPack.mDataPointers.cbegin(), mVariantPack.mDataPointers.cend(), diff --git a/src/include/miopen/handle.hpp b/src/include/miopen/handle.hpp index c9d03658b2..27d2227292 100644 --- a/src/include/miopen/handle.hpp +++ b/src/include/miopen/handle.hpp @@ -89,7 +89,7 @@ struct MIOPEN_EXPORT Handle : miopenHandle Handle(); Handle(miopenAcceleratorQueue_t stream); Handle(Handle&&) noexcept; - ~Handle(); + virtual ~Handle(); miopenAcceleratorQueue_t GetStream() const; void SetStream(miopenAcceleratorQueue_t streamID) const; @@ -122,9 +122,18 @@ struct MIOPEN_EXPORT Handle : miopenHandle auto GetKernels(const std::string& algorithm, const std::string& network_config) const { - return this->GetKernelsImpl(algorithm, network_config) | - boost::adaptors::transformed([this](Kernel k) { return this->Run(k); }); + auto kernels = this->GetKernelsImpl(algorithm, network_config); + + std::vector kernelInvokers; + kernelInvokers.resize(kernels.size()); + std::transform(kernels.begin(), + kernels.end(), + kernelInvokers.begin(), + [this](const Kernel& k) { return this->Run(k); }); + + return kernelInvokers; } + KernelInvoke GetKernel(const std::string& algorithm, const std::string& network_config) const { auto ks = this->GetKernelsImpl(algorithm, network_config); @@ -137,8 +146,8 @@ struct MIOPEN_EXPORT Handle : miopenHandle } KernelInvoke Run(Kernel k, bool coop_launch = false) const; - const std::vector& GetKernelsImpl(const std::string& algorithm, - const std::string& network_config) const; + std::vector GetKernelsImpl(const std::string& algorithm, + const std::string& network_config) const; Program LoadProgram(const fs::path& program_name, std::string params, @@ -156,7 +165,7 @@ struct MIOPEN_EXPORT Handle : miopenHandle std::size_t GetGlobalMemorySize() const; std::size_t GetImage3dMaxWidth() const; std::size_t GetWavefrontWidth() const; - std::size_t GetMaxComputeUnits() const; + virtual std::size_t GetMaxComputeUnits() const; std::size_t GetMaxHardwareComputeUnits() const { const std::size_t num_cu = this->GetMaxComputeUnits(); @@ -165,10 +174,10 @@ struct MIOPEN_EXPORT Handle : miopenHandle } std::size_t m_MaxMemoryAllocSizeCached = 0; - std::size_t GetMaxMemoryAllocSize(); - bool CooperativeLaunchSupported() const; + virtual std::size_t GetMaxMemoryAllocSize(); + virtual bool CooperativeLaunchSupported() const; - std::string GetDeviceName() const; + virtual std::string GetDeviceName() const; const TargetProperties& GetTargetProperties() const; private: @@ -254,7 +263,13 @@ struct MIOPEN_EXPORT Handle : miopenHandle { invokers.Register({config, solver}, invoker); if(algo.has_value()) - invokers.SetAsFound1_0(config, *algo, solver); + SetAsFound1_0(config, *algo, solver); + } + + void + SetAsFound1_0(const NetworkConfig& config, const AlgorithmName& algo, const std::string& solver) + { + invokers.SetAsFound1_0(config, algo, solver); } std::optional GetInvoker(const NetworkConfig& config, diff --git a/src/include/miopen/kernel_cache.hpp b/src/include/miopen/kernel_cache.hpp index cb3eeed527..05773c1c26 100644 --- a/src/include/miopen/kernel_cache.hpp +++ b/src/include/miopen/kernel_cache.hpp @@ -50,6 +50,7 @@ #include #include #include +#include namespace miopen { @@ -81,8 +82,7 @@ class KernelCache void ClearKernels(const std::string& algorithm, const std::string& network_config); - const std::vector& GetKernels(const std::string& algorithm, - const std::string& network_config); + std::vector GetKernels(const std::string& algorithm, const std::string& network_config); bool HasProgram(const fs::path& name, const std::string& params) const; void ClearProgram(const fs::path& name, const std::string& params); @@ -92,8 +92,11 @@ class KernelCache KernelCache(); private: + void AddKernelUnsafe(Key key, Kernel k, std::size_t cache_index); + KernelMap kernel_map; ProgramMap program_map; + mutable std::shared_mutex lock; }; } // namespace miopen diff --git a/src/include/miopen/kernel_info.hpp b/src/include/miopen/kernel_info.hpp index d2571afa32..d824fe5c78 100644 --- a/src/include/miopen/kernel_info.hpp +++ b/src/include/miopen/kernel_info.hpp @@ -48,6 +48,9 @@ struct KernelInfo fs::path kernel_file; std::string kernel_name; friend std::ostream& operator<<(std::ostream& os, const KernelInfo& k); + + /// configures the working set using hip-style indices + void ConfigureHip(size_t l0, size_t l1, size_t l2, size_t g0, size_t g1, size_t g2); }; std::vector PrecompileKernels(const Handle& h, diff --git a/src/include/miopen/kthvalue.hpp b/src/include/miopen/kthvalue.hpp new file mode 100644 index 0000000000..32cb008e0b --- /dev/null +++ b/src/include/miopen/kthvalue.hpp @@ -0,0 +1,48 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#ifndef MIOPEN_KTHVALUE_HPP_ +#define MIOPEN_KTHVALUE_HPP_ + +#include + +namespace miopen { + +struct Handle; +struct TensorDescriptor; + +MIOPEN_INTERNALS_EXPORT miopenStatus_t KthvalueForward(Handle& handle, + const TensorDescriptor& inputDesc, + ConstData_t input, + const TensorDescriptor& outputDesc, + Data_t output, + const TensorDescriptor& indicesDesc, + size_t* indices, + size_t k, + int32_t dim, + bool keepDim); + +} // namespace miopen +#endif // MIOPEN_KTHVALUE_HPP_ diff --git a/src/include/miopen/kthvalue/invoke_params.hpp b/src/include/miopen/kthvalue/invoke_params.hpp new file mode 100644 index 0000000000..701538f9b9 --- /dev/null +++ b/src/include/miopen/kthvalue/invoke_params.hpp @@ -0,0 +1,55 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include + +namespace miopen { + +namespace kthvalue { + +struct FwdInvokeParams : public miopen::InvokeParams +{ + FwdInvokeParams() = default; + + const TensorDescriptor* inputDesc = nullptr; + ConstData_t input = nullptr; + const TensorDescriptor* outputDesc = nullptr; + Data_t output = nullptr; + const TensorDescriptor* indicesDesc = nullptr; + size_t* indices = nullptr; + + size_t k = 1; + int32_t dim = 0; + bool keepDim = false; + std::size_t GetWorkspaceSize() const { return 0; } + Data_t GetWorkspace() const { return nullptr; } +}; + +} // namespace kthvalue + +} // namespace miopen diff --git a/src/include/miopen/kthvalue/problem_description.hpp b/src/include/miopen/kthvalue/problem_description.hpp new file mode 100644 index 0000000000..d597ed0e45 --- /dev/null +++ b/src/include/miopen/kthvalue/problem_description.hpp @@ -0,0 +1,129 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include + +namespace miopen { + +struct NetworkConfig; + +namespace kthvalue { + +struct FwdProblemDescription : ProblemDescriptionBase +{ + FwdProblemDescription(const TensorDescriptor& inputDesc_, + const TensorDescriptor& outputDesc_, + const TensorDescriptor& indicesDesc_, + int32_t dim_, + size_t k_, + bool keepDim_) + : inputDesc(inputDesc_), + outputDesc(outputDesc_), + indicesDesc(indicesDesc_), + dim(dim_), + k(k_), + keepDim(keepDim_) + { + if(k < 1 || k > inputDesc.GetLengths()[dim]) + { + MIOPEN_THROW(miopenStatusBadParm, + "Kthvalue: selected number k out of range for dimension"); + } + if(dim < 0 || dim >= inputDesc.GetNumDims()) + { + MIOPEN_THROW(miopenStatusBadParm, "Kthvalue: dim doesn't not exist"); + } + if(inputDesc.GetType() != outputDesc.GetType()) + { + MIOPEN_THROW(miopenStatusBadParm, "Reduce: Input, output tensor types do not match."); + } + if(!IsRightLength()) + { + MIOPEN_THROW(miopenStatusBadParm, + "Reduce: Input and output tensor dimension lengths do not match."); + } + if(outputDesc.GetLengths() != indicesDesc.GetLengths()) + { + MIOPEN_THROW(miopenStatusBadParm, + "Reduce: Output and indices tensor dimension lengths do not match."); + } + } + + bool IsRightLength() const + { + if(inputDesc.GetLengths().size() == 1) + return true; + + if(keepDim && inputDesc.GetNumDims() != outputDesc.GetNumDims()) + { + return false; + } + if(!keepDim && inputDesc.GetNumDims() != outputDesc.GetNumDims() + 1) + { + return false; + } + + int32_t posOut = 0; + for(int32_t i = 0; i < inputDesc.GetLengths().size(); i++) + { + if(i == dim) + { + if(!keepDim) + continue; + if(outputDesc.GetLengths()[posOut] != 1) + return false; + } + else if(inputDesc.GetLengths()[i] != outputDesc.GetLengths()[posOut]) + { + return false; + } + + posOut++; + } + return true; + } + + const TensorDescriptor& GetInputDesc() const { return inputDesc; } + const TensorDescriptor& GetOutputDesc() const { return outputDesc; } + const TensorDescriptor& GetIndicesDesc() const { return indicesDesc; } + int32_t GetDim() const { return dim; } + size_t GetK() const { return k; } + NetworkConfig MakeNetworkConfig() const override; + +public: + TensorDescriptor inputDesc; + TensorDescriptor outputDesc; + TensorDescriptor indicesDesc; + int32_t dim; + size_t k; + bool keepDim; +}; + +} // namespace kthvalue + +} // namespace miopen diff --git a/src/include/miopen/kthvalue/solvers.hpp b/src/include/miopen/kthvalue/solvers.hpp new file mode 100644 index 0000000000..9c58795730 --- /dev/null +++ b/src/include/miopen/kthvalue/solvers.hpp @@ -0,0 +1,55 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include + +namespace miopen { + +namespace solver { + +namespace kthvalue { + +using KthvalueFwdSolverBase = + NonTunableSolverBase; + +struct KthvalueFwd final : KthvalueFwdSolverBase +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext& context, + const miopen::kthvalue::FwdProblemDescription& problem) const override; + + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::kthvalue::FwdProblemDescription& problem) const override; +}; + +} // namespace kthvalue + +} // namespace solver + +} // namespace miopen diff --git a/src/include/miopen/mha/mha.hpp b/src/include/miopen/mha/mha.hpp index 296faa6892..aa75ad4f8e 100644 --- a/src/include/miopen/mha/mha.hpp +++ b/src/include/miopen/mha/mha.hpp @@ -54,6 +54,8 @@ struct MhaInputDescsForward TensorDescriptor dropoutSeedDesc; TensorDescriptor dropoutOffsetDesc; + TensorDescriptor biasDesc; + // output tensors TensorDescriptor oDesc; TensorDescriptor amaxODesc; @@ -129,6 +131,9 @@ struct MhaDataForward ConstData_t dropoutSeedData; ConstData_t dropoutOffsetData; + ConstData_t biasData; + miopenMhaMask_t mask; + // output tensors Data_t oData; Data_t amaxOData; diff --git a/src/include/miopen/mha/solvers.hpp b/src/include/miopen/mha/solvers.hpp index 6bac473a71..55423f63c7 100644 --- a/src/include/miopen/mha/solvers.hpp +++ b/src/include/miopen/mha/solvers.hpp @@ -77,6 +77,28 @@ struct MhaBackward final : MhaSolver MIOPEN_INTERNALS_EXPORT bool MayNeedWorkspace() const override; }; +struct MhaCKFlashAttentionV2Forward final : MhaSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + + MIOPEN_INTERNALS_EXPORT bool + IsApplicable(const ExecutionContext& context, + const miopen::mha::ProblemDescription& problem) const override; + + MIOPEN_INTERNALS_EXPORT ConvSolution + GetSolution(const ExecutionContext& context, + const miopen::mha::ProblemDescription& problem) const override; + + MIOPEN_INTERNALS_EXPORT std::size_t + GetWorkspaceSize(const ExecutionContext& context, + const miopen::mha::ProblemDescription& problem) const override; + + MIOPEN_INTERNALS_EXPORT bool MayNeedWorkspace() const override; +}; + } // namespace mha } // namespace solver diff --git a/src/include/miopen/mlo_internal.hpp b/src/include/miopen/mlo_internal.hpp index 43d2e1a067..6252df3087 100644 --- a/src/include/miopen/mlo_internal.hpp +++ b/src/include/miopen/mlo_internal.hpp @@ -119,6 +119,8 @@ inline int AlignUp(int val, unsigned step) return static_cast(((static_cast(val) + step - 1) / step) * step); } +inline size_t AlignUp(size_t val, size_t step) { return (val + step - 1) / step * step; } + namespace miopen { struct TensorDescriptor; diff --git a/src/include/miopen/multimarginloss.hpp b/src/include/miopen/multimarginloss.hpp new file mode 100644 index 0000000000..fa718923a2 --- /dev/null +++ b/src/include/miopen/multimarginloss.hpp @@ -0,0 +1,63 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#ifndef MIOPEN_MULTIMARGINLOSS_HPP_ +#define MIOPEN_MULTIMARGINLOSS_HPP_ + +#include "miopen/miopen.h" +#include + +namespace miopen { + +struct Handle; +struct TensorDescriptor; + +MIOPEN_INTERNALS_EXPORT std::size_t +GetMultiMarginLossForwardWorkspaceSize(Handle& handle, + const TensorDescriptor& iDesc, + const TensorDescriptor& tDesc, + const TensorDescriptor& wDesc, + const TensorDescriptor& oDesc, + long p, + float margin, + miopenLossReductionMode_t reduction); + +MIOPEN_INTERNALS_EXPORT miopenStatus_t MultiMarginLossForward(Handle& handle, + Data_t workspace, + size_t workspaceSizeInBytes, + const TensorDescriptor& iDesc, + ConstData_t i, + const TensorDescriptor& tDesc, + ConstData_t t, + const TensorDescriptor& wDesc, + ConstData_t w, + const TensorDescriptor& oDesc, + Data_t o, + long p, + float margin, + miopenLossReductionMode_t reduction); + +} // namespace miopen +#endif // _MIOPEN_MULTIMARGINLOSS_HPP_ diff --git a/src/include/miopen/multimarginloss/invoke_params.hpp b/src/include/miopen/multimarginloss/invoke_params.hpp new file mode 100644 index 0000000000..9d8a8e3498 --- /dev/null +++ b/src/include/miopen/multimarginloss/invoke_params.hpp @@ -0,0 +1,60 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +namespace miopen { +namespace multimarginloss { + +struct InvokeParams : public miopen::InvokeParams +{ + InvokeParams() = default; + + const TensorDescriptor* iDesc = nullptr; + const TensorDescriptor* tDesc = nullptr; + const TensorDescriptor* wDesc = nullptr; + const TensorDescriptor* oDesc = nullptr; + + ConstData_t i = nullptr; + ConstData_t t = nullptr; + ConstData_t w = nullptr; + Data_t o = nullptr; + + long p; + float margin; + + Data_t workspace = nullptr; + std::size_t workspace_size = 0; + std::size_t GetWorkspaceSize() const { return workspace_size; } + Data_t GetWorkspace() const { return workspace; } +}; + +} // namespace multimarginloss + +} // namespace miopen diff --git a/src/include/miopen/multimarginloss/problem_description.hpp b/src/include/miopen/multimarginloss/problem_description.hpp new file mode 100644 index 0000000000..db1c8fa875 --- /dev/null +++ b/src/include/miopen/multimarginloss/problem_description.hpp @@ -0,0 +1,138 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include "miopen/miopen.h" +#include +#include +#include + +namespace miopen { + +struct NetworkConfig; + +namespace multimarginloss { + +struct ForwardProblemDescription : ProblemDescriptionBase +{ + ForwardProblemDescription(const TensorDescriptor& iDesc_, + const TensorDescriptor& tDesc_, + const TensorDescriptor& wDesc_, + const TensorDescriptor& oDesc_, + const long p_, + const float margin_, + const miopenLossReductionMode_t reduction_) + : iDesc(iDesc_), + tDesc(tDesc_), + wDesc(wDesc_), + oDesc(oDesc_), + p(p_), + margin(margin_), + reduction(reduction_) + { + if(iDesc.GetType() != oDesc.GetType() || iDesc.GetType() != wDesc.GetType()) + { + MIOPEN_THROW(miopenStatusBadParm, + "MultiMarginLoss: Input, output, weight tensor types do not match."); + } + if(tDesc.GetType() != miopenInt64) + { + MIOPEN_THROW(miopenStatusBadParm, + "MultiMarginLoss: Target tensor type should be miopenInt64."); + } + if(iDesc.GetNumDims() != 2) + { + MIOPEN_THROW(miopenStatusBadParm, "MultiMarginLoss: Input tensor need to be 2D tensor"); + } + if(tDesc.GetNumDims() != 1 || tDesc.GetLengths()[0] != iDesc.GetLengths()[0]) + { + MIOPEN_THROW(miopenStatusBadParm, + "MultiMarginLoss: Target tensor need to be 1D tensor. If input " + "tensor has shape (N, C) then target tensor must have shape (N)"); + } + if(wDesc.GetNumDims() != 1 || wDesc.GetLengths()[0] != iDesc.GetLengths()[1]) + { + MIOPEN_THROW(miopenStatusBadParm, + "MultiMarginLoss: Weight tensor need to be 1D tensor. If input " + "tensor has shape (N, C) then weight tensor must have shape (C)"); + } + // Check output tensor dimension + if(reduction == MIOPEN_LOSS_REDUCTION_NONE) + { + // non-reduction case + if(oDesc.GetNumDims() != 1 || oDesc.GetLengths()[0] != iDesc.GetLengths()[0]) + { + MIOPEN_THROW(miopenStatusBadParm, + "MultiMarginLoss: Output tensor need to be " + "1D tensor. If input " + "tensor has shape (N, C) then output tensor must have shape (N)"); + } + } + else + { + // reduction case + if(oDesc.GetNumDims() != 1 || oDesc.GetLengths()[0] != 1) + { + MIOPEN_THROW(miopenStatusBadParm, + "MultiMarginLoss: Output tensor need to be a scalar."); + } + } + // Check p value + if(p != 1 && p != 2) + { + MIOPEN_THROW(miopenStatusBadParm, "MultiMarginLoss: p need to be equal 1 or 2."); + } + } + + const TensorDescriptor& GetiDesc() const { return iDesc; } + const TensorDescriptor& GettDesc() const { return tDesc; } + const TensorDescriptor& GetwDesc() const { return wDesc; } + const TensorDescriptor& GetoDesc() const { return oDesc; } + long Getp() const { return p; } + float Getmargin() const { return margin; } + miopenLossReductionMode_t Getreduction() const { return reduction; } + bool allContiguousTensor() const + { + return iDesc.IsContiguous() && tDesc.IsContiguous() && wDesc.IsContiguous() && + oDesc.IsContiguous(); + } + + NetworkConfig MakeNetworkConfig() const override; + +private: + TensorDescriptor iDesc; + TensorDescriptor tDesc; + TensorDescriptor wDesc; + TensorDescriptor oDesc; + long p; + float margin; + miopenLossReductionMode_t reduction; +}; + +} // namespace multimarginloss + +} // namespace miopen diff --git a/src/include/miopen/multimarginloss/solvers.hpp b/src/include/miopen/multimarginloss/solvers.hpp new file mode 100644 index 0000000000..82c5c79910 --- /dev/null +++ b/src/include/miopen/multimarginloss/solvers.hpp @@ -0,0 +1,65 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include + +namespace miopen { + +namespace solver { + +namespace multimarginloss { + +using ForwardMultiMarginLossSolver = + NonTunableSolverBase; + +struct MultiMarginLossForward final : ForwardMultiMarginLossSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + bool + IsImprovementOverROCm(const ExecutionContext& context, + const miopen::multimarginloss::ForwardProblemDescription& problem) const; + bool + IsApplicable(const ExecutionContext& context, + const miopen::multimarginloss::ForwardProblemDescription& problem) const override; + ConvSolution + GetSolution(const ExecutionContext& context, + const miopen::multimarginloss::ForwardProblemDescription& problem) const override; + std::size_t GetWorkspaceSize( + const ExecutionContext& context, + const miopen::multimarginloss::ForwardProblemDescription& problem) const override; + bool MayNeedWorkspace() const override { return true; } +}; + +} // namespace multimarginloss + +} // namespace solver + +} // namespace miopen diff --git a/src/include/miopen/nogpu/handle_impl.hpp b/src/include/miopen/nogpu/handle_impl.hpp index 36e3aa5b76..3a1ebdf295 100644 --- a/src/include/miopen/nogpu/handle_impl.hpp +++ b/src/include/miopen/nogpu/handle_impl.hpp @@ -48,7 +48,9 @@ struct HandleImpl bool enable_profiling = false; StreamPtr stream = nullptr; rocblas_handle_ptr rhandle_; +#if MIOPEN_USE_HIPBLASLT hipblasLt_handle_ptr hip_blasLt_handle; +#endif float profiling_result = 0.0; int device = -1; std::string device_name; diff --git a/src/include/miopen/pooling.hpp b/src/include/miopen/pooling.hpp index 0ab5ffa1c7..26805f6df8 100644 --- a/src/include/miopen/pooling.hpp +++ b/src/include/miopen/pooling.hpp @@ -113,6 +113,8 @@ struct MIOPEN_EXPORT PoolingDescriptor : miopenPoolingDescriptor void SetWorkspaceIndexMode(miopenPoolingWorkspaceIndexMode_t workspace_index); miopenPoolingMode_t GetMode() const; + + bool ModeIsAveraging() const; miopenPaddingMode_t GetPaddingMode() const; diff --git a/src/include/miopen/pooling/poolingNdNhwcArgs.hpp b/src/include/miopen/pooling/poolingNdNhwcArgs.hpp new file mode 100644 index 0000000000..5084f28b4f --- /dev/null +++ b/src/include/miopen/pooling/poolingNdNhwcArgs.hpp @@ -0,0 +1,45 @@ +#pragma once + +struct poolingNdNhwcArgs +{ + uint32_t filter_d; + uint32_t filter_h; + uint32_t filter_w; + + uint32_t filter_d_stride; + uint32_t filter_h_stride; + uint32_t filter_w_stride; + + uint32_t filter_d_pad; + uint32_t filter_h_pad; + uint32_t filter_w_pad; + + uint32_t all_n; + uint32_t all_c; + + uint32_t bot_d; + uint32_t bot_h; + uint32_t bot_w; + + uint32_t bot_n_stride; + uint32_t bot_c_stride; + uint32_t bot_d_stride; + uint32_t bot_h_stride; + uint32_t bot_w_stride; + + uint32_t top_d; + uint32_t top_h; + uint32_t top_w; + + uint32_t top_n_stride; + uint32_t top_c_stride; + uint32_t top_d_stride; + uint32_t top_h_stride; + uint32_t top_w_stride; + + uint32_t mask_n_stride; + uint32_t mask_c_stride; + uint32_t mask_d_stride; + uint32_t mask_h_stride; + uint32_t mask_w_stride; +}; diff --git a/src/include/miopen/pooling/solvers.hpp b/src/include/miopen/pooling/solvers.hpp index 0d0e35755a..d8836fcdc0 100644 --- a/src/include/miopen/pooling/solvers.hpp +++ b/src/include/miopen/pooling/solvers.hpp @@ -66,6 +66,30 @@ struct PoolingForwardNd final : PoolingSolver const miopen::pooling::ProblemDescription& problem) const override; }; +struct PoolingForwardCk2d final : PoolingSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext& context, + const miopen::pooling::ProblemDescription& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::pooling::ProblemDescription& problem) const override; + std::size_t GetWorkspaceSize(const ExecutionContext& context, + const miopen::pooling::ProblemDescription& problem) const override; +}; + +struct PoolingForwardCkNd final : PoolingSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext& context, + const miopen::pooling::ProblemDescription& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::pooling::ProblemDescription& problem) const override; + std::size_t GetWorkspaceSize(const ExecutionContext& context, + const miopen::pooling::ProblemDescription& problem) const override; +}; + struct PoolingForwardNaive final : PoolingSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } @@ -79,6 +103,18 @@ struct PoolingForwardNaive final : PoolingSolver const miopen::pooling::ProblemDescription& problem) const override; }; +struct PoolingForwardNDNhwcNaive final : PoolingSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext& context, + const miopen::pooling::ProblemDescription& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::pooling::ProblemDescription& problem) const override; + std::size_t GetWorkspaceSize(const ExecutionContext& context, + const miopen::pooling::ProblemDescription& problem) const override; +}; + template struct PoolingFwdNCHWTransposingSolver : TransposingSolver, PoolingSolver, @@ -145,6 +181,18 @@ struct PoolingBackward2d final : PoolingSolver const miopen::pooling::ProblemDescription& problem) const override; }; +struct PoolingBackwardCk2d final : PoolingSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext& context, + const miopen::pooling::ProblemDescription& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::pooling::ProblemDescription& problem) const override; + std::size_t GetWorkspaceSize(const ExecutionContext& context, + const miopen::pooling::ProblemDescription& problem) const override; +}; + struct PoolingBackwardNd final : PoolingSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } @@ -157,6 +205,18 @@ struct PoolingBackwardNd final : PoolingSolver const miopen::pooling::ProblemDescription& problem) const override; }; +struct PoolingBackwardCkNd final : PoolingSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext& context, + const miopen::pooling::ProblemDescription& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::pooling::ProblemDescription& problem) const override; + std::size_t GetWorkspaceSize(const ExecutionContext& context, + const miopen::pooling::ProblemDescription& problem) const override; +}; + template struct PoolingBwdNCHWTransposingSolver : TransposingSolver, PoolingSolver, diff --git a/src/include/miopen/precalc_xorwow_skipahead_matrices.hpp b/src/include/miopen/precalc_xorwow_skipahead_matrices.hpp deleted file mode 100644 index 89ee3e38c8..0000000000 --- a/src/include/miopen/precalc_xorwow_skipahead_matrices.hpp +++ /dev/null @@ -1,3755 +0,0 @@ -#define XORWOW_DIM 5 -#define XORWOW_BITS 32 -#define XORWOW_PRECALC_MATRICES_SZ (XORWOW_BITS * XORWOW_DIM * XORWOW_DIM) -#define XORWOW_PRECALC_MATRICES_NUM 32 -#define XORWOW_JUMP_LOG2 2 -#define XORWOW_JUMP_LOG2_MASK ((1 << XORWOW_JUMP_LOG2) - 1) -#define XORWOW_SEQUENCE_JUMP_LOG2 67 - -static const unsigned int - precalc_xorwow_skipahead_matrices[XORWOW_PRECALC_MATRICES_NUM][XORWOW_PRECALC_MATRICES_SZ] = { - { - 0, 0, 0, 0, 3, 0, 0, - 0, 0, 6, 0, 0, 0, 0, - 15, 0, 0, 0, 0, 30, 0, - 0, 0, 0, 60, 0, 0, 0, - 0, 120, 0, 0, 0, 0, 240, - 0, 0, 0, 0, 480, 0, 0, - 0, 0, 960, 0, 0, 0, 0, - 1920, 0, 0, 0, 0, 3840, 0, - 0, 0, 0, 7680, 0, 0, 0, - 0, 15360, 0, 0, 0, 0, 30720, - 0, 0, 0, 0, 61440, 0, 0, - 0, 0, 122880, 0, 0, 0, 0, - 245760, 0, 0, 0, 0, 491520, 0, - 0, 0, 0, 983040, 0, 0, 0, - 0, 1966080, 0, 0, 0, 0, 3932160, - 0, 0, 0, 0, 7864320, 0, 0, - 0, 0, 15728640, 0, 0, 0, 0, - 31457280, 0, 0, 0, 0, 62914560, 0, - 0, 0, 0, 125829120, 0, 0, 0, - 0, 251658240, 0, 0, 0, 0, 503316480, - 0, 0, 0, 0, 1006632960, 0, 0, - 0, 0, 2013265920, 0, 0, 0, 0, - 4026531840, 0, 0, 0, 0, 3758096384, 1, - 0, 0, 0, 0, 2, 0, 0, - 0, 0, 4, 0, 0, 0, 0, - 8, 0, 0, 0, 0, 16, 0, - 0, 0, 0, 32, 0, 0, 0, - 0, 64, 0, 0, 0, 0, 128, - 0, 0, 0, 0, 256, 0, 0, - 0, 0, 512, 0, 0, 0, 0, - 1024, 0, 0, 0, 0, 2048, 0, - 0, 0, 0, 4096, 0, 0, 0, - 0, 8192, 0, 0, 0, 0, 16384, - 0, 0, 0, 0, 32768, 0, 0, - 0, 0, 65536, 0, 0, 0, 0, - 131072, 0, 0, 0, 0, 262144, 0, - 0, 0, 0, 524288, 0, 0, 0, - 0, 1048576, 0, 0, 0, 0, 2097152, - 0, 0, 0, 0, 4194304, 0, 0, - 0, 0, 8388608, 0, 0, 0, 0, - 16777216, 0, 0, 0, 0, 33554432, 0, - 0, 0, 0, 67108864, 0, 0, 0, - 0, 134217728, 0, 0, 0, 0, 268435456, - 0, 0, 0, 0, 536870912, 0, 0, - 0, 0, 1073741824, 0, 0, 0, 0, - 2147483648, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 2, 0, 0, - 0, 0, 4, 0, 0, 0, 0, - 8, 0, 0, 0, 0, 16, 0, - 0, 0, 0, 32, 0, 0, 0, - 0, 64, 0, 0, 0, 0, 128, - 0, 0, 0, 0, 256, 0, 0, - 0, 0, 512, 0, 0, 0, 0, - 1024, 0, 0, 0, 0, 2048, 0, - 0, 0, 0, 4096, 0, 0, 0, - 0, 8192, 0, 0, 0, 0, 16384, - 0, 0, 0, 0, 32768, 0, 0, - 0, 0, 65536, 0, 0, 0, 0, - 131072, 0, 0, 0, 0, 262144, 0, - 0, 0, 0, 524288, 0, 0, 0, - 0, 1048576, 0, 0, 0, 0, 2097152, - 0, 0, 0, 0, 4194304, 0, 0, - 0, 0, 8388608, 0, 0, 0, 0, - 16777216, 0, 0, 0, 0, 33554432, 0, - 0, 0, 0, 67108864, 0, 0, 0, - 0, 134217728, 0, 0, 0, 0, 268435456, - 0, 0, 0, 0, 536870912, 0, 0, - 0, 0, 1073741824, 0, 0, 0, 0, - 2147483648, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 2, 0, 0, - 0, 0, 4, 0, 0, 0, 0, - 8, 0, 0, 0, 0, 16, 0, - 0, 0, 0, 32, 0, 0, 0, - 0, 64, 0, 0, 0, 0, 128, - 0, 0, 0, 0, 256, 0, 0, - 0, 0, 512, 0, 0, 0, 0, - 1024, 0, 0, 0, 0, 2048, 0, - 0, 0, 0, 4096, 0, 0, 0, - 0, 8192, 0, 0, 0, 0, 16384, - 0, 0, 0, 0, 32768, 0, 0, - 0, 0, 65536, 0, 0, 0, 0, - 131072, 0, 0, 0, 0, 262144, 0, - 0, 0, 0, 524288, 0, 0, 0, - 0, 1048576, 0, 0, 0, 0, 2097152, - 0, 0, 0, 0, 4194304, 0, 0, - 0, 0, 8388608, 0, 0, 0, 0, - 16777216, 0, 0, 0, 0, 33554432, 0, - 0, 0, 0, 67108864, 0, 0, 0, - 0, 134217728, 0, 0, 0, 0, 268435456, - 0, 0, 0, 0, 536870912, 0, 0, - 0, 0, 1073741824, 0, 0, 0, 0, - 2147483648, 0, 0, 0, 0, 0, 1, - 17, 0, 0, 0, 2, 34, 0, - 0, 0, 4, 68, 0, 0, 0, - 8, 136, 0, 0, 0, 16, 272, - 0, 0, 0, 32, 544, 0, 0, - 0, 64, 1088, 0, 0, 0, 128, - 2176, 0, 0, 0, 256, 4352, 0, - 0, 0, 512, 8704, 0, 0, 0, - 1024, 17408, 0, 0, 0, 2048, 34816, - 0, 0, 0, 4096, 69632, 0, 0, - 0, 8192, 139264, 0, 0, 0, 16384, - 278528, 0, 0, 0, 32768, 557056, 0, - 0, 0, 65536, 1114112, 0, 0, 0, - 131072, 2228224, 0, 0, 0, 262144, 4456448, - 0, 0, 0, 524288, 8912896, 0, 0, - 0, 1048576, 17825792, 0, 0, 0, 2097152, - 35651584, 0, 0, 0, 4194304, 71303168, 0, - 0, 0, 8388608, 142606336, 0, 0, 0, - 16777216, 285212672, 0, 0, 0, 33554432, 570425344, - 0, 0, 0, 67108864, 1140850688, 0, 0, - 0, 134217728, 2281701376, 0, 0, 0, 268435456, - 268435456, 0, 0, 0, 536870912, 536870912, 0, - 0, 0, 1073741824, 1073741824, 0, 0, 0, - 2147483648, 2147483648, - }, - { - 0, 3, 51, 771, 13107, 0, 6, - 102, 1542, 26214, 0, 15, 255, 3855, - 65535, 0, 30, 510, 7710, 131070, 0, - 60, 1020, 15420, 262140, 0, 120, 2040, - 30840, 524280, 0, 240, 4080, 61680, 1048560, - 0, 480, 8160, 123360, 2097120, 0, 960, - 16320, 246720, 4194240, 0, 1920, 32640, 493440, - 8388480, 0, 3840, 65280, 986880, 16776960, 0, - 7680, 130560, 1973760, 33553920, 0, 15360, 261120, - 3947520, 67107840, 0, 30720, 522240, 7895040, 134215680, - 0, 61440, 1044480, 15790080, 268431360, 0, 122880, - 2088960, 31580160, 536862720, 0, 245760, 4177920, 63160320, - 1073725440, 0, 491520, 8355840, 126320640, 2147450880, 0, - 983040, 16711680, 252641280, 4294901760, 0, 1966080, 33423360, - 505282560, 4294836224, 0, 3932160, 66846720, 1010565120, 4294705152, - 0, 7864320, 133693440, 2021130240, 4294443008, 0, 15728640, - 267386880, 4042260480, 4293918720, 0, 31457280, 534773760, 3789553664, - 4292870144, 0, 62914560, 1069547520, 3284140032, 4290772992, 0, - 125829120, 2139095040, 2273312768, 4286578688, 0, 251658240, 4278190080, - 251658240, 4278190080, 0, 503316480, 4261412864, 503316480, 4261412864, - 0, 1006632960, 4227858432, 1006632960, 4227858432, 0, 2013265920, - 4160749568, 2013265920, 4160749568, 0, 4026531840, 4026531840, 4026531840, - 4026531840, 0, 3758096384, 3758096384, 3758096384, 3758096384, 0, - 0, 3, 51, 771, 0, 0, 6, - 102, 1542, 0, 0, 15, 255, 3855, - 0, 0, 30, 510, 7710, 0, 0, - 60, 1020, 15420, 0, 0, 120, 2040, - 30840, 0, 0, 240, 4080, 61680, 0, - 0, 480, 8160, 123360, 0, 0, 960, - 16320, 246720, 0, 0, 1920, 32640, 493440, - 0, 0, 3840, 65280, 986880, 0, 0, - 7680, 130560, 1973760, 0, 0, 15360, 261120, - 3947520, 0, 0, 30720, 522240, 7895040, 0, - 0, 61440, 1044480, 15790080, 0, 0, 122880, - 2088960, 31580160, 0, 0, 245760, 4177920, 63160320, - 0, 0, 491520, 8355840, 126320640, 0, 0, - 983040, 16711680, 252641280, 0, 0, 1966080, 33423360, - 505282560, 0, 0, 3932160, 66846720, 1010565120, 0, - 0, 7864320, 133693440, 2021130240, 0, 0, 15728640, - 267386880, 4042260480, 0, 0, 31457280, 534773760, 3789553664, - 0, 0, 62914560, 1069547520, 3284140032, 0, 0, - 125829120, 2139095040, 2273312768, 0, 0, 251658240, 4278190080, - 251658240, 0, 0, 503316480, 4261412864, 503316480, 0, - 0, 1006632960, 4227858432, 1006632960, 0, 0, 2013265920, - 4160749568, 2013265920, 0, 0, 4026531840, 4026531840, 4026531840, - 0, 0, 3758096384, 3758096384, 3758096384, 0, 0, - 0, 3, 51, 0, 0, 0, 6, - 102, 0, 0, 0, 15, 255, 0, - 0, 0, 30, 510, 0, 0, 0, - 60, 1020, 0, 0, 0, 120, 2040, - 0, 0, 0, 240, 4080, 0, 0, - 0, 480, 8160, 0, 0, 0, 960, - 16320, 0, 0, 0, 1920, 32640, 0, - 0, 0, 3840, 65280, 0, 0, 0, - 7680, 130560, 0, 0, 0, 15360, 261120, - 0, 0, 0, 30720, 522240, 0, 0, - 0, 61440, 1044480, 0, 0, 0, 122880, - 2088960, 0, 0, 0, 245760, 4177920, 0, - 0, 0, 491520, 8355840, 0, 0, 0, - 983040, 16711680, 0, 0, 0, 1966080, 33423360, - 0, 0, 0, 3932160, 66846720, 0, 0, - 0, 7864320, 133693440, 0, 0, 0, 15728640, - 267386880, 0, 0, 0, 31457280, 534773760, 0, - 0, 0, 62914560, 1069547520, 0, 0, 0, - 125829120, 2139095040, 0, 0, 0, 251658240, 4278190080, - 0, 0, 0, 503316480, 4261412864, 0, 0, - 0, 1006632960, 4227858432, 0, 0, 0, 2013265920, - 4160749568, 0, 0, 0, 4026531840, 4026531840, 0, - 0, 0, 3758096384, 3758096384, 0, 0, 0, - 0, 3, 0, 0, 0, 0, 6, - 0, 0, 0, 0, 15, 0, 0, - 0, 0, 30, 0, 0, 0, 0, - 60, 0, 0, 0, 0, 120, 0, - 0, 0, 0, 240, 0, 0, 0, - 0, 480, 0, 0, 0, 0, 960, - 0, 0, 0, 0, 1920, 0, 0, - 0, 0, 3840, 0, 0, 0, 0, - 7680, 0, 0, 0, 0, 15360, 0, - 0, 0, 0, 30720, 0, 0, 0, - 0, 61440, 0, 0, 0, 0, 122880, - 0, 0, 0, 0, 245760, 0, 0, - 0, 0, 491520, 0, 0, 0, 0, - 983040, 0, 0, 0, 0, 1966080, 0, - 0, 0, 0, 3932160, 0, 0, 0, - 0, 7864320, 0, 0, 0, 0, 15728640, - 0, 0, 0, 0, 31457280, 0, 0, - 0, 0, 62914560, 0, 0, 0, 0, - 125829120, 0, 0, 0, 0, 251658240, 0, - 0, 0, 0, 503316480, 0, 0, 0, - 0, 1006632960, 0, 0, 0, 0, 2013265920, - 0, 0, 0, 0, 4026531840, 0, 0, - 0, 0, 3758096384, 1, 17, 257, 4369, - 65537, 2, 34, 514, 8738, 131074, 4, - 68, 1028, 17476, 262148, 8, 136, 2056, - 34952, 524296, 16, 272, 4112, 69904, 1048592, - 32, 544, 8224, 139808, 2097184, 64, 1088, - 16448, 279616, 4194368, 128, 2176, 32896, 559232, - 8388736, 256, 4352, 65792, 1118464, 16777472, 512, - 8704, 131584, 2236928, 33554944, 1024, 17408, 263168, - 4473856, 67109888, 2048, 34816, 526336, 8947712, 134219776, - 4096, 69632, 1052672, 17895424, 268439552, 8192, 139264, - 2105344, 35790848, 536879104, 16384, 278528, 4210688, 71581696, - 1073758208, 32768, 557056, 8421376, 143163392, 2147516416, 65536, - 1114112, 16842752, 286326784, 65536, 131072, 2228224, 33685504, - 572653568, 131072, 262144, 4456448, 67371008, 1145307136, 262144, - 524288, 8912896, 134742016, 2290614272, 524288, 1048576, 17825792, - 269484032, 286261248, 1048576, 2097152, 35651584, 538968064, 572522496, - 2097152, 4194304, 71303168, 1077936128, 1145044992, 4194304, 8388608, - 142606336, 2155872256, 2290089984, 8388608, 16777216, 285212672, 16777216, - 285212672, 16777216, 33554432, 570425344, 33554432, 570425344, 33554432, - 67108864, 1140850688, 67108864, 1140850688, 67108864, 134217728, 2281701376, - 134217728, 2281701376, 134217728, 268435456, 268435456, 268435456, 268435456, - 268435456, 536870912, 536870912, 536870912, 536870912, 536870912, 1073741824, - 1073741824, 1073741824, 1073741824, 1073741824, 2147483648, 2147483648, 2147483648, - 2147483648, 2147483648, - }, - { - 85009117, 335741939, 1412632518, 386859243, 1741437244, 152139416, 403047142, - 2556825231, 505087203, 4287193174, 335609039, 336528191, 1425998811, 456920088, - 2832198590, 724748988, 3625845630, 1509824181, 3330088197, 2710488401, 1431742057, - 1077674236, 1140592489, 2096905276, 3007294393, 2863484114, 1081606648, 1207443154, - 972585080, 2793363314, 1432000919, 1089470704, 1341132452, 3019109363, 2362285522, - 1790260014, 2178941408, 2682264904, 1743251430, 429603751, 359294556, 62915520, - 1069562512, 3486502860, 859207501, 3939814584, 125831040, 2139125024, 2678038424, - 1718415002, 363436400, 251662080, 4278250048, 1061109552, 3436830004, 3948098272, - 503324160, 4261532800, 2122219104, 2310257256, 380003776, 1006648320, 4228098304, - 4244438208, 3278337232, 3981233024, 2013296640, 4161229312, 4193909120, 2530142624, - 446273280, 4026593280, 4027491328, 871625472, 4254978880, 4113772032, 3758219264, - 3760015360, 2011686400, 3946555008, 711351296, 3221471232, 3225063424, 4291808256, - 108481792, 2496444416, 2147975168, 2155159552, 4020213760, 485399040, 3919147008, - 983040, 15351808, 255799296, 3923588096, 322101248, 1966080, 299139072, - 511598592, 3283773440, 3865427968, 3932160, 4087939072, 1023197184, 1467273216, - 214663168, 7864320, 4149346304, 2046394368, 3202981888, 3650551808, 3236954112, - 1050935296, 871563264, 2916302848, 1932394496, 2447376384, 1833435136, 2011561984, - 2342944768, 643563520, 868220928, 177209344, 4291559424, 122486784, 2360868864, - 2004877312, 85983232, 4019716096, 3734634496, 3647995904, 1056964608, 3661627392, - 254803968, 2905866240, 1658847232, 2113929216, 3028287488, 3730833408, 2322071552, - 3586129920, 4227858432, 1761607680, 2092957696, 80740352, 2071986176, 4160749568, - 3523215360, 964689920, 429916160, 3875536896, 4026531840, 2751463424, 1929379840, - 4081057792, 503316480, 3758096384, 2281701376, 4127195136, 3397386240, 1316635, - 85009117, 335741939, 1412632518, 386859243, 1580547, 152139416, 403047142, - 2556825231, 505087203, 1317672, 335609039, 336528191, 1425998811, 456920088, - 1574501, 724748988, 3625845630, 1509824181, 3330088197, 15612, 1431742057, - 1077674236, 1140592489, 2096905276, 31224, 2863484114, 1081606648, 1207443154, - 972585080, 62451, 1432000919, 1089470704, 1341132452, 3019109363, 124902, - 1790260014, 2178941408, 2682264904, 1743251430, 249804, 359294556, 62915520, - 1069562512, 3486502860, 499608, 3939814584, 125831040, 2139125024, 2678038424, - 999216, 363436400, 251662080, 4278250048, 1061109552, 3223223904, 3948098272, - 503324160, 4261532800, 2122219104, 1077738688, 380003776, 1006648320, 4228098304, - 4244438208, 1081735552, 3981233024, 2013296640, 4161229312, 4193909120, 1089729280, - 446273280, 4026593280, 4027491328, 871625472, 2179458560, 4113772032, 3758219264, - 3760015360, 2011686400, 63949824, 711351296, 3221471232, 3225063424, 4291808256, - 127899648, 2496444416, 2147975168, 2155159552, 4020213760, 255799296, 3919147008, - 983040, 15351808, 255799296, 3732824064, 322101248, 1966080, 299139072, - 511598592, 2096939008, 3865427968, 3932160, 4087939072, 1023197184, 972652544, - 214663168, 7864320, 4149346304, 2046394368, 3019046912, 3650551808, 3236954112, - 1050935296, 871563264, 1743126528, 1932394496, 2447376384, 1833435136, 2011561984, - 3486253056, 643563520, 868220928, 177209344, 4291559424, 2677538816, 2360868864, - 2004877312, 85983232, 4019716096, 1060110336, 3647995904, 1056964608, 3661627392, - 254803968, 3193962496, 1658847232, 2113929216, 3028287488, 3730833408, 3166699520, - 3586129920, 4227858432, 1761607680, 2092957696, 3112173568, 2071986176, 4160749568, - 3523215360, 964689920, 1929379840, 3875536896, 4026531840, 2751463424, 1929379840, - 4127195136, 503316480, 3758096384, 2281701376, 4127195136, 332854, 1316635, - 85009117, 335741939, 1412632518, 596079, 1580547, 152139416, 403047142, - 2556825231, 1316075, 1317672, 335609039, 336528191, 1425998811, 2824661, - 1574501, 724748988, 3625845630, 1509824181, 5571497, 15612, 1431742057, - 1077674236, 1140592489, 11142994, 31224, 2863484114, 1081606648, 1207443154, - 22285988, 62451, 1432000919, 1089470704, 1341132452, 44571976, 124902, - 1790260014, 2178941408, 2682264904, 89143952, 249804, 359294556, 62915520, - 1069562512, 178287904, 499608, 3939814584, 125831040, 2139125024, 356575808, - 999216, 363436400, 251662080, 4278250048, 713151616, 3223223904, 3948098272, - 503324160, 4261532800, 1426303232, 1077738688, 380003776, 1006648320, 4228098304, - 2852606464, 1081735552, 3981233024, 2013296640, 4161229312, 1410245632, 1089729280, - 446273280, 4026593280, 4027491328, 1746749440, 2179458560, 4113772032, 3758219264, - 3760015360, 272273408, 63949824, 711351296, 3221471232, 3225063424, 3765772288, - 127899648, 2496444416, 2147975168, 2155159552, 15351808, 255799296, 3919147008, - 983040, 15351808, 3251929088, 3732824064, 322101248, 1966080, 299139072, - 1135149056, 2096939008, 3865427968, 3932160, 4087939072, 1196556288, 972652544, - 214663168, 7864320, 4149346304, 1319370752, 3019046912, 3650551808, 3236954112, - 1050935296, 2638741504, 1743126528, 1932394496, 2447376384, 1833435136, 982515712, - 3486253056, 643563520, 868220928, 177209344, 1965031424, 2677538816, 2360868864, - 2004877312, 85983232, 3930062848, 1060110336, 3647995904, 1056964608, 3661627392, - 3565158400, 3193962496, 1658847232, 2113929216, 3028287488, 2835349504, 3166699520, - 3586129920, 4227858432, 1761607680, 1375731712, 3112173568, 2071986176, 4160749568, - 3523215360, 2751463424, 1929379840, 3875536896, 4026531840, 2751463424, 2281701376, - 4127195136, 503316480, 3758096384, 2281701376, 5123, 332854, 1316635, - 85009117, 335741939, 6150, 596079, 1580547, 152139416, 403047142, - 5135, 1316075, 1317672, 335609039, 336528191, 6174, 2824661, - 1574501, 724748988, 3625845630, 60, 5571497, 15612, 1431742057, - 1077674236, 120, 11142994, 31224, 2863484114, 1081606648, 240, - 22285988, 62451, 1432000919, 1089470704, 480, 44571976, 124902, - 1790260014, 2178941408, 960, 89143952, 249804, 359294556, 62915520, - 1920, 178287904, 499608, 3939814584, 125831040, 3840, 356575808, - 999216, 363436400, 251662080, 7680, 713151616, 3223223904, 3948098272, - 503324160, 15360, 1426303232, 1077738688, 380003776, 1006648320, 30720, - 2852606464, 1081735552, 3981233024, 2013296640, 61440, 1410245632, 1089729280, - 446273280, 4026593280, 122880, 1746749440, 2179458560, 4113772032, 3758219264, - 245760, 272273408, 63949824, 711351296, 3221471232, 491520, 3765772288, - 127899648, 2496444416, 2147975168, 983040, 15351808, 255799296, 3919147008, - 983040, 3223191552, 3251929088, 3732824064, 322101248, 1966080, 1077673984, - 1135149056, 2096939008, 3865427968, 3932160, 1081606144, 1196556288, 972652544, - 214663168, 7864320, 1089470464, 1319370752, 3019046912, 3650551808, 3236954112, - 2178940928, 2638741504, 1743126528, 1932394496, 2447376384, 62914560, 982515712, - 3486253056, 643563520, 868220928, 125829120, 1965031424, 2677538816, 2360868864, - 2004877312, 251658240, 3930062848, 1060110336, 3647995904, 1056964608, 503316480, - 3565158400, 3193962496, 1658847232, 2113929216, 1006632960, 2835349504, 3166699520, - 3586129920, 4227858432, 2013265920, 1375731712, 3112173568, 2071986176, 4160749568, - 4026531840, 2751463424, 1929379840, 3875536896, 4026531840, 3758096384, 2281701376, - 4127195136, 503316480, 3758096384, 201392209, 3423671362, 218366296, 3713336838, - 206572594, 402785186, 2552372100, 436928947, 3130605370, 463476848, 262468, - 4461835, 68158800, 1158700908, 20971524, 524680, 8919318, 136513955, - 2316537326, 25165852, 3222274064, 3239051564, 3494187077, 3558090985, 3221225500, - 2149580832, 2183135832, 2693406858, 2821214674, 2147483704, 4194368, 71304368, - 1091846420, 1347462055, 64, 8388736, 142608736, 2183692840, 2694924110, - 3221225600, 16777472, 285217472, 72418384, 1094880924, 1342177536, 33554944, - 570434944, 144836768, 2189761848, 2684355072, 67109888, 1140869888, 289673536, - 84556400, 1073742848, 134219776, 2281739776, 579347072, 169112800, 2147485696, - 268439552, 268512256, 1158694144, 69790144, 4096, 536879104, 537024512, - 2317388288, 3360805760, 8192, 1073758208, 1074049024, 339809280, 1352902400, - 16384, 2147516416, 2148098048, 3900844032, 1632062976, 32768, 65536, - 1228800, 17059840, 311335936, 65536, 131072, 2457600, 34119680, - 622671872, 131072, 262144, 4915200, 68239360, 1245343744, 262144, - 524288, 9830400, 136478720, 2490687488, 524288, 1048576, 288096256, - 272957440, 954843136, 3222274048, 2097152, 3797417984, 545914880, 2983428096, - 2149580800, 4194304, 78643200, 1091829760, 2745630720, 4194304, 3229614080, - 3378511872, 1109917696, 2270035968, 8388608, 1358954496, 1119879168, 1414529024, - 513540096, 16777216, 2717908992, 2239758336, 2829058048, 1027080192, 33554432, - 1140850688, 184549376, 1363148800, 2054160384, 3288334336, 2281701376, 369098752, - 2726297600, 4108320768, 2281701376, 268435456, 738197504, 2231369728, 968884224, - 3959422976, 536870912, 1476395008, 167772160, 3011510272, 3355443200, 1073741824, - 2952790016, 335544320, 1728053248, 2147483648, 2147483648, 1610612736, 3892314112, - 503316480, 0, - }, - { - 1939838472, 1412147404, 166205219, 1757484276, 2905930693, 2345662040, 2845657161, - 253454719, 2661974169, 303781080, 4075331504, 31014156, 244538930, 3752264221, - 992575155, 219309525, 246620060, 215640989, 4125020723, 2016731730, 3236558869, - 297169276, 3293566751, 1867504216, 210423272, 2531663658, 499723753, 1730625896, - 189236880, 3388575408, 2433358422, 1368961148, 3134096848, 2827836415, 3888822753, - 4172043647, 3379360748, 2651760955, 1345081091, 627692776, 189423917, 1927379456, - 4004336944, 2995932065, 1882016234, 2551113616, 1576396048, 1299792730, 2151240795, - 2154814108, 4292139924, 3555849728, 943986992, 3169912733, 2631635779, 3478094562, - 1285558544, 3716074330, 2780749859, 3911106510, 4175656994, 1731832828, 1275401375, - 937322456, 3802094750, 1145506936, 1008905193, 1718801768, 645739137, 1356219146, - 827886816, 1722154800, 2242776733, 754630810, 772070504, 249481170, 2608123425, - 2087201889, 3200968096, 3292110026, 841433255, 477543427, 1878882709, 705347364, - 4003860146, 3194913138, 2616490007, 357561212, 2446098297, 2955680594, 2512991743, - 637464579, 1209132455, 1341312804, 612108672, 2455017713, 1749147666, 4020226825, - 2873924220, 499405095, 1837614076, 1227604028, 714577577, 165950208, 442290261, - 489077752, 216760440, 42151250, 426862080, 2810242474, 4112075489, 3514761468, - 4101921371, 982512636, 500792667, 4286077681, 198050301, 1858712743, 2913642493, - 3547545255, 3981929169, 2944140287, 2286578015, 3422343167, 1239123295, 2026367394, - 3269986302, 3028402878, 2709637886, 1096011710, 294584132, 3086749695, 3324400975, - 1164394495, 4290155855, 543687304, 4008517630, 836370334, 1876426750, 2362048414, - 3578325264, 3221487612, 2671154748, 3395518460, 2018383420, 2131029536, 2165829624, - 697661816, 1336049656, 3309365624, 4259639360, 3423548400, 2416417776, 1633698800, - 1630071792, 41950336, 3423478496, 2885608160, 3943744224, 677380832, 4179285363, - 1939838472, 1412147404, 166205219, 1757484276, 3838244595, 2345662040, 2845657161, - 253454719, 2661974169, 138737288, 4075331504, 31014156, 244538930, 3752264221, - 1503392345, 219309525, 246620060, 215640989, 4125020723, 1759481152, 3236558869, - 297169276, 3293566751, 1867504216, 3898070400, 2531663658, 499723753, 1730625896, - 189236880, 2610231010, 2433358422, 1368961148, 3134096848, 2827836415, 3903474593, - 4172043647, 3379360748, 2651760955, 1345081091, 1267864331, 189423917, 1927379456, - 4004336944, 2995932065, 3452816347, 2551113616, 1576396048, 1299792730, 2151240795, - 1222520631, 4292139924, 3555849728, 943986992, 3169912733, 3260130211, 3478094562, - 1285558544, 3716074330, 2780749859, 3039362306, 4175656994, 1731832828, 1275401375, - 937322456, 3236754932, 1145506936, 1008905193, 1718801768, 645739137, 1358079399, - 827886816, 1722154800, 2242776733, 754630810, 1748663943, 249481170, 2608123425, - 2087201889, 3200968096, 698076610, 841433255, 477543427, 1878882709, 705347364, - 3692794996, 3194913138, 2616490007, 357561212, 2446098297, 2771068186, 2512991743, - 637464579, 1209132455, 1341312804, 27937268, 2455017713, 1749147666, 4020226825, - 2873924220, 1673040956, 1837614076, 1227604028, 714577577, 165950208, 528340088, - 489077752, 216760440, 42151250, 426862080, 1646215396, 4112075489, 3514761468, - 4101921371, 982512636, 2095821304, 4286077681, 198050301, 1858712743, 2913642493, - 277300160, 3981929169, 2944140287, 2286578015, 3422343167, 1178044288, 2026367394, - 3269986302, 3028402878, 2709637886, 2234191616, 294584132, 3086749695, 3324400975, - 1164394495, 136978944, 543687304, 4008517630, 836370334, 1876426750, 3275253760, - 3578325264, 3221487612, 2671154748, 3395518460, 3942394880, 2131029536, 2165829624, - 697661816, 1336049656, 3265045504, 4259639360, 3423548400, 2416417776, 1633698800, - 3943712768, 41950336, 3423478496, 2885608160, 3943744224, 2293593009, 4179285363, - 1939838472, 1412147404, 166205219, 715714152, 3838244595, 2345662040, 2845657161, - 253454719, 3758048260, 138737288, 4075331504, 31014156, 244538930, 370671650, - 1503392345, 219309525, 246620060, 215640989, 2219162331, 1759481152, 3236558869, - 297169276, 3293566751, 135243402, 3898070400, 2531663658, 499723753, 1730625896, - 3142293713, 2610231010, 2433358422, 1368961148, 3134096848, 486949791, 3903474593, - 4172043647, 3379360748, 2651760955, 3172880550, 1267864331, 189423917, 1927379456, - 4004336944, 191463910, 3452816347, 2551113616, 1576396048, 1299792730, 4411574, - 1222520631, 4292139924, 3555849728, 943986992, 3073348038, 3260130211, 3478094562, - 1285558544, 3716074330, 3098363790, 3039362306, 4175656994, 1731832828, 1275401375, - 468159532, 3236754932, 1145506936, 1008905193, 1718801768, 1092964081, 1358079399, - 827886816, 1722154800, 2242776733, 53128947, 1748663943, 249481170, 2608123425, - 2087201889, 1960144614, 698076610, 841433255, 477543427, 1878882709, 1505419004, - 3692794996, 3194913138, 2616490007, 357561212, 2823143358, 2771068186, 2512991743, - 637464579, 1209132455, 1991737212, 27937268, 2455017713, 1749147666, 4020226825, - 2907896812, 1673040956, 1837614076, 1227604028, 714577577, 3633969112, 528340088, - 489077752, 216760440, 42151250, 2886728356, 1646215396, 4112075489, 3514761468, - 4101921371, 3507686008, 2095821304, 4286077681, 198050301, 1858712743, 1463806912, - 277300160, 3981929169, 2944140287, 2286578015, 4137888640, 1178044288, 2026367394, - 3269986302, 3028402878, 1276820224, 2234191616, 294584132, 3086749695, 3324400975, - 4274031104, 136978944, 543687304, 4008517630, 836370334, 2978609152, 3275253760, - 3578325264, 3221487612, 2671154748, 2296777728, 3942394880, 2131029536, 2165829624, - 697661816, 1086645248, 3265045504, 4259639360, 3423548400, 2416417776, 2295121920, - 3943712768, 41950336, 3423478496, 2885608160, 3290486993, 2293593009, 4179285363, - 1939838472, 1412147404, 3718742914, 715714152, 3838244595, 2345662040, 2845657161, - 3251034248, 3758048260, 138737288, 4075331504, 31014156, 2257801369, 370671650, - 1503392345, 219309525, 246620060, 1375177854, 2219162331, 1759481152, 3236558869, - 297169276, 2981812236, 135243402, 3898070400, 2531663658, 499723753, 1103465850, - 3142293713, 2610231010, 2433358422, 1368961148, 2570001060, 486949791, 3903474593, - 4172043647, 3379360748, 1922171925, 3172880550, 1267864331, 189423917, 1927379456, - 1359812359, 191463910, 3452816347, 2551113616, 1576396048, 2518549525, 4411574, - 1222520631, 4292139924, 3555849728, 949028615, 3073348038, 3260130211, 3478094562, - 1285558544, 4113039486, 3098363790, 3039362306, 4175656994, 1731832828, 1827471372, - 468159532, 3236754932, 1145506936, 1008905193, 1626341859, 1092964081, 1358079399, - 827886816, 1722154800, 1069547583, 53128947, 1748663943, 249481170, 2608123425, - 3162506114, 1960144614, 698076610, 841433255, 477543427, 3641706484, 1505419004, - 3692794996, 3194913138, 2616490007, 3623882586, 2823143358, 2771068186, 2512991743, - 637464579, 16785012, 1991737212, 27937268, 2455017713, 1749147666, 2348825660, - 2907896812, 1673040956, 1837614076, 1227604028, 2579527800, 3633969112, 528340088, - 489077752, 216760440, 3628134628, 2886728356, 1646215396, 4112075489, 3514761468, - 1602085368, 3507686008, 2095821304, 4286077681, 198050301, 2501362624, 1463806912, - 277300160, 3981929169, 2944140287, 4112467840, 4137888640, 1178044288, 2026367394, - 3269986302, 3356184320, 1276820224, 2234191616, 294584132, 3086749695, 366387712, - 4274031104, 136978944, 543687304, 4008517630, 1006135296, 2978609152, 3275253760, - 3578325264, 3221487612, 3104844800, 2296777728, 3942394880, 2131029536, 2165829624, - 1874371584, 1086645248, 3265045504, 4259639360, 3423548400, 2975352832, 2295121920, - 3943712768, 41950336, 3423478496, 989898496, 3410688577, 2331788830, 3546482013, - 813828841, 1865093068, 3265457506, 3795669738, 2119696024, 4285651426, 3333834629, - 3451487261, 2090324595, 1816963648, 932961512, 2470761029, 3401764108, 3421619354, - 4199624502, 589386372, 879396240, 3372470254, 2693109296, 2424215996, 38442268, - 1882087724, 171397600, 2024561281, 183095586, 3282207272, 3402177296, 1859195498, - 413109947, 2839537944, 1632143648, 3742715856, 388696500, 1748703733, 3563198567, - 3826785440, 2896086528, 3989037829, 1478787788, 1390277813, 2123320736, 3416516800, - 2056564203, 2584895011, 1605192736, 2475623616, 3856499712, 3439657984, 708088129, - 1501395566, 1302184960, 1360092352, 1645630430, 1425230387, 3369488824, 2979863936, - 869212432, 150548847, 1097557362, 655939640, 316553344, 3761918508, 3958338094, - 141744600, 1412214640, 1859689984, 3200680981, 3883058679, 999801880, 3946079738, - 1876072704, 194381849, 2177533995, 1584707624, 3053768410, 2593051904, 3458076673, - 4047442835, 3545972808, 3441793178, 194975744, 1731731470, 4168755162, 2628944732, - 2125675784, 3119906816, 960774145, 2646626078, 2152793157, 3049156634, 672464896, - 3046932493, 3700727536, 2152335477, 575986696, 671940608, 2208366608, 1454456125, - 937760016, 4103979069, 2737668096, 1179779104, 1030912634, 1041902112, 2032909434, - 2274230272, 2089025605, 3050632421, 2428784965, 140658149, 4254138368, 1745354889, - 711584249, 2746523017, 2551006457, 1100808192, 1494221073, 3422999489, 2696954129, - 976716737, 2653421568, 3806331426, 3690047362, 1481392674, 3817015170, 2353004544, - 286262340, 2300534532, 4206449732, 15339268, 2894069760, 488376456, 1489927688, - 1196583048, 652746248, 2214592512, 69904, 1006205200, 2322628880, 1229515024, - 2617245696, 3423527456, 1964953120, 4260938272, 386199072, 1744830464, 1342444608, - 1069330496, 2138592320, 3185897536, 1073741824, 1342493824, 3780942976, 1771066496, - 2189433984, 2147483648, - }, - { - 1804684571, 2106089606, 1533056158, 2870216110, 3618155659, 3789871366, 4246691682, - 3667072763, 1212241769, 3152390668, 2973497449, 2958641966, 2088805328, 717518631, - 2401090860, 3606967204, 952637656, 59827581, 1291486682, 1499453515, 2053994857, - 563998083, 4094000396, 1163546899, 1003843565, 654565639, 1070907026, 4217851863, - 426034251, 1721352737, 278404469, 3899800390, 1063362170, 1162348262, 3153545093, - 3249996223, 186674553, 2616406148, 3137968354, 1282784965, 1495068058, 3033760361, - 2278144523, 3192245769, 719586342, 2602548287, 3386583150, 355354345, 3252815848, - 2178056037, 2283016801, 3005955037, 3340254490, 802791670, 251122316, 3705188626, - 1252262272, 3989036796, 3527490452, 2047131255, 1447170583, 3373930285, 2895037457, - 209341805, 1820357643, 3712392731, 685796521, 1322920440, 814388470, 1357857147, - 434430265, 2650681935, 1371566728, 58783716, 2273435933, 3498513198, 792571900, - 1447808772, 3513385860, 99175889, 1105434360, 1484146625, 3327194068, 242672513, - 3552105593, 1425844616, 2871928454, 1124633561, 607610433, 2130018608, 1610235673, - 2844230432, 2748082340, 994392866, 450823250, 2912535126, 2574390988, 3974009252, - 78696582, 649682891, 3980917176, 3221419689, 960695436, 729221508, 358358845, - 3392407691, 472711005, 295914899, 3005191796, 3078521977, 3370011868, 509135340, - 1965939519, 2086465877, 2457949822, 1324152522, 762289386, 3618693997, 233730715, - 2873984650, 31168606, 3367142977, 2851851305, 3251660053, 4209768406, 3298190175, - 901235185, 1564391510, 2352686527, 1008150482, 578573310, 3462447127, 2482873876, - 1790221257, 2255375608, 2335345651, 1381450613, 2866805101, 1495073163, 519905259, - 3184556473, 1076378339, 2692926127, 970097715, 4013407916, 4014350363, 2476927059, - 1989070516, 2640060069, 1987784589, 1880989003, 3861138803, 451743296, 1987067871, - 1975657871, 3397816882, 2309900530, 4108425851, 4063867233, 3319482186, 2621772886, - 1804684571, 2106089606, 1533056158, 2870216110, 611557097, 3789871366, 4246691682, - 3667072763, 1212241769, 3389551988, 2973497449, 2958641966, 2088805328, 717518631, - 2460955430, 3606967204, 952637656, 59827581, 1291486682, 3531087304, 2053994857, - 563998083, 4094000396, 1163546899, 1242934125, 654565639, 1070907026, 4217851863, - 426034251, 3034416129, 278404469, 3899800390, 1063362170, 1162348262, 4258714417, - 3249996223, 186674553, 2616406148, 3137968354, 639885806, 1495068058, 3033760361, - 2278144523, 3192245769, 4159910300, 2602548287, 3386583150, 355354345, 3252815848, - 1555885880, 2283016801, 3005955037, 3340254490, 802791670, 2948774612, 3705188626, - 1252262272, 3989036796, 3527490452, 2107826711, 1447170583, 3373930285, 2895037457, - 209341805, 3763367196, 3712392731, 685796521, 1322920440, 814388470, 1986168339, - 434430265, 2650681935, 1371566728, 58783716, 1423189187, 3498513198, 792571900, - 1447808772, 3513385860, 315969823, 1105434360, 1484146625, 3327194068, 242672513, - 3336228275, 1425844616, 2871928454, 1124633561, 607610433, 1762052458, 1610235673, - 2844230432, 2748082340, 994392866, 3771702243, 2912535126, 2574390988, 3974009252, - 78696582, 1626628844, 3980917176, 3221419689, 960695436, 729221508, 382092233, - 3392407691, 472711005, 295914899, 3005191796, 514297204, 3370011868, 509135340, - 1965939519, 2086465877, 3975975091, 1324152522, 762289386, 3618693997, 233730715, - 455322516, 31168606, 3367142977, 2851851305, 3251660053, 3952189603, 3298190175, - 901235185, 1564391510, 2352686527, 826181452, 578573310, 3462447127, 2482873876, - 1790221257, 1529242773, 2335345651, 1381450613, 2866805101, 1495073163, 877718651, - 3184556473, 1076378339, 2692926127, 970097715, 299344245, 4014350363, 2476927059, - 1989070516, 2640060069, 3844531327, 1880989003, 3861138803, 451743296, 1987067871, - 3272848161, 3397816882, 2309900530, 4108425851, 4063867233, 834288064, 2621772886, - 1804684571, 2106089606, 1533056158, 304865970, 611557097, 3789871366, 4246691682, - 3667072763, 2728206193, 3389551988, 2973497449, 2958641966, 2088805328, 3895037582, - 2460955430, 3606967204, 952637656, 59827581, 2349212526, 3531087304, 2053994857, - 563998083, 4094000396, 4028900485, 1242934125, 654565639, 1070907026, 4217851863, - 1663452176, 3034416129, 278404469, 3899800390, 1063362170, 2721441405, 4258714417, - 3249996223, 186674553, 2616406148, 4228837490, 639885806, 1495068058, 3033760361, - 2278144523, 2820661772, 4159910300, 2602548287, 3386583150, 355354345, 1815256314, - 1555885880, 2283016801, 3005955037, 3340254490, 2166514144, 2948774612, 3705188626, - 1252262272, 3989036796, 751187322, 2107826711, 1447170583, 3373930285, 2895037457, - 2809311944, 3763367196, 3712392731, 685796521, 1322920440, 936300677, 1986168339, - 434430265, 2650681935, 1371566728, 1308015359, 1423189187, 3498513198, 792571900, - 1447808772, 3065349526, 315969823, 1105434360, 1484146625, 3327194068, 1038676789, - 3336228275, 1425844616, 2871928454, 1124633561, 2956422231, 1762052458, 1610235673, - 2844230432, 2748082340, 3603862093, 3771702243, 2912535126, 2574390988, 3974009252, - 1691332448, 1626628844, 3980917176, 3221419689, 960695436, 3120142427, 382092233, - 3392407691, 472711005, 295914899, 4101686983, 514297204, 3370011868, 509135340, - 1965939519, 3015736706, 3975975091, 1324152522, 762289386, 3618693997, 2395097989, - 455322516, 31168606, 3367142977, 2851851305, 30511955, 3952189603, 3298190175, - 901235185, 1564391510, 2606298633, 826181452, 578573310, 3462447127, 2482873876, - 4159642946, 1529242773, 2335345651, 1381450613, 2866805101, 1782913669, 877718651, - 3184556473, 1076378339, 2692926127, 1730328819, 299344245, 4014350363, 2476927059, - 1989070516, 1425685614, 3844531327, 1880989003, 3861138803, 451743296, 889237383, - 3272848161, 3397816882, 2309900530, 4108425851, 1155723231, 834288064, 2621772886, - 1804684571, 2106089606, 2387009004, 304865970, 611557097, 3789871366, 4246691682, - 1405709661, 2728206193, 3389551988, 2973497449, 2958641966, 3183906006, 3895037582, - 2460955430, 3606967204, 952637656, 1345432763, 2349212526, 3531087304, 2053994857, - 563998083, 3749011414, 4028900485, 1242934125, 654565639, 1070907026, 1072342672, - 1663452176, 3034416129, 278404469, 3899800390, 3566652188, 2721441405, 4258714417, - 3249996223, 186674553, 4001263143, 4228837490, 639885806, 1495068058, 3033760361, - 4278332644, 2820661772, 4159910300, 2602548287, 3386583150, 838831089, 1815256314, - 1555885880, 2283016801, 3005955037, 3377397178, 2166514144, 2948774612, 3705188626, - 1252262272, 2414422575, 751187322, 2107826711, 1447170583, 3373930285, 1253755033, - 2809311944, 3763367196, 3712392731, 685796521, 3238624475, 936300677, 1986168339, - 434430265, 2650681935, 1642290570, 1308015359, 1423189187, 3498513198, 792571900, - 173318140, 3065349526, 315969823, 1105434360, 1484146625, 4103797777, 1038676789, - 3336228275, 1425844616, 2871928454, 1797745765, 2956422231, 1762052458, 1610235673, - 2844230432, 2180656608, 3603862093, 3771702243, 2912535126, 2574390988, 1183098390, - 1691332448, 1626628844, 3980917176, 3221419689, 2645203959, 3120142427, 382092233, - 3392407691, 472711005, 1659659070, 4101686983, 514297204, 3370011868, 509135340, - 483888155, 3015736706, 3975975091, 1324152522, 762289386, 1259948064, 2395097989, - 455322516, 31168606, 3367142977, 339990414, 30511955, 3952189603, 3298190175, - 901235185, 3097920065, 2606298633, 826181452, 578573310, 3462447127, 1548039839, - 4159642946, 1529242773, 2335345651, 1381450613, 2173079994, 1782913669, 877718651, - 3184556473, 1076378339, 1570275057, 1730328819, 299344245, 4014350363, 2476927059, - 1845882881, 1425685614, 3844531327, 1880989003, 3861138803, 1322409081, 889237383, - 3272848161, 3397816882, 2309900530, 3505447982, 3430136873, 1319796589, 4202423979, - 3184732284, 2910356648, 2534615223, 3854465731, 768821792, 2205052576, 1348983754, - 1300250188, 2919181738, 2520178732, 3967243685, 2646012002, 1784678658, 741302051, - 3464753547, 194213376, 1482799064, 3009673860, 680824208, 741966796, 2381283369, - 3022877171, 1619439814, 3961433610, 1331297670, 1100110820, 1311672539, 1122110615, - 4056004850, 3413790176, 3148768822, 1242592694, 2925975727, 1879285134, 334328879, - 1318235222, 3140739559, 401691770, 3604288404, 3686496908, 770670945, 199139043, - 2092710473, 3914528993, 700991333, 2375775811, 858137308, 3490050165, 2389078291, - 1615607459, 3027969809, 820012549, 2085659484, 2654485136, 2630408646, 196481396, - 1119673274, 1026209692, 726501622, 2940737143, 3559571163, 2288027726, 1039212708, - 929664536, 1061981465, 186058675, 3537656152, 844176796, 2996217992, 1545798611, - 3031020656, 2248030435, 1665857580, 2905758082, 1269201312, 3031275084, 4034872841, - 983632400, 4188503190, 757119675, 2105920865, 4281032819, 2917801076, 3900010013, - 3910997169, 1729751422, 562313247, 3070846353, 2564238664, 4050540186, 4258833501, - 2270666053, 2207128401, 2990540001, 797768898, 2288390225, 3230323685, 1974727440, - 3327301426, 289857826, 3565889868, 2791014422, 2021097820, 3350378271, 3673707591, - 2610067927, 4255789547, 2682856590, 12563128, 1397542366, 237149400, 2233707508, - 3875573245, 2097374144, 175320773, 4103445984, 4089284323, 3610168130, 3084915964, - 680145366, 2571684685, 1132894909, 104640024, 193765521, 2338202907, 895271448, - 11499099, 1798066417, 1297412626, 2511347162, 3140535007, 2129963538, 700683199, - 2609700278, 2953463279, 2290844145, 1871316353, 3993801787, 2219413182, 2954453701, - 231283580, 1375331115, 207723994, 1799562537, 2056553564, 2513609799, 3542459627, - 3173012714, 3923404932, 217877755, 2095124912, 192024370, 1168134987, 1889598668, - 3014873069, 2033573343, - }, - { - 3465348660, 3623545008, 3505902593, 838034830, 1338018789, 2595329276, 3367746385, - 3197935201, 1439351946, 3585085571, 4165798087, 3634792639, 2359485974, 2772582925, - 1110186203, 3771562484, 1508694157, 1564641206, 2801985736, 2446107936, 3849126897, - 1842973671, 944408104, 2624631280, 2729080685, 3737368614, 858809173, 2289802345, - 2428186575, 3114742765, 716011303, 3443810690, 814132610, 517432787, 614445393, - 2930433345, 291178098, 2117644502, 2749446703, 311745701, 365684723, 1705418876, - 2213749318, 4011417220, 1842575651, 988348831, 94258998, 2771150272, 498058526, - 1344827813, 2961955291, 262703473, 1404034822, 1566595865, 2522381203, 1706522206, - 1203054806, 1273801539, 2070583465, 3913449936, 3231505231, 619636751, 3746997351, - 4103027837, 1205468203, 3355878253, 3433356888, 107785753, 2779092609, 1869691566, - 2555219983, 903319808, 3273374169, 2538926990, 979533870, 1356500860, 1661983738, - 1380761625, 2919458459, 1041142798, 1430817627, 517007606, 1421570516, 2371447300, - 2985632691, 3684889351, 3873926653, 788770697, 1854750277, 209332297, 1137299679, - 848527832, 3850486924, 4179307312, 2764470693, 1353191605, 4166891919, 2074703841, - 3373997532, 2013528640, 701389744, 841917592, 2065742268, 2721848192, 2566956680, - 3122896007, 1090761479, 921859028, 4086736376, 1837462309, 2579826431, 2436217134, - 839037727, 1072086642, 614518622, 3764758228, 1501128342, 3669108708, 1601407381, - 2899014005, 3268308948, 3337564231, 1986911578, 3379194930, 1950365753, 2098537451, - 51515980, 1176526086, 3213391582, 1059745735, 2273586703, 376085505, 1493749800, - 3970342143, 1620925244, 2165301314, 2332030190, 1864098798, 276747442, 2776569227, - 2992780663, 3027279789, 1074555384, 3481518659, 2499703783, 661805703, 3782305562, - 9186074, 2357407210, 2355922343, 2024733363, 485434612, 862379913, 1029706268, - 1512726310, 3834948354, 1435892840, 3297980694, 2831553800, 2111416471, 711321697, - 3465348660, 3623545008, 3505902593, 838034830, 1553436793, 2595329276, 3367746385, - 3197935201, 1439351946, 3198044157, 4165798087, 3634792639, 2359485974, 2772582925, - 836042976, 3771562484, 1508694157, 1564641206, 2801985736, 1190371491, 3849126897, - 1842973671, 944408104, 2624631280, 410746791, 3737368614, 858809173, 2289802345, - 2428186575, 1542325976, 716011303, 3443810690, 814132610, 517432787, 1649301063, - 2930433345, 291178098, 2117644502, 2749446703, 3955511579, 365684723, 1705418876, - 2213749318, 4011417220, 2753632862, 988348831, 94258998, 2771150272, 498058526, - 3314106168, 2961955291, 262703473, 1404034822, 1566595865, 3590367097, 1706522206, - 1203054806, 1273801539, 2070583465, 2340683261, 3231505231, 619636751, 3746997351, - 4103027837, 2785398766, 3355878253, 3433356888, 107785753, 2779092609, 1608451840, - 2555219983, 903319808, 3273374169, 2538926990, 645164419, 1356500860, 1661983738, - 1380761625, 2919458459, 2260224548, 1430817627, 517007606, 1421570516, 2371447300, - 1636004496, 3684889351, 3873926653, 788770697, 1854750277, 1345251011, 1137299679, - 848527832, 3850486924, 4179307312, 3576574608, 1353191605, 4166891919, 2074703841, - 3373997532, 183447754, 701389744, 841917592, 2065742268, 2721848192, 2109289891, - 3122896007, 1090761479, 921859028, 4086736376, 2212730874, 2579826431, 2436217134, - 839037727, 1072086642, 55934784, 3764758228, 1501128342, 3669108708, 1601407381, - 516550987, 3268308948, 3337564231, 1986911578, 3379194930, 3973484473, 2098537451, - 51515980, 1176526086, 3213391582, 4251661633, 2273586703, 376085505, 1493749800, - 3970342143, 3190791788, 2165301314, 2332030190, 1864098798, 276747442, 2991976613, - 2992780663, 3027279789, 1074555384, 3481518659, 1399789494, 661805703, 3782305562, - 9186074, 2357407210, 1942736967, 2024733363, 485434612, 862379913, 1029706268, - 4122704494, 3834948354, 1435892840, 3297980694, 2831553800, 1210092654, 711321697, - 3465348660, 3623545008, 3505902593, 3443231198, 1553436793, 2595329276, 3367746385, - 3197935201, 1304974987, 3198044157, 4165798087, 3634792639, 2359485974, 3518323362, - 836042976, 3771562484, 1508694157, 1564641206, 3577633375, 1190371491, 3849126897, - 1842973671, 944408104, 1854555112, 410746791, 3737368614, 858809173, 2289802345, - 3622671731, 1542325976, 716011303, 3443810690, 814132610, 296197011, 1649301063, - 2930433345, 291178098, 2117644502, 1056271538, 3955511579, 365684723, 1705418876, - 2213749318, 1258535671, 2753632862, 988348831, 94258998, 2771150272, 3669902097, - 3314106168, 2961955291, 262703473, 1404034822, 1654433938, 3590367097, 1706522206, - 1203054806, 1273801539, 2448138887, 2340683261, 3231505231, 619636751, 3746997351, - 1454088394, 2785398766, 3355878253, 3433356888, 107785753, 689323470, 1608451840, - 2555219983, 903319808, 3273374169, 1603842392, 645164419, 1356500860, 1661983738, - 1380761625, 2814639423, 2260224548, 1430817627, 517007606, 1421570516, 1938805701, - 1636004496, 3684889351, 3873926653, 788770697, 4238900666, 1345251011, 1137299679, - 848527832, 3850486924, 108793827, 3576574608, 1353191605, 4166891919, 2074703841, - 3780897861, 183447754, 701389744, 841917592, 2065742268, 3036602746, 2109289891, - 3122896007, 1090761479, 921859028, 3499985398, 2212730874, 2579826431, 2436217134, - 839037727, 3520354700, 55934784, 3764758228, 1501128342, 3669108708, 1601010847, - 516550987, 3268308948, 3337564231, 1986911578, 2704241781, 3973484473, 2098537451, - 51515980, 1176526086, 3602010532, 4251661633, 2273586703, 376085505, 1493749800, - 2922957328, 3190791788, 2165301314, 2332030190, 1864098798, 1649666443, 2991976613, - 2992780663, 3027279789, 1074555384, 2848531519, 1399789494, 661805703, 3782305562, - 9186074, 320781315, 1942736967, 2024733363, 485434612, 862379913, 3598892066, - 4122704494, 3834948354, 1435892840, 3297980694, 545184652, 1210092654, 711321697, - 3465348660, 3623545008, 1173753045, 3443231198, 1553436793, 2595329276, 3367746385, - 2444634476, 1304974987, 3198044157, 4165798087, 3634792639, 1837035806, 3518323362, - 836042976, 3771562484, 1508694157, 2899021294, 3577633375, 1190371491, 3849126897, - 1842973671, 1614215215, 1854555112, 410746791, 3737368614, 858809173, 525745365, - 3622671731, 1542325976, 716011303, 3443810690, 566299749, 296197011, 1649301063, - 2930433345, 291178098, 1987532525, 1056271538, 3955511579, 365684723, 1705418876, - 2321222760, 1258535671, 2753632862, 988348831, 94258998, 2986060366, 3669902097, - 3314106168, 2961955291, 262703473, 604452796, 1654433938, 3590367097, 1706522206, - 1203054806, 1894894069, 2448138887, 2340683261, 3231505231, 619636751, 6680729, - 1454088394, 2785398766, 3355878253, 3433356888, 2025591660, 689323470, 1608451840, - 2555219983, 903319808, 3430384385, 1603842392, 645164419, 1356500860, 1661983738, - 2108736152, 2814639423, 2260224548, 1430817627, 517007606, 2973658959, 1938805701, - 1636004496, 3684889351, 3873926653, 2283691941, 4238900666, 1345251011, 1137299679, - 848527832, 45551112, 108793827, 3576574608, 1353191605, 4166891919, 3776615962, - 3780897861, 183447754, 701389744, 841917592, 3830639316, 3036602746, 2109289891, - 3122896007, 1090761479, 1931255897, 3499985398, 2212730874, 2579826431, 2436217134, - 3272166055, 3520354700, 55934784, 3764758228, 1501128342, 1567864246, 1601010847, - 516550987, 3268308948, 3337564231, 3918802424, 2704241781, 3973484473, 2098537451, - 51515980, 3551394489, 3602010532, 4251661633, 2273586703, 376085505, 885459498, - 2922957328, 3190791788, 2165301314, 2332030190, 3197056515, 1649666443, 2991976613, - 2992780663, 3027279789, 2385348906, 2848531519, 1399789494, 661805703, 3782305562, - 2163075465, 320781315, 1942736967, 2024733363, 485434612, 2680597981, 3598892066, - 4122704494, 3834948354, 1435892840, 2499644163, 2704575422, 2579557838, 673530532, - 493730767, 1124557747, 1908629439, 2821949504, 1743112513, 2849457841, 2344409314, - 3479159262, 4260973770, 2991970754, 3812641863, 2229319917, 2466968521, 1766353737, - 3216591612, 2113272648, 364370737, 1893001758, 2608875275, 4224057183, 3546705413, - 1999778009, 348872225, 2470564216, 1417878284, 2709790112, 3579129936, 2137971615, - 4046639861, 2841156930, 391544737, 2056567354, 737657378, 3877904725, 578930752, - 1759172471, 3383278785, 1047197514, 649468151, 3452867243, 1792089520, 63936215, - 3909143729, 3753489875, 734314122, 2490530916, 3043874586, 1504812057, 59001199, - 2493748676, 2552438622, 1889694845, 3715397860, 2817245010, 3841049206, 816106718, - 2176130406, 640254735, 12376903, 3000264936, 3304116079, 1620334094, 2109391765, - 1348210951, 2237645681, 1207768272, 1562894669, 2156631655, 1387193235, 3154858817, - 633510901, 2312190757, 402878244, 2501565021, 2984409334, 4167491216, 3614267292, - 3078552271, 971722322, 3065543880, 2307584190, 491480322, 2068673112, 1929780632, - 178549964, 983979983, 2769314886, 4214442042, 2977609682, 25450683, 3075212658, - 1571149568, 3531670561, 42782504, 425601306, 428715214, 497250251, 693520802, - 166426814, 1786382125, 2712003995, 3610802197, 2076490757, 404822980, 3953184772, - 1655231947, 3594351577, 3068232274, 3771730346, 4110519574, 3534704897, 2375277865, - 3597780202, 3472676002, 1350276449, 3218248239, 3589255283, 3253132633, 1769885529, - 3792812294, 120332643, 1219374788, 3608889019, 2386099811, 858495304, 1284785543, - 331370962, 2259419662, 2519864134, 3194739432, 2669074511, 2565559140, 3378072004, - 2647801475, 265068954, 1464416963, 1232787612, 4160089759, 2510685972, 670300081, - 2509357766, 1981891975, 4161588397, 1371924626, 44760868, 634955171, 1187096933, - 3324788972, 3576888559, 2801347752, 3730298395, 1702170762, 4206083415, 741409141, - 3649731355, 1025429529, - }, - { - 91444490, 628576944, 4069219862, 2253058925, 492354082, 1191182242, 1565180119, - 2257613723, 456055162, 605712223, 953365104, 3104638527, 1133984729, 2662828416, - 2134948274, 1921384447, 843719355, 588432962, 1734575434, 2924140067, 483396548, - 3848838894, 3155476556, 1760928304, 4168059840, 3279827269, 2644461735, 4168565656, - 3951563569, 1276805504, 1708974143, 1878547888, 3465220024, 3062086782, 2801401651, - 1510428126, 716404149, 1646021208, 3534932385, 1186585561, 651997355, 282914223, - 352224857, 3764407517, 1059868753, 1971798134, 978904005, 976413661, 4039544152, - 498989693, 2565125471, 2782642813, 3537961025, 1194967362, 169217024, 3491609, - 1319592872, 1630206561, 2497130840, 1685008996, 2828944016, 3301346775, 2893072371, - 2606559798, 4026138031, 2664450619, 691091062, 1079640113, 1417637732, 4081852209, - 2197910648, 2310382370, 1000957047, 959936499, 2844551811, 2272766890, 31122394, - 2742925483, 1121884686, 57929089, 2468361281, 2982007782, 2371576893, 177782593, - 3603584577, 672057044, 2108452841, 1671338057, 3386908223, 1243029765, 805157552, - 1271858417, 1621249501, 1804851492, 1321010403, 751773221, 1517221627, 822709871, - 104533154, 3578182264, 640541709, 421086624, 4233576392, 3729339369, 197460644, - 773140636, 2158026018, 1756785611, 4011575991, 3569445500, 736117181, 2456162322, - 1168189787, 3651312675, 1070291988, 268231205, 541474497, 3316168972, 3546990856, - 830417208, 725960194, 2044207227, 3188997938, 2383298579, 3350316374, 3575011225, - 1553111865, 1285013027, 749371711, 766611716, 598195098, 2139882719, 2062405428, - 3634702446, 3015263295, 223311969, 2622859522, 3888492701, 2955257225, 582625650, - 3563756446, 2886083960, 1907546514, 454650902, 3287277541, 625828138, 2991888140, - 1935326370, 4031152256, 702881509, 1427632724, 1345475301, 2577560804, 2858595147, - 2533191188, 185662179, 536505093, 3747894147, 111551030, 370373207, 2293908590, - 91444490, 628576944, 4069219862, 2253058925, 1671484924, 1191182242, 1565180119, - 2257613723, 456055162, 3411094744, 953365104, 3104638527, 1133984729, 2662828416, - 2000630022, 1921384447, 843719355, 588432962, 1734575434, 3293926122, 483396548, - 3848838894, 3155476556, 1760928304, 146876953, 3279827269, 2644461735, 4168565656, - 3951563569, 3976156700, 1708974143, 1878547888, 3465220024, 3062086782, 1999154400, - 1510428126, 716404149, 1646021208, 3534932385, 2479551429, 651997355, 282914223, - 352224857, 3764407517, 1275979651, 1971798134, 978904005, 976413661, 4039544152, - 300654823, 2565125471, 2782642813, 3537961025, 1194967362, 3123973648, 3491609, - 1319592872, 1630206561, 2497130840, 1437913158, 2828944016, 3301346775, 2893072371, - 2606559798, 2153172585, 2664450619, 691091062, 1079640113, 1417637732, 17137237, - 2197910648, 2310382370, 1000957047, 959936499, 802137134, 2272766890, 31122394, - 2742925483, 1121884686, 3909775167, 2468361281, 2982007782, 2371576893, 177782593, - 3319492525, 672057044, 2108452841, 1671338057, 3386908223, 1878151473, 805157552, - 1271858417, 1621249501, 1804851492, 3215921223, 751773221, 1517221627, 822709871, - 104533154, 361845001, 640541709, 421086624, 4233576392, 3729339369, 2655936801, - 773140636, 2158026018, 1756785611, 4011575991, 587202971, 736117181, 2456162322, - 1168189787, 3651312675, 2517883370, 268231205, 541474497, 3316168972, 3546990856, - 2037251305, 725960194, 2044207227, 3188997938, 2383298579, 2665008587, 3575011225, - 1553111865, 1285013027, 749371711, 2163964019, 598195098, 2139882719, 2062405428, - 3634702446, 2788202059, 223311969, 2622859522, 3888492701, 2955257225, 740986174, - 3563756446, 2886083960, 1907546514, 454650902, 2426323587, 625828138, 2991888140, - 1935326370, 4031152256, 1831149435, 1427632724, 1345475301, 2577560804, 2858595147, - 3977153945, 185662179, 536505093, 3747894147, 111551030, 4131587422, 2293908590, - 91444490, 628576944, 4069219862, 2408189350, 1671484924, 1191182242, 1565180119, - 2257613723, 1338069254, 3411094744, 953365104, 3104638527, 1133984729, 631497759, - 2000630022, 1921384447, 843719355, 588432962, 3280318959, 3293926122, 483396548, - 3848838894, 3155476556, 1777918163, 146876953, 3279827269, 2644461735, 4168565656, - 2786264663, 3976156700, 1708974143, 1878547888, 3465220024, 2793923820, 1999154400, - 1510428126, 716404149, 1646021208, 3102243824, 2479551429, 651997355, 282914223, - 352224857, 3767702588, 1275979651, 1971798134, 978904005, 976413661, 1951622548, - 300654823, 2565125471, 2782642813, 3537961025, 2186817324, 3123973648, 3491609, - 1319592872, 1630206561, 1075424534, 1437913158, 2828944016, 3301346775, 2893072371, - 207992406, 2153172585, 2664450619, 691091062, 1079640113, 3114255216, 17137237, - 2197910648, 2310382370, 1000957047, 2548008553, 802137134, 2272766890, 31122394, - 2742925483, 4069482373, 3909775167, 2468361281, 2982007782, 2371576893, 2807823912, - 3319492525, 672057044, 2108452841, 1671338057, 12831353, 1878151473, 805157552, - 1271858417, 1621249501, 461887094, 3215921223, 751773221, 1517221627, 822709871, - 1317394918, 361845001, 640541709, 421086624, 4233576392, 3385587450, 2655936801, - 773140636, 2158026018, 1756785611, 1475601973, 587202971, 736117181, 2456162322, - 1168189787, 911455077, 2517883370, 268231205, 541474497, 3316168972, 1500275507, - 2037251305, 725960194, 2044207227, 3188997938, 2036633808, 2665008587, 3575011225, - 1553111865, 1285013027, 87868216, 2163964019, 598195098, 2139882719, 2062405428, - 517907301, 2788202059, 223311969, 2622859522, 3888492701, 3926046234, 740986174, - 3563756446, 2886083960, 1907546514, 1911066215, 2426323587, 625828138, 2991888140, - 1935326370, 2031853435, 1831149435, 1427632724, 1345475301, 2577560804, 3509674153, - 3977153945, 185662179, 536505093, 3747894147, 1711714600, 4131587422, 2293908590, - 91444490, 628576944, 3370678255, 2408189350, 1671484924, 1191182242, 1565180119, - 3786239592, 1338069254, 3411094744, 953365104, 3104638527, 3659647225, 631497759, - 2000630022, 1921384447, 843719355, 3364831282, 3280318959, 3293926122, 483396548, - 3848838894, 3131266478, 1777918163, 146876953, 3279827269, 2644461735, 4156372383, - 2786264663, 3976156700, 1708974143, 1878547888, 2168041590, 2793923820, 1999154400, - 1510428126, 716404149, 3392113666, 3102243824, 2479551429, 651997355, 282914223, - 2085613514, 3767702588, 1275979651, 1971798134, 978904005, 503506384, 1951622548, - 300654823, 2565125471, 2782642813, 1458431750, 2186817324, 3123973648, 3491609, - 1319592872, 452433679, 1075424534, 1437913158, 2828944016, 3301346775, 2333281307, - 207992406, 2153172585, 2664450619, 691091062, 3553502652, 3114255216, 17137237, - 2197910648, 2310382370, 3153689868, 2548008553, 802137134, 2272766890, 31122394, - 468580641, 4069482373, 3909775167, 2468361281, 2982007782, 1445286890, 2807823912, - 3319492525, 672057044, 2108452841, 1755577669, 12831353, 1878151473, 805157552, - 1271858417, 2623540912, 461887094, 3215921223, 751773221, 1517221627, 3922191946, - 1317394918, 361845001, 640541709, 421086624, 2173849516, 3385587450, 2655936801, - 773140636, 2158026018, 1085377158, 1475601973, 587202971, 736117181, 2456162322, - 2158960374, 911455077, 2517883370, 268231205, 541474497, 943191315, 1500275507, - 2037251305, 725960194, 2044207227, 2481150802, 2036633808, 2665008587, 3575011225, - 1553111865, 2301231777, 87868216, 2163964019, 598195098, 2139882719, 2007840238, - 517907301, 2788202059, 223311969, 2622859522, 151920263, 3926046234, 740986174, - 3563756446, 2886083960, 1338937928, 1911066215, 2426323587, 625828138, 2991888140, - 2652286195, 2031853435, 1831149435, 1427632724, 1345475301, 289801789, 3509674153, - 3977153945, 185662179, 536505093, 2727322952, 3980498348, 2529622213, 1903052964, - 3564714651, 2281240568, 533384122, 277613480, 1815540358, 282763841, 3669112623, - 2572859425, 195220178, 1210883545, 2359703600, 1187537824, 675732974, 325036095, - 708091465, 2556854604, 701006284, 2378459191, 1863513103, 2690918197, 4237307694, - 1356483501, 2160905652, 521809106, 974368613, 3136010957, 2722488678, 3711515637, - 2296341459, 4233729945, 1196247571, 3031398071, 515543502, 1314129776, 3235373306, - 1303165859, 1820568009, 559099351, 186876368, 1076102111, 1218809551, 1790301111, - 4130210229, 768125358, 1132864749, 4262563773, 2294411020, 4092943985, 2558108246, - 3737664949, 2219923393, 724326159, 4134105682, 4188752746, 3615233671, 1526018731, - 2281637916, 2459490295, 3637342666, 777862587, 39962002, 3772005832, 997473319, - 574843584, 3356551974, 1265234427, 1698059437, 534747571, 1465532164, 3263029035, - 534512444, 2343092827, 2375685652, 2497926141, 2377933621, 2212335180, 261114084, - 172755755, 2737085495, 2225257145, 148605658, 1353911796, 357753009, 1778732943, - 497635558, 4136467976, 2837964962, 4045039047, 2485296762, 1587587183, 4042904168, - 3184240963, 2393293696, 915444966, 2299938515, 3351580749, 506575598, 1541916825, - 3465300401, 525927458, 681152801, 331660975, 3624685846, 2994172100, 3274369082, - 3638287602, 815689760, 1710961092, 2775607076, 2175058103, 3252688367, 2936890483, - 2746319120, 2736754, 1646031035, 2448701214, 2886833213, 3689830606, 3292798106, - 300773646, 3125160783, 1247453205, 2746275624, 4011063775, 904135764, 876847374, - 366267234, 2541269205, 131376648, 1805948133, 3383589530, 2350119829, 2513170439, - 4096158499, 4229211520, 2992048272, 1338522080, 1187391335, 2898563453, 2163088451, - 1417971677, 2047421551, 902282791, 1143943232, 3568431811, 4059861993, 193362198, - 2509297125, 3968551582, 2175686117, 3568936881, 1853177468, 2134063169, 2919389416, - 1124914545, 1209806738, - }, - { - 1199972651, 1035834631, 3177798370, 860834162, 3741677748, 3780327829, 1693730265, - 1643429511, 559568669, 2758650294, 647308222, 3901603996, 1778653821, 3618523672, - 2154201067, 4261179460, 3285764480, 3334002738, 3215795953, 91368462, 1883994950, - 1506873376, 1527780962, 4046354597, 4081676034, 2389066602, 1574939945, 427845396, - 2714836263, 1259019491, 2493238133, 2584034689, 3151382431, 2171033919, 176883719, - 2031844862, 1272380790, 1298975901, 4087222847, 1524000054, 311436877, 3627785554, - 1889491722, 2938069193, 2771940687, 2756955968, 4289348777, 263514583, 887207028, - 3522902525, 2273246349, 835377715, 2897243319, 204645450, 1775911983, 639470242, - 2856296318, 3032942383, 2845501282, 1979082575, 202834023, 1876303820, 1434703409, - 4240524132, 848853780, 4188621628, 928095314, 876412914, 3446576392, 3235688990, - 4021419931, 2483628986, 3155781890, 399997246, 1642535200, 3872575068, 1577956550, - 3606228634, 609914462, 653194726, 4048067248, 2500767965, 1125167825, 3707628088, - 1819135158, 1875618971, 3865851141, 328215079, 1695889194, 2040280471, 3384684457, - 2540504961, 293050253, 525570078, 2655676443, 1392199429, 3370444585, 1937915855, - 2229636250, 247937142, 2534538765, 365841057, 2449431033, 2456532429, 101910696, - 1247069485, 1523958293, 2473285670, 473709728, 3026667113, 2071968844, 324025193, - 423064436, 3870800061, 3977393138, 3632553233, 352757977, 1584833348, 3173248650, - 1159857686, 1501841977, 1751860798, 617281070, 1958012761, 4031667102, 3232142321, - 3087428595, 2380824676, 1194087757, 1542961747, 4163350364, 1721646249, 1672791861, - 2900511710, 24973500, 1705444176, 713642505, 3017719513, 2090715200, 3521434070, - 37117223, 1948295454, 3055840561, 3476120789, 3994249388, 527899063, 4285770666, - 1075524023, 2594223535, 392943522, 171012646, 3515750082, 3414659054, 3501852926, - 1493283737, 2662104279, 2033464928, 90134967, 363058647, 3289266998, 2470752727, - 1199972651, 1035834631, 3177798370, 860834162, 1791097822, 3780327829, 1693730265, - 1643429511, 559568669, 3503319486, 647308222, 3901603996, 1778653821, 3618523672, - 4294594427, 4261179460, 3285764480, 3334002738, 3215795953, 212518363, 1883994950, - 1506873376, 1527780962, 4046354597, 2398655600, 2389066602, 1574939945, 427845396, - 2714836263, 2744363872, 2493238133, 2584034689, 3151382431, 2171033919, 2787053497, - 2031844862, 1272380790, 1298975901, 4087222847, 2342953154, 311436877, 3627785554, - 1889491722, 2938069193, 2026656505, 2756955968, 4289348777, 263514583, 887207028, - 2097276163, 2273246349, 835377715, 2897243319, 204645450, 4233399907, 639470242, - 2856296318, 3032942383, 2845501282, 28260330, 202834023, 1876303820, 1434703409, - 4240524132, 2455670466, 4188621628, 928095314, 876412914, 3446576392, 117581687, - 4021419931, 2483628986, 3155781890, 399997246, 4254101087, 3872575068, 1577956550, - 3606228634, 609914462, 4003279048, 4048067248, 2500767965, 1125167825, 3707628088, - 922020515, 1875618971, 3865851141, 328215079, 1695889194, 625773097, 3384684457, - 2540504961, 293050253, 525570078, 2592805114, 1392199429, 3370444585, 1937915855, - 2229636250, 3190958614, 2534538765, 365841057, 2449431033, 2456532429, 3778669305, - 1247069485, 1523958293, 2473285670, 473709728, 720895889, 2071968844, 324025193, - 423064436, 3870800061, 3535536111, 3632553233, 352757977, 1584833348, 3173248650, - 2649344603, 1501841977, 1751860798, 617281070, 1958012761, 778965559, 3232142321, - 3087428595, 2380824676, 1194087757, 3880222002, 4163350364, 1721646249, 1672791861, - 2900511710, 702936770, 1705444176, 713642505, 3017719513, 2090715200, 1477858694, - 37117223, 1948295454, 3055840561, 3476120789, 464173532, 527899063, 4285770666, - 1075524023, 2594223535, 2872629966, 171012646, 3515750082, 3414659054, 3501852926, - 1631555059, 2662104279, 2033464928, 90134967, 363058647, 4112991722, 2470752727, - 1199972651, 1035834631, 3177798370, 4152098951, 1791097822, 3780327829, 1693730265, - 1643429511, 153020604, 3503319486, 647308222, 3901603996, 1778653821, 221887019, - 4294594427, 4261179460, 3285764480, 3334002738, 3340918862, 212518363, 1883994950, - 1506873376, 1527780962, 430180116, 2398655600, 2389066602, 1574939945, 427845396, - 1683639957, 2744363872, 2493238133, 2584034689, 3151382431, 752704472, 2787053497, - 2031844862, 1272380790, 1298975901, 1528220628, 2342953154, 311436877, 3627785554, - 1889491722, 2576495467, 2026656505, 2756955968, 4289348777, 263514583, 3778019638, - 2097276163, 2273246349, 835377715, 2897243319, 1060067446, 4233399907, 639470242, - 2856296318, 3032942383, 2351047932, 28260330, 202834023, 1876303820, 1434703409, - 3094305336, 2455670466, 4188621628, 928095314, 876412914, 3785385583, 117581687, - 4021419931, 2483628986, 3155781890, 1867816730, 4254101087, 3872575068, 1577956550, - 3606228634, 3081878598, 4003279048, 4048067248, 2500767965, 1125167825, 928465955, - 922020515, 1875618971, 3865851141, 328215079, 173810260, 625773097, 3384684457, - 2540504961, 293050253, 2645143254, 2592805114, 1392199429, 3370444585, 1937915855, - 162781360, 3190958614, 2534538765, 365841057, 2449431033, 3105377832, 3778669305, - 1247069485, 1523958293, 2473285670, 800971948, 720895889, 2071968844, 324025193, - 423064436, 52577992, 3535536111, 3632553233, 352757977, 1584833348, 3305908059, - 2649344603, 1501841977, 1751860798, 617281070, 264880505, 778965559, 3232142321, - 3087428595, 2380824676, 1127761012, 3880222002, 4163350364, 1721646249, 1672791861, - 2368512339, 702936770, 1705444176, 713642505, 3017719513, 197200752, 1477858694, - 37117223, 1948295454, 3055840561, 1588372042, 464173532, 527899063, 4285770666, - 1075524023, 2124039914, 2872629966, 171012646, 3515750082, 3414659054, 818571456, - 1631555059, 2662104279, 2033464928, 90134967, 952712086, 4112991722, 2470752727, - 1199972651, 1035834631, 888975816, 4152098951, 1791097822, 3780327829, 1693730265, - 3406785510, 153020604, 3503319486, 647308222, 3901603996, 3753248472, 221887019, - 4294594427, 4261179460, 3285764480, 1861431346, 3340918862, 212518363, 1883994950, - 1506873376, 2695939612, 430180116, 2398655600, 2389066602, 1574939945, 2852159074, - 1683639957, 2744363872, 2493238133, 2584034689, 1952065633, 752704472, 2787053497, - 2031844862, 1272380790, 3530505866, 1528220628, 2342953154, 311436877, 3627785554, - 3410473245, 2576495467, 2026656505, 2756955968, 4289348777, 2856163034, 3778019638, - 2097276163, 2273246349, 835377715, 3127280755, 1060067446, 4233399907, 639470242, - 2856296318, 2615775011, 2351047932, 28260330, 202834023, 1876303820, 619308202, - 3094305336, 2455670466, 4188621628, 928095314, 3764894047, 3785385583, 117581687, - 4021419931, 2483628986, 3759839215, 1867816730, 4254101087, 3872575068, 1577956550, - 1687107439, 3081878598, 4003279048, 4048067248, 2500767965, 2804044146, 928465955, - 922020515, 1875618971, 3865851141, 2359176389, 173810260, 625773097, 3384684457, - 2540504961, 3665420733, 2645143254, 2592805114, 1392199429, 3370444585, 1604709429, - 162781360, 3190958614, 2534538765, 365841057, 3843585067, 3105377832, 3778669305, - 1247069485, 1523958293, 293374051, 800971948, 720895889, 2071968844, 324025193, - 3342361801, 52577992, 3535536111, 3632553233, 352757977, 1386594581, 3305908059, - 2649344603, 1501841977, 1751860798, 3160423601, 264880505, 778965559, 3232142321, - 3087428595, 3814775120, 1127761012, 3880222002, 4163350364, 1721646249, 3640773034, - 2368512339, 702936770, 1705444176, 713642505, 1717761787, 197200752, 1477858694, - 37117223, 1948295454, 896215772, 1588372042, 464173532, 527899063, 4285770666, - 3441409029, 2124039914, 2872629966, 171012646, 3515750082, 2216687886, 818571456, - 1631555059, 2662104279, 2033464928, 369438400, 329003658, 1503365029, 4215790910, - 3264377550, 733526983, 2935318632, 1792331479, 608347530, 392723097, 1330445854, - 3473004271, 1267636682, 2150566972, 2664910943, 2591861637, 409769584, 2943326880, - 3746302819, 3162268832, 1028663260, 3206607045, 832105292, 2119405275, 538318455, - 2981192295, 861775416, 609718403, 3531204230, 1904759571, 1262633751, 2375133081, - 460454984, 946700253, 3763898311, 1571175213, 3124410107, 2413420216, 2664177543, - 3241803820, 3968067371, 1234860999, 1130471500, 772727786, 247203117, 576455235, - 246297007, 2027348597, 764933887, 3812479771, 1825807084, 4072281412, 2156865781, - 1286484847, 1966749063, 2479269303, 423506843, 3070938758, 653091413, 2267423132, - 2004263526, 1374490719, 3871990628, 841138314, 1260317857, 3887432433, 4025147569, - 764233331, 1794763428, 3005903468, 877926770, 2466593927, 2971729561, 3203070565, - 4198500026, 815665759, 2434508139, 1840456368, 2279000427, 17077200, 3178380570, - 990304199, 3578008580, 1965763660, 1640352477, 750159594, 2047409402, 3576308245, - 544920564, 1730124869, 1194761386, 3280315505, 147334027, 2870674244, 2076860776, - 1100947675, 2482772161, 401966468, 1610650855, 193868446, 3808157106, 1509130117, - 1324484736, 3852893217, 1059179497, 4053543778, 2557844172, 3282312002, 682550058, - 4281899173, 137171998, 3239159214, 2258610918, 426724741, 3502660993, 135977383, - 429929363, 3984458137, 964026748, 2182019070, 3836562946, 515026869, 359030455, - 1301694917, 2300414803, 2364654981, 3804876710, 171119249, 2646785698, 4283509387, - 3628087763, 1748227044, 3037141234, 3000413256, 23007314, 3598880509, 4160517314, - 112205578, 1677675411, 734881643, 2830770338, 3470317145, 3306806569, 2635040943, - 2671367560, 3528996498, 3878886478, 3114253828, 2721384408, 3175226991, 1393767271, - 2651623266, 3767978376, 1269699398, 1100964192, 4169085845, 2086718107, 1286251099, - 764751784, 3006878591, - }, - { - 2565473087, 1149521056, 3529037691, 630435548, 73598765, 1467331930, 3988027050, - 2771962200, 91261543, 980989218, 2227515435, 236831608, 2872772569, 2330469327, - 1654035853, 2883791516, 4170143763, 126418114, 127789935, 2114249438, 2933346767, - 639483386, 1532399845, 2182422151, 741069317, 2376371063, 3398508789, 3828295651, - 3963199356, 4156483769, 4206759111, 1266176088, 3210273687, 432131993, 667709537, - 874477513, 2304714957, 629309008, 116453438, 3051811727, 3490241985, 3355968243, - 2304043871, 2724990029, 1095724699, 2408437363, 1433161037, 3245468546, 2494529842, - 4204170637, 1966342448, 3092333073, 1861880941, 3990012367, 3710334908, 2526395471, - 1884691351, 2145882162, 2561288457, 2253122309, 1154858044, 1643256991, 3172857504, - 1096492713, 2848827103, 799826424, 3094672168, 3535834360, 4213256737, 1131757994, - 520495112, 575315345, 3823364867, 2424349582, 3604795017, 310789314, 4207205257, - 553462404, 2918228443, 2568360580, 3863565851, 874197736, 3329267685, 1186352580, - 3928193054, 1780200631, 4088289456, 3323217870, 2758854947, 3111637417, 990374143, - 2080149357, 4047813631, 2019887940, 578660736, 2145680301, 2328411541, 1572704242, - 405739686, 1869350271, 2046317220, 4021497634, 1385163990, 1935250885, 1132987169, - 581690993, 3172043012, 628071512, 2851125739, 2735324847, 2847267504, 3408334906, - 3352976111, 706277272, 2971786942, 2811957324, 3578703606, 1126685543, 2671169997, - 31952251, 2802110464, 2391618856, 3031260674, 1165714541, 2411388800, 2825634835, - 101928462, 477629709, 4257022506, 3281706767, 2576087732, 736533968, 2543083137, - 3430523686, 3272172013, 3056925798, 341993500, 406782950, 1770032304, 125786076, - 1321359723, 2901696227, 1890958265, 3610842776, 1772227311, 1564088598, 914173231, - 3734092059, 1652333721, 2386645282, 329706426, 1022239203, 1832393502, 4064995802, - 3497852986, 1046436763, 366391010, 2237068647, 2887356463, 304718827, 3969799795, - 2565473087, 1149521056, 3529037691, 630435548, 3758124054, 1467331930, 3988027050, - 2771962200, 91261543, 836545831, 2227515435, 236831608, 2872772569, 2330469327, - 3439193753, 2883791516, 4170143763, 126418114, 127789935, 1648940583, 2933346767, - 639483386, 1532399845, 2182422151, 2470139222, 2376371063, 3398508789, 3828295651, - 3963199356, 2997263135, 4206759111, 1266176088, 3210273687, 432131993, 2416600665, - 874477513, 2304714957, 629309008, 116453438, 2586542760, 3490241985, 3355968243, - 2304043871, 2724990029, 452934545, 2408437363, 1433161037, 3245468546, 2494529842, - 2244403710, 1966342448, 3092333073, 1861880941, 3990012367, 2774994234, 2526395471, - 1884691351, 2145882162, 2561288457, 2303702146, 1154858044, 1643256991, 3172857504, - 1096492713, 130979316, 799826424, 3094672168, 3535834360, 4213256737, 935499492, - 520495112, 575315345, 3823364867, 2424349582, 2272973265, 310789314, 4207205257, - 553462404, 2918228443, 2613016888, 3863565851, 874197736, 3329267685, 1186352580, - 4106984978, 1780200631, 4088289456, 3323217870, 2758854947, 1559861146, 990374143, - 2080149357, 4047813631, 2019887940, 1133329900, 2145680301, 2328411541, 1572704242, - 405739686, 63633520, 2046317220, 4021497634, 1385163990, 1935250885, 1762959503, - 581690993, 3172043012, 628071512, 2851125739, 3726073981, 2847267504, 3408334906, - 3352976111, 706277272, 3817450114, 2811957324, 3578703606, 1126685543, 2671169997, - 2749086326, 2802110464, 2391618856, 3031260674, 1165714541, 2210258428, 2825634835, - 101928462, 477629709, 4257022506, 2679409844, 2576087732, 736533968, 2543083137, - 3430523686, 1122549807, 3056925798, 341993500, 406782950, 1770032304, 2617760292, - 1321359723, 2901696227, 1890958265, 3610842776, 2666109620, 1564088598, 914173231, - 3734092059, 1652333721, 3456779008, 329706426, 1022239203, 1832393502, 4064995802, - 4006865520, 1046436763, 366391010, 2237068647, 2887356463, 1479646555, 3969799795, - 2565473087, 1149521056, 3529037691, 2379195579, 3758124054, 1467331930, 3988027050, - 2771962200, 1796797949, 836545831, 2227515435, 236831608, 2872772569, 544017308, - 3439193753, 2883791516, 4170143763, 126418114, 3811390247, 1648940583, 2933346767, - 639483386, 1532399845, 4165970043, 2470139222, 2376371063, 3398508789, 3828295651, - 4066952157, 2997263135, 4206759111, 1266176088, 3210273687, 560560354, 2416600665, - 874477513, 2304714957, 629309008, 2010844440, 2586542760, 3490241985, 3355968243, - 2304043871, 855615381, 452934545, 2408437363, 1433161037, 3245468546, 3813880871, - 2244403710, 1966342448, 3092333073, 1861880941, 3334256651, 2774994234, 2526395471, - 1884691351, 2145882162, 3500193798, 2303702146, 1154858044, 1643256991, 3172857504, - 3480843206, 130979316, 799826424, 3094672168, 3535834360, 915442396, 935499492, - 520495112, 575315345, 3823364867, 2876158574, 2272973265, 310789314, 4207205257, - 553462404, 2184663001, 2613016888, 3863565851, 874197736, 3329267685, 3447734684, - 4106984978, 1780200631, 4088289456, 3323217870, 2748493470, 1559861146, 990374143, - 2080149357, 4047813631, 2728282767, 1133329900, 2145680301, 2328411541, 1572704242, - 3396987326, 63633520, 2046317220, 4021497634, 1385163990, 1582181054, 1762959503, - 581690993, 3172043012, 628071512, 2790170929, 3726073981, 2847267504, 3408334906, - 3352976111, 1211075015, 3817450114, 2811957324, 3578703606, 1126685543, 1946225412, - 2749086326, 2802110464, 2391618856, 3031260674, 453222948, 2210258428, 2825634835, - 101928462, 477629709, 410621659, 2679409844, 2576087732, 736533968, 2543083137, - 1101977922, 1122549807, 3056925798, 341993500, 406782950, 3057489804, 2617760292, - 1321359723, 2901696227, 1890958265, 4035843698, 2666109620, 1564088598, 914173231, - 3734092059, 908525903, 3456779008, 329706426, 1022239203, 1832393502, 4024857205, - 4006865520, 1046436763, 366391010, 2237068647, 1564059380, 1479646555, 3969799795, - 2565473087, 1149521056, 2808155917, 2379195579, 3758124054, 1467331930, 3988027050, - 810008243, 1796797949, 836545831, 2227515435, 236831608, 608273331, 544017308, - 3439193753, 2883791516, 4170143763, 3309288977, 3811390247, 1648940583, 2933346767, - 639483386, 1685761277, 4165970043, 2470139222, 2376371063, 3398508789, 4275493636, - 4066952157, 2997263135, 4206759111, 1266176088, 333592630, 560560354, 2416600665, - 874477513, 2304714957, 1438974661, 2010844440, 2586542760, 3490241985, 3355968243, - 2556368068, 855615381, 452934545, 2408437363, 1433161037, 4061232080, 3813880871, - 2244403710, 1966342448, 3092333073, 3412770364, 3334256651, 2774994234, 2526395471, - 1884691351, 1414627588, 3500193798, 2303702146, 1154858044, 1643256991, 2245958719, - 3480843206, 130979316, 799826424, 3094672168, 2214560871, 915442396, 935499492, - 520495112, 575315345, 3894763683, 2876158574, 2272973265, 310789314, 4207205257, - 3203740771, 2184663001, 2613016888, 3863565851, 874197736, 3371653768, 3447734684, - 4106984978, 1780200631, 4088289456, 378312754, 2748493470, 1559861146, 990374143, - 2080149357, 554816113, 2728282767, 1133329900, 2145680301, 2328411541, 4249979994, - 3396987326, 63633520, 2046317220, 4021497634, 4185731269, 1582181054, 1762959503, - 581690993, 3172043012, 3142596028, 2790170929, 3726073981, 2847267504, 3408334906, - 2556911142, 1211075015, 3817450114, 2811957324, 3578703606, 1480672978, 1946225412, - 2749086326, 2802110464, 2391618856, 3986823297, 453222948, 2210258428, 2825634835, - 101928462, 26373721, 410621659, 2679409844, 2576087732, 736533968, 888001208, - 1101977922, 1122549807, 3056925798, 341993500, 3243663736, 3057489804, 2617760292, - 1321359723, 2901696227, 1652018736, 4035843698, 2666109620, 1564088598, 914173231, - 1857869366, 908525903, 3456779008, 329706426, 1022239203, 2622178179, 4024857205, - 4006865520, 1046436763, 366391010, 3722250905, 2880126367, 4102186560, 1642831571, - 2222486636, 2572764729, 2046028516, 3507603612, 1703451134, 89818497, 1961701523, - 3704300476, 3563143931, 1609575644, 1599081111, 1047838539, 2779312926, 2065354728, - 956677756, 2073145924, 726634994, 119064196, 2046275296, 2105141632, 1023267361, - 1204528080, 623740611, 1419328884, 933734693, 2030900835, 2556538268, 1672647866, - 3125658368, 2221217376, 1097330641, 3214790630, 4276041578, 2397216525, 3916900004, - 330223096, 3915966823, 2646760259, 1724289351, 4015221358, 2338587000, 110922222, - 2314933196, 4026908935, 3272487985, 2685115305, 84271650, 731354215, 2358136447, - 1069348214, 2676811333, 1386266810, 1364512901, 4154449904, 3469122709, 54276972, - 560967905, 2363475740, 331250049, 3024074455, 186605617, 389582566, 1258386782, - 703909543, 3968367083, 1553533794, 3699576213, 1145761343, 921983735, 3573813763, - 1280477631, 3365842435, 1618458494, 2621328991, 1534006198, 2307669227, 4192335609, - 1338050203, 785284052, 4227164890, 2874735332, 3655821191, 2911684671, 3266454200, - 2679968625, 1191162601, 456550349, 1143881236, 3560103440, 2253437876, 3683014001, - 1087142366, 1462192975, 1076595768, 3227872159, 1842092988, 148227073, 3812110998, - 1317300278, 3068446245, 3376284001, 3164402992, 2730404635, 2848239579, 3008959791, - 2901849226, 1234485739, 869158554, 245101118, 1724974650, 3851803199, 922411232, - 3046280696, 3284392523, 3528264590, 2802364078, 381450957, 1741009694, 4222244451, - 102929888, 1668474417, 3881791214, 1429483134, 1938365051, 1023690708, 3333855520, - 3238705869, 2602245525, 3059586169, 720438965, 2120786297, 453980990, 1048501876, - 4060576583, 3537810796, 3892882814, 691572481, 3899584121, 1582529013, 3260326865, - 2358704826, 1607030801, 1035900449, 3442507859, 1406737127, 249758705, 1535363329, - 893329207, 51912312, 3440532856, 3736385218, 295452658, 2379709553, 1647382020, - 2363679860, 2998779887, - }, - { - 4209102573, 2387104994, 1221484586, 1726143957, 3263877318, 3362559187, 282442925, - 2418524976, 3196072648, 3174695999, 2072047145, 2985823503, 2132951745, 2298545297, - 2495977670, 1397656146, 2086257884, 3834366725, 3862532368, 3583329522, 1543996818, - 2192688115, 3081427696, 2656520743, 8772004, 2476324234, 3600148050, 1168683794, - 3219143568, 108768238, 1339513738, 447593731, 2742877256, 2488536667, 4189834432, - 808657962, 2422880287, 390864786, 3381554683, 760628048, 353395922, 3577556262, - 2482413928, 507756643, 839344953, 3505184848, 3945044582, 2414915836, 2313624497, - 1832728088, 2036999647, 1369090013, 3264575895, 1096327239, 3483440128, 3999302048, - 2761563885, 2882627112, 3126073009, 1749658776, 3152482044, 3040022505, 3249451214, - 2933713956, 2861715096, 1314806730, 932941454, 4276317539, 343449784, 1913556027, - 1493892363, 2539517630, 2046391233, 3046108187, 28742917, 4009448584, 530945117, - 3165875131, 1018448712, 110256395, 3550192264, 1279873435, 2276349621, 517650895, - 1957973772, 619869608, 4260458157, 2281748739, 2489253174, 2220997989, 3787481606, - 508630251, 3761850170, 3992979014, 2298047038, 3506428315, 1279341556, 3293496518, - 1313470495, 1021100687, 3113171268, 798494760, 2981622008, 4152623583, 576409629, - 2312811213, 992326282, 261645450, 1818084365, 3357150904, 144093, 1937589359, - 2016990596, 4273422066, 588267732, 3592151118, 3846596932, 1198111464, 944363907, - 1288613766, 1707163456, 4020906747, 1161127694, 2303844076, 2632591611, 3877442490, - 2453788473, 1725876694, 1193989740, 2650581453, 1937459187, 361099994, 3566745727, - 3658112707, 3612317412, 2684702277, 2880928862, 2044313931, 1866044828, 3528429465, - 130421713, 2658878825, 1566180833, 1572228417, 531947625, 3774861000, 1894712110, - 1319199233, 865634052, 2602102379, 3389730171, 3878969250, 107983959, 1601930856, - 2511728925, 2146946013, 497511195, 720616881, 699892123, 2404505137, 2656498433, - 4209102573, 2387104994, 1221484586, 1726143957, 1267363185, 3362559187, 282442925, - 2418524976, 3196072648, 2942944206, 2072047145, 2985823503, 2132951745, 2298545297, - 4079341490, 1397656146, 2086257884, 3834366725, 3862532368, 3991197972, 1543996818, - 2192688115, 3081427696, 2656520743, 825853576, 2476324234, 3600148050, 1168683794, - 3219143568, 528751585, 1339513738, 447593731, 2742877256, 2488536667, 4025362081, - 808657962, 2422880287, 390864786, 3381554683, 2682225618, 353395922, 3577556262, - 2482413928, 507756643, 3979211244, 3505184848, 3945044582, 2414915836, 2313624497, - 1841224078, 2036999647, 1369090013, 3264575895, 1096327239, 607843308, 3999302048, - 2761563885, 2882627112, 3126073009, 1241524975, 3152482044, 3040022505, 3249451214, - 2933713956, 420486142, 1314806730, 932941454, 4276317539, 343449784, 2231505736, - 1493892363, 2539517630, 2046391233, 3046108187, 2351652097, 4009448584, 530945117, - 3165875131, 1018448712, 1683392491, 3550192264, 1279873435, 2276349621, 517650895, - 4036312766, 619869608, 4260458157, 2281748739, 2489253174, 1686790154, 3787481606, - 508630251, 3761850170, 3992979014, 1745325013, 3506428315, 1279341556, 3293496518, - 1313470495, 3066312306, 3113171268, 798494760, 2981622008, 4152623583, 3871822467, - 2312811213, 992326282, 261645450, 1818084365, 3681154045, 144093, 1937589359, - 2016990596, 4273422066, 2361898985, 3592151118, 3846596932, 1198111464, 944363907, - 2866279694, 1707163456, 4020906747, 1161127694, 2303844076, 3044280908, 3877442490, - 2453788473, 1725876694, 1193989740, 2049617934, 1937459187, 361099994, 3566745727, - 3658112707, 934740227, 2684702277, 2880928862, 2044313931, 1866044828, 1814569183, - 130421713, 2658878825, 1566180833, 1572228417, 1784679035, 3774861000, 1894712110, - 1319199233, 865634052, 283642947, 3389730171, 3878969250, 107983959, 1601930856, - 3698217362, 2146946013, 497511195, 720616881, 699892123, 2117385156, 2656498433, - 4209102573, 2387104994, 1221484586, 3495886368, 1267363185, 3362559187, 282442925, - 2418524976, 3489510655, 2942944206, 2072047145, 2985823503, 2132951745, 885541635, - 4079341490, 1397656146, 2086257884, 3834366725, 1049969755, 3991197972, 1543996818, - 2192688115, 3081427696, 2141948440, 825853576, 2476324234, 3600148050, 1168683794, - 5160254, 528751585, 1339513738, 447593731, 2742877256, 3033397497, 4025362081, - 808657962, 2422880287, 390864786, 3191593886, 2682225618, 353395922, 3577556262, - 2482413928, 1185107868, 3979211244, 3505184848, 3945044582, 2414915836, 3030493909, - 1841224078, 2036999647, 1369090013, 3264575895, 3054343366, 607843308, 3999302048, - 2761563885, 2882627112, 3912854189, 1241524975, 3152482044, 3040022505, 3249451214, - 55140065, 420486142, 1314806730, 932941454, 4276317539, 1055315026, 2231505736, - 1493892363, 2539517630, 2046391233, 4174985470, 2351652097, 4009448584, 530945117, - 3165875131, 2168411768, 1683392491, 3550192264, 1279873435, 2276349621, 1875092822, - 4036312766, 619869608, 4260458157, 2281748739, 98823023, 1686790154, 3787481606, - 508630251, 3761850170, 2636025017, 1745325013, 3506428315, 1279341556, 3293496518, - 978338993, 3066312306, 3113171268, 798494760, 2981622008, 2712384846, 3871822467, - 2312811213, 992326282, 261645450, 66982935, 3681154045, 144093, 1937589359, - 2016990596, 3390191329, 2361898985, 3592151118, 3846596932, 1198111464, 1857959320, - 2866279694, 1707163456, 4020906747, 1161127694, 913091437, 3044280908, 3877442490, - 2453788473, 1725876694, 4254455215, 2049617934, 1937459187, 361099994, 3566745727, - 2914687409, 934740227, 2684702277, 2880928862, 2044313931, 1515195925, 1814569183, - 130421713, 2658878825, 1566180833, 2753417020, 1784679035, 3774861000, 1894712110, - 1319199233, 287161774, 283642947, 3389730171, 3878969250, 107983959, 3057929912, - 3698217362, 2146946013, 497511195, 720616881, 3570251850, 2117385156, 2656498433, - 4209102573, 2387104994, 2940868252, 3495886368, 1267363185, 3362559187, 282442925, - 2510419746, 3489510655, 2942944206, 2072047145, 2985823503, 978430777, 885541635, - 4079341490, 1397656146, 2086257884, 134380865, 1049969755, 3991197972, 1543996818, - 2192688115, 1205081471, 2141948440, 825853576, 2476324234, 3600148050, 228461601, - 5160254, 528751585, 1339513738, 447593731, 2852356745, 3033397497, 4025362081, - 808657962, 2422880287, 3287655095, 3191593886, 2682225618, 353395922, 3577556262, - 2542841784, 1185107868, 3979211244, 3505184848, 3945044582, 2905156498, 3030493909, - 1841224078, 2036999647, 1369090013, 4246605417, 3054343366, 607843308, 3999302048, - 2761563885, 3611911899, 3912854189, 1241524975, 3152482044, 3040022505, 3215633820, - 55140065, 420486142, 1314806730, 932941454, 2708752494, 1055315026, 2231505736, - 1493892363, 2539517630, 962728637, 4174985470, 2351652097, 4009448584, 530945117, - 3370859357, 2168411768, 1683392491, 3550192264, 1279873435, 3028448904, 1875092822, - 4036312766, 619869608, 4260458157, 199178828, 98823023, 1686790154, 3787481606, - 508630251, 4205010983, 2636025017, 1745325013, 3506428315, 1279341556, 683127445, - 978338993, 3066312306, 3113171268, 798494760, 2823693013, 2712384846, 3871822467, - 2312811213, 992326282, 3701928286, 66982935, 3681154045, 144093, 1937589359, - 1117717039, 3390191329, 2361898985, 3592151118, 3846596932, 1072660054, 1857959320, - 2866279694, 1707163456, 4020906747, 2503116219, 913091437, 3044280908, 3877442490, - 2453788473, 1815274499, 4254455215, 2049617934, 1937459187, 361099994, 3771108073, - 2914687409, 934740227, 2684702277, 2880928862, 3591322975, 1515195925, 1814569183, - 130421713, 2658878825, 354587729, 2753417020, 1784679035, 3774861000, 1894712110, - 1799044969, 287161774, 283642947, 3389730171, 3878969250, 1229815186, 3057929912, - 3698217362, 2146946013, 497511195, 3121882901, 426537369, 3852284416, 4050544256, - 3148944089, 878474231, 1369575859, 2206199765, 870626886, 494668165, 613011290, - 3246772867, 1040178461, 2396959353, 2105449571, 456758967, 4134137960, 3525051481, - 3633445497, 2895048060, 2008411846, 2194012253, 2326112129, 2956901044, 2297039362, - 3400824024, 42139718, 4212208866, 3874761488, 2361955811, 1890446075, 864533345, - 474524842, 2283847731, 283971243, 3607219686, 280870706, 4188549522, 659660119, - 2460943922, 4252134362, 922033031, 3615474721, 1691563300, 3002653770, 2414043617, - 2251931324, 752654714, 4188343161, 305594960, 1320443323, 797027061, 2347530104, - 3608843538, 2717312892, 1841295453, 1574467161, 823626340, 2244853583, 2648217758, - 141742826, 1605436472, 745763543, 3275460028, 3166960370, 2655678693, 3964037210, - 945054703, 998173049, 1014527437, 3424443612, 281835352, 826817508, 260462513, - 2849967970, 3447294061, 3670173947, 2430650055, 4134905457, 3798172627, 2156572681, - 2600148034, 2773013892, 3290397106, 1740507705, 3450254627, 3613087060, 440045928, - 1230555006, 980805434, 2107958250, 526555374, 3150741277, 4283672024, 193019043, - 786035243, 3002832578, 3938336183, 4209865002, 1005950967, 3533346582, 3196886974, - 83962845, 1882902787, 3595687446, 2927597311, 2728550762, 2750900392, 1474254316, - 1509832112, 1763262792, 2706181276, 538294991, 353565565, 18133995, 1719731406, - 3311085516, 2018821960, 300367686, 2628312935, 1151449661, 2178805970, 3288321196, - 535051857, 1623270973, 2761151808, 2701048972, 317681607, 2281427601, 719748170, - 351452298, 2191958596, 4000232015, 335837771, 4158081521, 3779404077, 1998444133, - 3849605095, 1532231791, 2930266419, 4203951289, 748423654, 1993082867, 451159852, - 488781053, 2438982775, 2222815270, 543209242, 1241562465, 2868868009, 4201052877, - 2438841764, 2151708682, 2426958921, 1520654642, 1990098337, 1070792755, 2308394635, - 1442389785, 705615044, - }, - { - 973368008, 1221885324, 2086331970, 2323744198, 280145759, 1795442656, 2984366093, - 3532172763, 323888669, 851950179, 4198638255, 899943985, 4087912561, 2935341503, - 1443752852, 3991058999, 3547259355, 35779889, 1076308344, 4075444807, 186174448, - 3542284780, 660388677, 2777400132, 1092226205, 2418702276, 1307933032, 1940510003, - 1932005362, 4016036211, 387339882, 2969593895, 3453134349, 1382709098, 1795814140, - 1588159469, 1216733801, 2227378121, 2063027627, 582454582, 3364657275, 3466973302, - 484564303, 1489261596, 2270291560, 2008178784, 2284268924, 2229317366, 644797709, - 1213921542, 99331403, 3027640949, 1137722852, 2991506109, 1432805987, 931795812, - 1075567424, 28963219, 1462245461, 3781444706, 521233400, 1891915904, 3774338085, - 1635359313, 2356111795, 4121073768, 1045110727, 2822507066, 1087914587, 3744509525, - 911370656, 181884066, 1944539735, 290356444, 3598887471, 4236934380, 3224468239, - 457546246, 4119337570, 37700432, 655783844, 1423101410, 1693002969, 3287768267, - 928748421, 4074128009, 3081088543, 2882833790, 3180154875, 1094657682, 2388253717, - 4173455215, 794709427, 3363292346, 67786868, 3786597763, 380587236, 2345941620, - 560232318, 2137123833, 619747082, 1050293267, 2537845069, 1407302835, 433399526, - 1083185007, 1893842085, 3711748584, 4225838280, 3863317129, 2043467942, 2799650657, - 3590486611, 1231938950, 215905995, 155811669, 806806587, 2732631168, 1621659281, - 632403616, 401165422, 2661074778, 4156963191, 3691812937, 3767271627, 2834948318, - 2877210497, 2420260153, 733172233, 1771708940, 3102718549, 2468707423, 1857088312, - 3176535032, 1908570295, 3966666208, 605079895, 2982506620, 3721694730, 1640691570, - 3764975545, 3257514114, 1826578604, 1358557411, 4049610348, 615820785, 3355718142, - 1734641780, 2958744617, 274522187, 3198436002, 4077346785, 2890101344, 4012464346, - 1288365365, 96583076, 2656389382, 1858181040, 2717010340, 2032153178, 349324012, - 973368008, 1221885324, 2086331970, 2323744198, 253685576, 1795442656, 2984366093, - 3532172763, 323888669, 248935329, 4198638255, 899943985, 4087912561, 2935341503, - 3213394756, 3991058999, 3547259355, 35779889, 1076308344, 1987715385, 186174448, - 3542284780, 660388677, 2777400132, 2071022105, 2418702276, 1307933032, 1940510003, - 1932005362, 144370664, 387339882, 2969593895, 3453134349, 1382709098, 2394736611, - 1588159469, 1216733801, 2227378121, 2063027627, 4064263898, 3364657275, 3466973302, - 484564303, 1489261596, 3405101812, 2008178784, 2284268924, 2229317366, 644797709, - 2560273821, 99331403, 3027640949, 1137722852, 2991506109, 1446442417, 931795812, - 1075567424, 28963219, 1462245461, 1201513613, 521233400, 1891915904, 3774338085, - 1635359313, 2815447944, 4121073768, 1045110727, 2822507066, 1087914587, 2485035329, - 911370656, 181884066, 1944539735, 290356444, 2078819341, 4236934380, 3224468239, - 457546246, 4119337570, 2666895496, 655783844, 1423101410, 1693002969, 3287768267, - 3595439673, 4074128009, 3081088543, 2882833790, 3180154875, 872453917, 2388253717, - 4173455215, 794709427, 3363292346, 4188764388, 3786597763, 380587236, 2345941620, - 560232318, 625538006, 619747082, 1050293267, 2537845069, 1407302835, 2128289331, - 1083185007, 1893842085, 3711748584, 4225838280, 2486133065, 2043467942, 2799650657, - 3590486611, 1231938950, 928582681, 155811669, 806806587, 2732631168, 1621659281, - 1163969880, 401165422, 2661074778, 4156963191, 3691812937, 2322579561, 2834948318, - 2877210497, 2420260153, 733172233, 170239236, 3102718549, 2468707423, 1857088312, - 3176535032, 3868693408, 3966666208, 605079895, 2982506620, 3721694730, 2066859537, - 3764975545, 3257514114, 1826578604, 1358557411, 2964604045, 615820785, 3355718142, - 1734641780, 2958744617, 4091225681, 3198436002, 4077346785, 2890101344, 4012464346, - 2612861218, 96583076, 2656389382, 1858181040, 2717010340, 3639170895, 349324012, - 973368008, 1221885324, 2086331970, 2258432445, 253685576, 1795442656, 2984366093, - 3532172763, 3831166882, 248935329, 4198638255, 899943985, 4087912561, 715173523, - 3213394756, 3991058999, 3547259355, 35779889, 2393072396, 1987715385, 186174448, - 3542284780, 660388677, 3731857267, 2071022105, 2418702276, 1307933032, 1940510003, - 4262274779, 144370664, 387339882, 2969593895, 3453134349, 1923698215, 2394736611, - 1588159469, 1216733801, 2227378121, 2907069566, 4064263898, 3364657275, 3466973302, - 484564303, 2234542580, 3405101812, 2008178784, 2284268924, 2229317366, 1349323372, - 2560273821, 99331403, 3027640949, 1137722852, 4200786664, 1446442417, 931795812, - 1075567424, 28963219, 1659632304, 1201513613, 521233400, 1891915904, 3774338085, - 763590809, 2815447944, 4121073768, 1045110727, 2822507066, 4131040734, 2485035329, - 911370656, 181884066, 1944539735, 4104473807, 2078819341, 4236934380, 3224468239, - 457546246, 1241850776, 2666895496, 655783844, 1423101410, 1693002969, 2025898966, - 3595439673, 4074128009, 3081088543, 2882833790, 218474476, 872453917, 2388253717, - 4173455215, 794709427, 250328312, 4188764388, 3786597763, 380587236, 2345941620, - 1937652040, 625538006, 619747082, 1050293267, 2537845069, 1140055765, 2128289331, - 1083185007, 1893842085, 3711748584, 2298055548, 2486133065, 2043467942, 2799650657, - 3590486611, 1235949580, 928582681, 155811669, 806806587, 2732631168, 4046198728, - 1163969880, 401165422, 2661074778, 4156963191, 2003518762, 2322579561, 2834948318, - 2877210497, 2420260153, 326741418, 170239236, 3102718549, 2468707423, 1857088312, - 3936056808, 3868693408, 3966666208, 605079895, 2982506620, 2354705582, 2066859537, - 3764975545, 3257514114, 1826578604, 3017501686, 2964604045, 615820785, 3355718142, - 1734641780, 1681548103, 4091225681, 3198436002, 4077346785, 2890101344, 416470693, - 2612861218, 96583076, 2656389382, 1858181040, 3104217288, 3639170895, 349324012, - 973368008, 1221885324, 601524567, 2258432445, 253685576, 1795442656, 2984366093, - 1875491903, 3831166882, 248935329, 4198638255, 899943985, 2182697927, 715173523, - 3213394756, 3991058999, 3547259355, 1472237612, 2393072396, 1987715385, 186174448, - 3542284780, 2160848139, 3731857267, 2071022105, 2418702276, 1307933032, 3815354311, - 4262274779, 144370664, 387339882, 2969593895, 4240850623, 1923698215, 2394736611, - 1588159469, 1216733801, 322523795, 2907069566, 4064263898, 3364657275, 3466973302, - 2920715858, 2234542580, 3405101812, 2008178784, 2284268924, 1936025139, 1349323372, - 2560273821, 99331403, 3027640949, 859541953, 4200786664, 1446442417, 931795812, - 1075567424, 1876635772, 1659632304, 1201513613, 521233400, 1891915904, 3949233865, - 763590809, 2815447944, 4121073768, 1045110727, 2522258582, 4131040734, 2485035329, - 911370656, 181884066, 2467886009, 4104473807, 2078819341, 4236934380, 3224468239, - 603014155, 1241850776, 2666895496, 655783844, 1423101410, 673119756, 2025898966, - 3595439673, 4074128009, 3081088543, 4100445818, 218474476, 872453917, 2388253717, - 4173455215, 2395519424, 250328312, 4188764388, 3786597763, 380587236, 4292608797, - 1937652040, 625538006, 619747082, 1050293267, 2989616803, 1140055765, 2128289331, - 1083185007, 1893842085, 3313934002, 2298055548, 2486133065, 2043467942, 2799650657, - 1804808801, 1235949580, 928582681, 155811669, 806806587, 2864892828, 4046198728, - 1163969880, 401165422, 2661074778, 2946769376, 2003518762, 2322579561, 2834948318, - 2877210497, 2647485275, 326741418, 170239236, 3102718549, 2468707423, 2520336801, - 3936056808, 3868693408, 3966666208, 605079895, 2949706551, 2354705582, 2066859537, - 3764975545, 3257514114, 495003693, 3017501686, 2964604045, 615820785, 3355718142, - 3799230297, 1681548103, 4091225681, 3198436002, 4077346785, 258363842, 416470693, - 2612861218, 96583076, 2656389382, 2198085634, 1607235362, 694172175, 4194347563, - 2665732891, 3419430286, 597070176, 2749480905, 3937535348, 3639873850, 2050067843, - 4045290683, 2964298196, 3631595287, 1409808193, 121765438, 2129412744, 2497437101, - 2664102876, 1773441464, 1708052456, 2923764322, 3350385352, 1592204280, 1118221370, - 3416016313, 116121364, 1179473397, 1497519022, 902569114, 3840281863, 2783662797, - 1712084322, 1982884601, 3625797892, 4222938993, 3231134134, 3046745397, 446484563, - 1133869192, 2622178726, 3881085862, 4012894217, 391734322, 2089696890, 1304197030, - 2663978386, 1685998658, 4238620912, 448351665, 2724524045, 1038754164, 413586547, - 3107681687, 1454664365, 3353731192, 128440996, 565817989, 127978294, 1043863326, - 1747369107, 2772246481, 172569313, 2740699699, 3417082503, 2103702630, 2139566116, - 1378864710, 2444170529, 3234220221, 3974870858, 1965162347, 956763257, 3932467825, - 1077337271, 4084837149, 3699147465, 1882164226, 236113740, 3116302858, 2730529598, - 3449804672, 4002370655, 2011573068, 1551746089, 3917496971, 2852603, 1682999535, - 2764817908, 2489487254, 261936311, 3122421452, 1199382345, 2617247590, 1909026938, - 3156073069, 1492533764, 97847107, 1260892586, 187242945, 1286471861, 1763024967, - 127723419, 210606273, 228546401, 3249879676, 482069954, 383075106, 3263105259, - 2242748676, 1105681409, 4033144425, 4017983282, 1670425353, 4040882785, 1780687273, - 1405678015, 3076115981, 2713472488, 286336494, 3664225263, 502759060, 777620620, - 635590826, 132236203, 1866406173, 1235046453, 2859554298, 121814656, 944976320, - 1946494170, 2548097575, 415384946, 3934685646, 1507032178, 1383654007, 2906269630, - 566005756, 3118733139, 2969075870, 1834096359, 3263358416, 1322979710, 904583023, - 3582075094, 2298898632, 2491891941, 660351763, 2548592542, 3009200751, 2116595110, - 2623212287, 4167133624, 196759529, 3038645579, 1769704552, 1713233322, 2127497999, - 3849458221, 3769872265, - }, - { - 51082211, 3347503176, 3221768777, 2986163981, 3937460013, 1716372908, 4132024211, - 3035957293, 1791036224, 4214815056, 1645540011, 2629793790, 3185505897, 1855718954, - 495459467, 1728339314, 2318396341, 3396602050, 3273624616, 2625973148, 1762905939, - 1706358553, 1088424264, 2764318930, 1026127380, 1128004134, 2724553694, 41981087, - 2304966004, 1342745986, 3425554050, 3537673465, 752263676, 280905885, 929399589, - 4090689526, 2141254732, 1469491656, 2593100469, 1096008340, 3316809312, 1698245188, - 1656427920, 3081873338, 1750515301, 3850483440, 4081834419, 1225164947, 1244139942, - 1972663124, 116832506, 3097397897, 961156503, 1899056660, 1659173175, 1890464921, - 1891872926, 191259956, 1735304734, 4246751855, 4285601625, 1495519933, 3398829761, - 997567482, 2277782972, 1286922996, 3120709698, 15569196, 191501283, 3916619528, - 1552075789, 4259725643, 2837166910, 2231584792, 1912204495, 2597304083, 4147585653, - 1021482843, 2003417305, 186794491, 3608172979, 1991038123, 97515853, 34341352, - 4163491231, 1046079304, 4105813389, 3589099183, 3970004064, 3899560802, 4148331147, - 2267137817, 85024486, 3019925981, 3069231953, 1563669137, 314080592, 2943111861, - 2838243982, 726216848, 2621853102, 2355885175, 3359668856, 1111296541, 2330283124, - 3626558972, 4290084148, 1632078, 1047440803, 1350377197, 2490578842, 2366345698, - 1628128899, 860259543, 1937956234, 2833820527, 329818923, 648489148, 1791961202, - 1652322723, 1513419073, 149629345, 2468961221, 3711837973, 2377333831, 1434755773, - 3808719305, 2513270108, 2701064683, 3097011724, 303393137, 1346302239, 1852307302, - 850106025, 2473124483, 2853497268, 3786573704, 2604101162, 1446573486, 506925220, - 3138967488, 2973528682, 2752811123, 2890321579, 1037196362, 1335670403, 1560253777, - 1437495434, 2700525242, 4259933972, 3870707795, 4141538580, 3375331039, 3081538601, - 3129978494, 3689191993, 1933431212, 2196145886, 87814045, 878611347, 812542698, - 51082211, 3347503176, 3221768777, 2986163981, 277257023, 1716372908, 4132024211, - 3035957293, 1791036224, 2713916211, 1645540011, 2629793790, 3185505897, 1855718954, - 3800150234, 1728339314, 2318396341, 3396602050, 3273624616, 2530282967, 1762905939, - 1706358553, 1088424264, 2764318930, 4224669506, 1128004134, 2724553694, 41981087, - 2304966004, 484289311, 3425554050, 3537673465, 752263676, 280905885, 657028134, - 4090689526, 2141254732, 1469491656, 2593100469, 2089385540, 3316809312, 1698245188, - 1656427920, 3081873338, 2750354264, 3850483440, 4081834419, 1225164947, 1244139942, - 3432605739, 116832506, 3097397897, 961156503, 1899056660, 2234120716, 1890464921, - 1891872926, 191259956, 1735304734, 125359575, 4285601625, 1495519933, 3398829761, - 997567482, 4034254942, 1286922996, 3120709698, 15569196, 191501283, 2090684174, - 1552075789, 4259725643, 2837166910, 2231584792, 3412758413, 2597304083, 4147585653, - 1021482843, 2003417305, 2464533361, 3608172979, 1991038123, 97515853, 34341352, - 2634732952, 1046079304, 4105813389, 3589099183, 3970004064, 4263828421, 4148331147, - 2267137817, 85024486, 3019925981, 3229477751, 1563669137, 314080592, 2943111861, - 2838243982, 1274664774, 2621853102, 2355885175, 3359668856, 1111296541, 1615234696, - 3626558972, 4290084148, 1632078, 1047440803, 623898652, 2490578842, 2366345698, - 1628128899, 860259543, 2097114662, 2833820527, 329818923, 648489148, 1791961202, - 1730000077, 1513419073, 149629345, 2468961221, 3711837973, 3255238414, 1434755773, - 3808719305, 2513270108, 2701064683, 1635042488, 303393137, 1346302239, 1852307302, - 850106025, 3523245944, 2853497268, 3786573704, 2604101162, 1446573486, 1011238489, - 3138967488, 2973528682, 2752811123, 2890321579, 1044586909, 1335670403, 1560253777, - 1437495434, 2700525242, 608940900, 3870707795, 4141538580, 3375331039, 3081538601, - 728626935, 3689191993, 1933431212, 2196145886, 87814045, 646732047, 812542698, - 51082211, 3347503176, 3221768777, 1783601443, 277257023, 1716372908, 4132024211, - 3035957293, 3363442238, 2713916211, 1645540011, 2629793790, 3185505897, 2066587565, - 3800150234, 1728339314, 2318396341, 3396602050, 2279941522, 2530282967, 1762905939, - 1706358553, 1088424264, 2989326347, 4224669506, 1128004134, 2724553694, 41981087, - 2348931916, 484289311, 3425554050, 3537673465, 752263676, 3169550883, 657028134, - 4090689526, 2141254732, 1469491656, 1152943917, 2089385540, 3316809312, 1698245188, - 1656427920, 1808689833, 2750354264, 3850483440, 4081834419, 1225164947, 2422106046, - 3432605739, 116832506, 3097397897, 961156503, 1581804167, 2234120716, 1890464921, - 1891872926, 191259956, 4113708001, 125359575, 4285601625, 1495519933, 3398829761, - 3473435310, 4034254942, 1286922996, 3120709698, 15569196, 3122200488, 2090684174, - 1552075789, 4259725643, 2837166910, 2838170407, 3412758413, 2597304083, 4147585653, - 1021482843, 954000150, 2464533361, 3608172979, 1991038123, 97515853, 3832321348, - 2634732952, 1046079304, 4105813389, 3589099183, 1288675572, 4263828421, 4148331147, - 2267137817, 85024486, 1080403742, 3229477751, 1563669137, 314080592, 2943111861, - 3357655593, 1274664774, 2621853102, 2355885175, 3359668856, 3722440291, 1615234696, - 3626558972, 4290084148, 1632078, 4263556325, 623898652, 2490578842, 2366345698, - 1628128899, 3623136669, 2097114662, 2833820527, 329818923, 648489148, 592747007, - 1730000077, 1513419073, 149629345, 2468961221, 3766709284, 3255238414, 1434755773, - 3808719305, 2513270108, 1958651003, 1635042488, 303393137, 1346302239, 1852307302, - 579487408, 3523245944, 2853497268, 3786573704, 2604101162, 4183724981, 1011238489, - 3138967488, 2973528682, 2752811123, 3074709397, 1044586909, 1335670403, 1560253777, - 1437495434, 1237099522, 608940900, 3870707795, 4141538580, 3375331039, 2032507604, - 728626935, 3689191993, 1933431212, 2196145886, 4008131891, 646732047, 812542698, - 51082211, 3347503176, 3622107037, 1783601443, 277257023, 1716372908, 4132024211, - 1264285659, 3363442238, 2713916211, 1645540011, 2629793790, 2179309595, 2066587565, - 3800150234, 1728339314, 2318396341, 580990822, 2279941522, 2530282967, 1762905939, - 1706358553, 2826056883, 2989326347, 4224669506, 1128004134, 2724553694, 1486392636, - 2348931916, 484289311, 3425554050, 3537673465, 2497657189, 3169550883, 657028134, - 4090689526, 2141254732, 1019244016, 1152943917, 2089385540, 3316809312, 1698245188, - 2921739456, 1808689833, 2750354264, 3850483440, 4081834419, 108425527, 2422106046, - 3432605739, 116832506, 3097397897, 3902994002, 1581804167, 2234120716, 1890464921, - 1891872926, 3428861050, 4113708001, 125359575, 4285601625, 1495519933, 3350053832, - 3473435310, 4034254942, 1286922996, 3120709698, 2906201347, 3122200488, 2090684174, - 1552075789, 4259725643, 1965598685, 2838170407, 3412758413, 2597304083, 4147585653, - 3335631208, 954000150, 2464533361, 3608172979, 1991038123, 3788034599, 3832321348, - 2634732952, 1046079304, 4105813389, 2362460804, 1288675572, 4263828421, 4148331147, - 2267137817, 2767331798, 1080403742, 3229477751, 1563669137, 314080592, 1737897403, - 3357655593, 1274664774, 2621853102, 2355885175, 57997639, 3722440291, 1615234696, - 3626558972, 4290084148, 3703113369, 4263556325, 623898652, 2490578842, 2366345698, - 73788443, 3623136669, 2097114662, 2833820527, 329818923, 253042650, 592747007, - 1730000077, 1513419073, 149629345, 4248302934, 3766709284, 3255238414, 1434755773, - 3808719305, 3698431827, 1958651003, 1635042488, 303393137, 1346302239, 3872433842, - 579487408, 3523245944, 2853497268, 3786573704, 3459185849, 4183724981, 1011238489, - 3138967488, 2973528682, 2605373899, 3074709397, 1044586909, 1335670403, 1560253777, - 4069724875, 1237099522, 608940900, 3870707795, 4141538580, 2550307954, 2032507604, - 728626935, 3689191993, 1933431212, 1177640824, 4168589688, 1361487780, 2649612520, - 2181448948, 2176288560, 947907377, 3772487849, 2002599877, 3353450532, 2013145251, - 2357912348, 2316997609, 2355480213, 255142205, 751273749, 4049362748, 1434168014, - 2069693747, 6910933, 1352778547, 2413649875, 4238683558, 484497407, 522555106, - 1848417180, 3001805499, 2264939603, 3606143565, 1782482647, 2955918436, 3471474379, - 2814133839, 3779337475, 3522102195, 1551792178, 3742001759, 3433504551, 472918932, - 3835854229, 4259163014, 4103952359, 1989474190, 1792448078, 1517735224, 1958036884, - 2277922531, 2856192348, 4294188732, 2674247971, 643649427, 3847742408, 1512435795, - 4236693554, 1749045838, 397093640, 2940663643, 4156440725, 188785143, 3894740830, - 897618321, 3333498692, 1623924612, 4111607062, 2242558573, 1565861815, 1177957654, - 4129621176, 2232443247, 3216995984, 3313294700, 791442469, 1782204490, 2452634246, - 1288014576, 1347365377, 112254281, 2044140398, 2479591984, 3293701920, 1062335151, - 1397230369, 2460086085, 2412349474, 1252633202, 3704541545, 2132499200, 2202058121, - 1981543691, 2683673516, 1198109770, 4279157703, 2224504258, 2188868731, 769314834, - 601313429, 3595357440, 561383123, 3444949507, 2127327734, 2865252582, 4181002098, - 2408426518, 309361635, 2377703815, 1109219406, 406287309, 1750179098, 3619129839, - 937928728, 884423945, 928407281, 4173634172, 1492070114, 2706943441, 1365883971, - 3076484301, 744370087, 4004118884, 2199449568, 879458863, 3197725005, 3590586547, - 59693002, 3614114662, 1499386564, 2914582708, 3751842429, 947438603, 1986129491, - 3747112289, 833777768, 2231649410, 1841607849, 3526253103, 3797105813, 1291185911, - 3188408549, 3311027691, 3983681758, 2246511800, 1271525377, 2996265908, 2396071405, - 902427181, 164636454, 2459631341, 635349368, 463309029, 1409367654, 849052250, - 3221786769, 2310288531, 2919204855, 1263130532, 1215722704, 3497322658, 2840687222, - 2185004161, 1507335864, - }, - { - 2052557448, 2879065999, 22933757, 2160014758, 3736092460, 3556641619, 1350613766, - 2107757927, 309323868, 3452852627, 3424626316, 545651740, 1935764720, 2349926457, - 3546577033, 862046434, 167198649, 338290297, 1145807303, 1571276102, 883549156, - 907871968, 638566313, 678764227, 3795356864, 3306095271, 1452688488, 621126888, - 1838613968, 2054107827, 1785040579, 454879400, 1952849106, 1907701866, 1639495252, - 1623968604, 2997422000, 2633878652, 1036670775, 2191465943, 3053533585, 1654709920, - 1950620393, 4177745509, 1742007818, 2888573892, 2825965566, 1399790365, 1307674482, - 310692416, 384880529, 3497622676, 1024664651, 1541456182, 1904670217, 1008618602, - 2816673160, 684112698, 3332034744, 3646613828, 962627614, 3072103948, 3713153075, - 476323310, 3243124597, 126319837, 2155412848, 228580793, 2142696490, 3442722759, - 2689599232, 426191419, 3157759186, 100239709, 494534049, 1259677734, 2889209278, - 1754641396, 1057778427, 2133253617, 1298500018, 3340348062, 3967049659, 2927469144, - 1503854147, 2271956463, 3004309866, 260248338, 2570702480, 2067671015, 3168497089, - 361311552, 2123195373, 2825457193, 2599488181, 507483626, 1201669979, 910763802, - 4158584821, 3116016424, 3375736126, 2857697336, 3112473104, 2683465481, 1495348009, - 681020485, 4044713962, 2443109893, 129994063, 1710251126, 820410567, 601527649, - 1007603132, 2096580480, 1942768885, 1984297765, 1888157243, 960265104, 527990410, - 1572910026, 1106822080, 1472807331, 2465011897, 3139401215, 3705452371, 393081842, - 3826516196, 2576499701, 4130037087, 4155028170, 2188282304, 2949056849, 1138928618, - 858751984, 3735375571, 496972334, 830265621, 1355757111, 909444416, 1337622259, - 2616327935, 2337227347, 2139876075, 4269663356, 796316592, 1893005585, 3958551664, - 1746456069, 2430709714, 4025417573, 3333292799, 1833607331, 3864559081, 3415700826, - 3291421244, 1987321873, 1792851165, 505718946, 3755903648, 3351468604, 4035552813, - 2052557448, 2879065999, 22933757, 2160014758, 2321387515, 3556641619, 1350613766, - 2107757927, 309323868, 1850700415, 3424626316, 545651740, 1935764720, 2349926457, - 979047283, 862046434, 167198649, 338290297, 1145807303, 4201635137, 883549156, - 907871968, 638566313, 678764227, 2637527083, 3306095271, 1452688488, 621126888, - 1838613968, 117966344, 1785040579, 454879400, 1952849106, 1907701866, 733998186, - 1623968604, 2997422000, 2633878652, 1036670775, 3360491537, 3053533585, 1654709920, - 1950620393, 4177745509, 1716078578, 2888573892, 2825965566, 1399790365, 1307674482, - 2776111761, 384880529, 3497622676, 1024664651, 1541456182, 618916624, 1008618602, - 2816673160, 684112698, 3332034744, 3340690804, 962627614, 3072103948, 3713153075, - 476323310, 902990902, 126319837, 2155412848, 228580793, 2142696490, 4254301999, - 2689599232, 426191419, 3157759186, 100239709, 3216403640, 1259677734, 2889209278, - 1754641396, 1057778427, 3221479262, 1298500018, 3340348062, 3967049659, 2927469144, - 3926654939, 2271956463, 3004309866, 260248338, 2570702480, 1879451653, 3168497089, - 361311552, 2123195373, 2825457193, 341920668, 507483626, 1201669979, 910763802, - 4158584821, 3748705813, 3375736126, 2857697336, 3112473104, 2683465481, 3336305747, - 681020485, 4044713962, 2443109893, 129994063, 2578353596, 820410567, 601527649, - 1007603132, 2096580480, 3155251071, 1984297765, 1888157243, 960265104, 527990410, - 2548692624, 1106822080, 1472807331, 2465011897, 3139401215, 736629379, 393081842, - 3826516196, 2576499701, 4130037087, 2440227627, 2188282304, 2949056849, 1138928618, - 858751984, 191805249, 496972334, 830265621, 1355757111, 909444416, 396738554, - 2616327935, 2337227347, 2139876075, 4269663356, 3932761947, 1893005585, 3958551664, - 1746456069, 2430709714, 3171160829, 3333292799, 1833607331, 3864559081, 3415700826, - 1332800826, 1987321873, 1792851165, 505718946, 3755903648, 1770588062, 4035552813, - 2052557448, 2879065999, 22933757, 3159941473, 2321387515, 3556641619, 1350613766, - 2107757927, 2669366188, 1850700415, 3424626316, 545651740, 1935764720, 3252475208, - 979047283, 862046434, 167198649, 338290297, 771814471, 4201635137, 883549156, - 907871968, 638566313, 184144160, 2637527083, 3306095271, 1452688488, 621126888, - 4275587594, 117966344, 1785040579, 454879400, 1952849106, 3806424990, 733998186, - 1623968604, 2997422000, 2633878652, 2670843077, 3360491537, 3053533585, 1654709920, - 1950620393, 3541927406, 1716078578, 2888573892, 2825965566, 1399790365, 3184295779, - 2776111761, 384880529, 3497622676, 1024664651, 723804135, 618916624, 1008618602, - 2816673160, 684112698, 3275521308, 3340690804, 962627614, 3072103948, 3713153075, - 2023106558, 902990902, 126319837, 2155412848, 228580793, 3978575748, 4254301999, - 2689599232, 426191419, 3157759186, 2446138116, 3216403640, 1259677734, 2889209278, - 1754641396, 1706032491, 3221479262, 1298500018, 3340348062, 3967049659, 3805001240, - 3926654939, 2271956463, 3004309866, 260248338, 294480880, 1879451653, 3168497089, - 361311552, 2123195373, 2080604411, 341920668, 507483626, 1201669979, 910763802, - 2012149356, 3748705813, 3375736126, 2857697336, 3112473104, 2935748807, 3336305747, - 681020485, 4044713962, 2443109893, 2862982895, 2578353596, 820410567, 601527649, - 1007603132, 1890290066, 3155251071, 1984297765, 1888157243, 960265104, 41870487, - 2548692624, 1106822080, 1472807331, 2465011897, 2382974023, 736629379, 393081842, - 3826516196, 2576499701, 4219335149, 2440227627, 2188282304, 2949056849, 1138928618, - 3785297102, 191805249, 496972334, 830265621, 1355757111, 3962907313, 396738554, - 2616327935, 2337227347, 2139876075, 552154011, 3932761947, 1893005585, 3958551664, - 1746456069, 895507243, 3171160829, 3333292799, 1833607331, 3864559081, 3564325554, - 1332800826, 1987321873, 1792851165, 505718946, 3245448088, 1770588062, 4035552813, - 2052557448, 2879065999, 3602157977, 3159941473, 2321387515, 3556641619, 1350613766, - 4101259055, 2669366188, 1850700415, 3424626316, 545651740, 2873707882, 3252475208, - 979047283, 862046434, 167198649, 654196140, 771814471, 4201635137, 883549156, - 907871968, 191965184, 184144160, 2637527083, 3306095271, 1452688488, 1562736568, - 4275587594, 117966344, 1785040579, 454879400, 3484019450, 3806424990, 733998186, - 1623968604, 2997422000, 273316614, 2670843077, 3360491537, 3053533585, 1654709920, - 591311873, 3541927406, 1716078578, 2888573892, 2825965566, 2277117038, 3184295779, - 2776111761, 384880529, 3497622676, 1086566797, 723804135, 618916624, 1008618602, - 2816673160, 3344392942, 3275521308, 3340690804, 962627614, 3072103948, 2910444460, - 2023106558, 902990902, 126319837, 2155412848, 337119596, 3978575748, 4254301999, - 2689599232, 426191419, 3471778695, 2446138116, 3216403640, 1259677734, 2889209278, - 4102983766, 1706032491, 3221479262, 1298500018, 3340348062, 2940293024, 3805001240, - 3926654939, 2271956463, 3004309866, 3634668003, 294480880, 1879451653, 3168497089, - 361311552, 3417679321, 2080604411, 341920668, 507483626, 1201669979, 3174274528, - 2012149356, 3748705813, 3375736126, 2857697336, 3929686609, 2935748807, 3336305747, - 681020485, 4044713962, 405011299, 2862982895, 2578353596, 820410567, 601527649, - 4281957726, 1890290066, 3155251071, 1984297765, 1888157243, 1978308818, 41870487, - 2548692624, 1106822080, 1472807331, 3701147046, 2382974023, 736629379, 393081842, - 3826516196, 3225163595, 4219335149, 2440227627, 2188282304, 2949056849, 3894577191, - 3785297102, 191805249, 496972334, 830265621, 4293577013, 3962907313, 396738554, - 2616327935, 2337227347, 3701032380, 552154011, 3932761947, 1893005585, 3958551664, - 4148575672, 895507243, 3171160829, 3333292799, 1833607331, 1596419195, 3564325554, - 1332800826, 1987321873, 1792851165, 3663406943, 3892533309, 247565591, 953356243, - 4103354183, 1908418768, 3915294912, 2390669489, 3865260287, 1818313429, 557880278, - 2499771815, 2618380525, 732785004, 1414011135, 2858311749, 3871596970, 2428464498, - 645476041, 683035653, 4079609082, 2404111028, 3332056297, 3054547484, 3616426087, - 1311379849, 3682136336, 3795847093, 1509718393, 541389178, 1103876446, 2549442278, - 3656600574, 3019560735, 523610761, 3889482885, 3080739216, 2359120072, 1034857006, - 63567637, 1520176098, 1741685274, 2330217396, 1429674399, 517809884, 2653145241, - 868296581, 646514407, 3166145188, 3023629813, 2333851648, 2967365394, 1828821737, - 3333092181, 445460259, 2682093551, 3655100102, 2592872076, 1588368999, 3964958220, - 755397374, 1912970603, 396253754, 4260038354, 1530898510, 2396805917, 3327501452, - 4235709361, 2762163349, 553869167, 3162483580, 1611891352, 248738605, 3403092967, - 2194464420, 113420452, 1752444845, 3770903547, 2397481985, 2866414964, 2555678075, - 2796010061, 762034588, 2679383682, 1848516655, 3857720381, 1119111363, 1829110546, - 2183620391, 1743838702, 3363053704, 2212810289, 966205413, 3897281091, 2148139678, - 2690229390, 427450194, 3516115778, 1864991059, 134448489, 3397232480, 3999530682, - 1927036992, 3170864927, 3879295489, 134554462, 3447324105, 86678510, 1656551206, - 2844494044, 2469678938, 2885597732, 2715483555, 3566904604, 462585182, 1922457093, - 3035264235, 2866504077, 2031456720, 1598555964, 2569915450, 3947972758, 290683210, - 2465427488, 3504862176, 793156806, 1722326752, 2706215067, 3818976191, 2007064241, - 552144413, 2692866408, 3975075075, 4293828741, 1123460373, 960845744, 1855626484, - 1876934434, 1343778249, 912185207, 127278206, 4168930635, 340393978, 65814528, - 2552086271, 2507474816, 1240220220, 1761964455, 2204917500, 4088965101, 1079310398, - 3071460742, 2188549805, 1064733776, 4191719087, 3221046115, 3772395288, 883516842, - 2077853840, 229484673, - }, - { - 448889887, 3508620909, 4164289950, 155254859, 298319697, 980080883, 3500794888, - 3974907245, 682778656, 382798811, 1500342771, 3942535492, 1039809505, 2126581011, - 561192171, 4046277638, 840733718, 1694555864, 241216466, 4182349979, 2525929010, - 386794637, 349755829, 2959959729, 686974318, 3243688353, 3911051908, 3917458620, - 441833800, 3164548257, 584185450, 450132281, 3528356519, 4275666503, 1317069624, - 817077137, 2945430988, 1532878265, 2542155552, 3348614029, 1419611574, 1245233100, - 1981161828, 1161647342, 2781439556, 3896025436, 2349200248, 1213899699, 860301545, - 1590934964, 3371591516, 2850926464, 2774569126, 907316453, 3541736952, 3572719697, - 278602945, 4257620354, 3396349537, 3144949411, 191271983, 2974056951, 2743594803, - 1119054633, 815666748, 920991498, 187861899, 2008325469, 1548504646, 3749744762, - 993523345, 1171349070, 4105576982, 1559471848, 2656434170, 2795453957, 3357293755, - 4260164297, 2211998873, 1783238785, 2831224398, 1704939914, 2626903427, 1148581053, - 849777796, 4219173763, 694869701, 1297370017, 3573985711, 1739242781, 3680794431, - 400850360, 909653264, 1496585542, 460982606, 828640603, 3993062500, 2145047281, - 1587836828, 912583500, 1234319994, 4276951314, 485282908, 1903750880, 1667769214, - 3950976882, 3711912938, 3626058764, 627857875, 436470402, 1753727232, 50241405, - 206782941, 612110492, 954016857, 2567547031, 3360482779, 820704062, 412722485, - 2044763466, 1915626743, 2703000434, 2755090057, 53587450, 2457122208, 1397065983, - 2822294224, 3024827428, 2201149820, 699377793, 157099022, 2792298089, 3927835437, - 1095494739, 1230723791, 2740420278, 2518077381, 3674832547, 2375246835, 923451748, - 3665432731, 1577970518, 2643388181, 4050379756, 1145072065, 1632232822, 2365350332, - 1126185680, 930842061, 3816331201, 1624573114, 3809118349, 1187817320, 945407897, - 63630679, 1852369563, 971772965, 2229069035, 2320405193, 3474864049, 1666937976, - 448889887, 3508620909, 4164289950, 155254859, 3157319819, 980080883, 3500794888, - 3974907245, 682778656, 3201604042, 1500342771, 3942535492, 1039809505, 2126581011, - 3235144326, 4046277638, 840733718, 1694555864, 241216466, 2728337326, 2525929010, - 386794637, 349755829, 2959959729, 20820947, 3243688353, 3911051908, 3917458620, - 441833800, 4143649787, 584185450, 450132281, 3528356519, 4275666503, 3541347868, - 817077137, 2945430988, 1532878265, 2542155552, 3199458552, 1419611574, 1245233100, - 1981161828, 1161647342, 958085276, 3896025436, 2349200248, 1213899699, 860301545, - 1701089635, 3371591516, 2850926464, 2774569126, 907316453, 1529987826, 3572719697, - 278602945, 4257620354, 3396349537, 4120000342, 191271983, 2974056951, 2743594803, - 1119054633, 4255116655, 920991498, 187861899, 2008325469, 1548504646, 100038488, - 993523345, 1171349070, 4105576982, 1559471848, 2523523381, 2795453957, 3357293755, - 4260164297, 2211998873, 3644225670, 2831224398, 1704939914, 2626903427, 1148581053, - 1292003378, 4219173763, 694869701, 1297370017, 3573985711, 2510138592, 3680794431, - 400850360, 909653264, 1496585542, 1738256576, 828640603, 3993062500, 2145047281, - 1587836828, 3478998519, 1234319994, 4276951314, 485282908, 1903750880, 746205619, - 3950976882, 3711912938, 3626058764, 627857875, 954627753, 1753727232, 50241405, - 206782941, 612110492, 2251018875, 2567547031, 3360482779, 820704062, 412722485, - 2120077037, 1915626743, 2703000434, 2755090057, 53587450, 2696843657, 1397065983, - 2822294224, 3024827428, 2201149820, 3308142895, 157099022, 2792298089, 3927835437, - 1095494739, 730099534, 2740420278, 2518077381, 3674832547, 2375246835, 2126745526, - 3665432731, 1577970518, 2643388181, 4050379756, 2987545029, 1632232822, 2365350332, - 1126185680, 930842061, 3140947362, 1624573114, 3809118349, 1187817320, 945407897, - 1282799903, 1852369563, 971772965, 2229069035, 2320405193, 670134249, 1666937976, - 448889887, 3508620909, 4164289950, 127045110, 3157319819, 980080883, 3500794888, - 3974907245, 2740953010, 3201604042, 1500342771, 3942535492, 1039809505, 306788856, - 3235144326, 4046277638, 840733718, 1694555864, 2260304655, 2728337326, 2525929010, - 386794637, 349755829, 3842816805, 20820947, 3243688353, 3911051908, 3917458620, - 3398227861, 4143649787, 584185450, 450132281, 3528356519, 550401017, 3541347868, - 817077137, 2945430988, 1532878265, 1045681234, 3199458552, 1419611574, 1245233100, - 1981161828, 1153297031, 958085276, 3896025436, 2349200248, 1213899699, 1451842347, - 1701089635, 3371591516, 2850926464, 2774569126, 1269128107, 1529987826, 3572719697, - 278602945, 4257620354, 2479560493, 4120000342, 191271983, 2974056951, 2743594803, - 4081110580, 4255116655, 920991498, 187861899, 2008325469, 1300371976, 100038488, - 993523345, 1171349070, 4105576982, 3010753279, 2523523381, 2795453957, 3357293755, - 4260164297, 207153762, 3644225670, 2831224398, 1704939914, 2626903427, 916783095, - 1292003378, 4219173763, 694869701, 1297370017, 3388725608, 2510138592, 3680794431, - 400850360, 909653264, 2421730678, 1738256576, 828640603, 3993062500, 2145047281, - 2123619770, 3478998519, 1234319994, 4276951314, 485282908, 4002661777, 746205619, - 3950976882, 3711912938, 3626058764, 1230937254, 954627753, 1753727232, 50241405, - 206782941, 460314337, 2251018875, 2567547031, 3360482779, 820704062, 1339598718, - 2120077037, 1915626743, 2703000434, 2755090057, 660730207, 2696843657, 1397065983, - 2822294224, 3024827428, 126840648, 3308142895, 157099022, 2792298089, 3927835437, - 2192535935, 730099534, 2740420278, 2518077381, 3674832547, 1879512787, 2126745526, - 3665432731, 1577970518, 2643388181, 832572764, 2987545029, 1632232822, 2365350332, - 1126185680, 3248646182, 3140947362, 1624573114, 3809118349, 1187817320, 4270855000, - 1282799903, 1852369563, 971772965, 2229069035, 3735782785, 670134249, 1666937976, - 448889887, 3508620909, 3681408470, 127045110, 3157319819, 980080883, 3500794888, - 3967872553, 2740953010, 3201604042, 1500342771, 3942535492, 613854690, 306788856, - 3235144326, 4046277638, 840733718, 3957877023, 2260304655, 2728337326, 2525929010, - 386794637, 1779451936, 3842816805, 20820947, 3243688353, 3911051908, 688470429, - 3398227861, 4143649787, 584185450, 450132281, 3381050556, 550401017, 3541347868, - 817077137, 2945430988, 1859551669, 1045681234, 3199458552, 1419611574, 1245233100, - 53681099, 1153297031, 958085276, 3896025436, 2349200248, 1796144514, 1451842347, - 1701089635, 3371591516, 2850926464, 1337394836, 1269128107, 1529987826, 3572719697, - 278602945, 46913829, 2479560493, 4120000342, 191271983, 2974056951, 1361976701, - 4081110580, 4255116655, 920991498, 187861899, 1237191391, 1300371976, 100038488, - 993523345, 1171349070, 3168325479, 3010753279, 2523523381, 2795453957, 3357293755, - 2142853843, 207153762, 3644225670, 2831224398, 1704939914, 2369686128, 916783095, - 1292003378, 4219173763, 694869701, 4150182218, 3388725608, 2510138592, 3680794431, - 400850360, 654034492, 2421730678, 1738256576, 828640603, 3993062500, 84735560, - 2123619770, 3478998519, 1234319994, 4276951314, 2545204994, 4002661777, 746205619, - 3950976882, 3711912938, 426068544, 1230937254, 954627753, 1753727232, 50241405, - 589286339, 460314337, 2251018875, 2567547031, 3360482779, 3279873953, 1339598718, - 2120077037, 1915626743, 2703000434, 2720159887, 660730207, 2696843657, 1397065983, - 2822294224, 3536645029, 126840648, 3308142895, 157099022, 2792298089, 485214530, - 2192535935, 730099534, 2740420278, 2518077381, 418832171, 1879512787, 2126745526, - 3665432731, 1577970518, 721018, 832572764, 2987545029, 1632232822, 2365350332, - 1769688764, 3248646182, 3140947362, 1624573114, 3809118349, 3561012744, 4270855000, - 1282799903, 1852369563, 971772965, 2160782957, 105464019, 2131462864, 335205049, - 3271229551, 1374396416, 4269753677, 1984596635, 37563880, 3956352262, 2168603656, - 311623712, 1593371323, 351020595, 3439337532, 3130874657, 3613343327, 695789539, - 609797513, 53642143, 1479027519, 1588831722, 262810641, 3418379977, 530167431, - 1962487963, 2410103328, 3360114680, 3548827677, 2735238248, 2136058369, 4013192489, - 4106245442, 2155966460, 3653971354, 1230293148, 3966689348, 3455336684, 3594979856, - 3178937309, 3983796170, 3617590004, 1727358326, 1121418876, 1022562029, 2437823131, - 2733424381, 452731958, 2983755220, 1674750403, 3110921909, 3514365950, 2193238341, - 2073801740, 669573402, 1824298084, 22336337, 3366446304, 1536043612, 2502297553, - 1409641611, 2399583184, 2593245170, 716832039, 4286149460, 814849965, 4239224908, - 2453627262, 976385355, 1846129423, 52096201, 88835472, 2621770794, 2491757130, - 1849417480, 576668065, 2186701850, 3357019214, 442191324, 3662645846, 3653766782, - 2254203663, 1169821059, 3735427676, 2246044748, 2635264668, 2647842566, 1435695450, - 1658777934, 2927080369, 1341088646, 3565982642, 221661496, 3246988243, 2718455491, - 483517148, 4181332651, 1143646375, 1720449423, 331164544, 539836322, 3485371630, - 1110077273, 4088985694, 145720169, 2382276586, 4276410795, 2051956774, 936524156, - 15415192, 1815949694, 272696290, 1495465483, 3102030383, 3546078241, 3272619595, - 759699322, 1161486824, 1146281812, 4194130649, 3936306436, 4077338125, 2127551216, - 2995077453, 209698652, 3836657987, 1782152220, 1642490089, 3695579542, 537862234, - 1696168156, 4022607, 3642864269, 54404878, 2925910542, 3444042482, 1931288691, - 2269375687, 614870298, 1139082272, 3672546472, 3255845763, 2987873616, 3436501734, - 380553853, 750118352, 750708138, 488564982, 2936846643, 3460652101, 3085496886, - 3734224010, 523359404, 2751912206, 3302219188, 2729509827, 1995554251, 2288103059, - 3289667468, 2860301591, - }, - { - 3481653941, 2111903071, 3569014882, 1149634763, 4206972571, 2948781360, 2576820949, - 2587099571, 3987042644, 4255777336, 2829594348, 3832744490, 3554499754, 787920018, - 695635693, 2746034685, 2078139227, 1144320548, 4020978225, 449503505, 3004993826, - 2045843139, 1604631401, 148449881, 457819243, 4089112489, 1713441237, 1790909556, - 3334464951, 3070008305, 811825474, 4089105370, 708239097, 1494832299, 2074902973, - 468898217, 1722559700, 2499754488, 2267939270, 650114709, 549502184, 4040463514, - 4228169080, 4094284819, 1599334548, 2992525399, 2107053637, 197348940, 1669884894, - 3982326753, 4259099320, 1862793542, 1751219817, 2701271514, 2507353222, 1488339939, - 4246544316, 3978321870, 132720476, 3020305599, 154822619, 2595474066, 1654579304, - 1997335204, 891320674, 3153502700, 601607977, 2695457160, 4137981809, 37584248, - 1674050253, 1805619463, 676369068, 2294902904, 658143166, 141452045, 2383327493, - 1222336195, 2628962123, 2378299402, 2724274090, 1783957650, 453206569, 3190116972, - 1480368955, 1145768764, 3628222572, 3108689607, 182547022, 360165920, 3378423016, - 1443723222, 2843274258, 1597581683, 664283285, 258077235, 3071875976, 240688930, - 988895736, 2965351284, 91332032, 941306162, 2464278288, 3493666272, 2437043750, - 2356658919, 24726067, 3025656863, 1343636659, 2408295270, 3097408183, 461428710, - 2449005423, 3220070834, 1418517867, 907095008, 428073188, 1938061314, 2094361729, - 2570445990, 346999411, 990247709, 1630488660, 2574142591, 1466590284, 1906935236, - 1592544037, 4168163186, 2773942807, 939392801, 1610069434, 1935303983, 2962954128, - 2490925509, 4103025390, 3614258069, 174125899, 4113855120, 2449365101, 3384244363, - 4115219971, 3187664453, 4021992190, 2959372973, 2946571025, 2144945539, 388172915, - 1125615727, 881693338, 3313110562, 859388069, 177786360, 4134747901, 616417204, - 2104495620, 783302897, 512784708, 1295821322, 3810209448, 2966899912, 2390608767, - 3481653941, 2111903071, 3569014882, 1149634763, 1385372463, 2948781360, 2576820949, - 2587099571, 3987042644, 2251144849, 2829594348, 3832744490, 3554499754, 787920018, - 73007125, 2746034685, 2078139227, 1144320548, 4020978225, 2729117517, 3004993826, - 2045843139, 1604631401, 148449881, 3343221736, 4089112489, 1713441237, 1790909556, - 3334464951, 1920962856, 811825474, 4089105370, 708239097, 1494832299, 2485576001, - 468898217, 1722559700, 2499754488, 2267939270, 2271486862, 549502184, 4040463514, - 4228169080, 4094284819, 3177940420, 2992525399, 2107053637, 197348940, 1669884894, - 3596140613, 4259099320, 1862793542, 1751219817, 2701271514, 1357847339, 1488339939, - 4246544316, 3978321870, 132720476, 344033794, 154822619, 2595474066, 1654579304, - 1997335204, 1849659590, 3153502700, 601607977, 2695457160, 4137981809, 3559496104, - 1674050253, 1805619463, 676369068, 2294902904, 1583197657, 141452045, 2383327493, - 1222336195, 2628962123, 3486106126, 2724274090, 1783957650, 453206569, 3190116972, - 1939413704, 1145768764, 3628222572, 3108689607, 182547022, 2911760834, 3378423016, - 1443723222, 2843274258, 1597581683, 3599911248, 258077235, 3071875976, 240688930, - 988895736, 4263328855, 91332032, 941306162, 2464278288, 3493666272, 1561559932, - 2356658919, 24726067, 3025656863, 1343636659, 257301433, 3097408183, 461428710, - 2449005423, 3220070834, 3544357262, 907095008, 428073188, 1938061314, 2094361729, - 4112109825, 346999411, 990247709, 1630488660, 2574142591, 1466763688, 1906935236, - 1592544037, 4168163186, 2773942807, 3608227467, 1610069434, 1935303983, 2962954128, - 2490925509, 825197245, 3614258069, 174125899, 4113855120, 2449365101, 167881680, - 4115219971, 3187664453, 4021992190, 2959372973, 1971633162, 2144945539, 388172915, - 1125615727, 881693338, 223946687, 859388069, 177786360, 4134747901, 616417204, - 722598357, 783302897, 512784708, 1295821322, 3810209448, 1589703161, 2390608767, - 3481653941, 2111903071, 3569014882, 2520719089, 1385372463, 2948781360, 2576820949, - 2587099571, 1427210741, 2251144849, 2829594348, 3832744490, 3554499754, 1257461820, - 73007125, 2746034685, 2078139227, 1144320548, 3065859797, 2729117517, 3004993826, - 2045843139, 1604631401, 36092756, 3343221736, 4089112489, 1713441237, 1790909556, - 1504385586, 1920962856, 811825474, 4089105370, 708239097, 4135459720, 2485576001, - 468898217, 1722559700, 2499754488, 1392696606, 2271486862, 549502184, 4040463514, - 4228169080, 2521060775, 3177940420, 2992525399, 2107053637, 197348940, 4225425195, - 3596140613, 4259099320, 1862793542, 1751219817, 3752827533, 1357847339, 1488339939, - 4246544316, 3978321870, 270743120, 344033794, 154822619, 2595474066, 1654579304, - 986127123, 1849659590, 3153502700, 601607977, 2695457160, 437034992, 3559496104, - 1674050253, 1805619463, 676369068, 956939381, 1583197657, 141452045, 2383327493, - 1222336195, 3287498300, 3486106126, 2724274090, 1783957650, 453206569, 3610364652, - 1939413704, 1145768764, 3628222572, 3108689607, 708259891, 2911760834, 3378423016, - 1443723222, 2843274258, 1498209005, 3599911248, 258077235, 3071875976, 240688930, - 3815218922, 4263328855, 91332032, 941306162, 2464278288, 3018835600, 1561559932, - 2356658919, 24726067, 3025656863, 368313673, 257301433, 3097408183, 461428710, - 2449005423, 3690066046, 3544357262, 907095008, 428073188, 1938061314, 2274317748, - 4112109825, 346999411, 990247709, 1630488660, 1584471638, 1466763688, 1906935236, - 1592544037, 4168163186, 473837206, 3608227467, 1610069434, 1935303983, 2962954128, - 391171548, 825197245, 3614258069, 174125899, 4113855120, 2095676907, 167881680, - 4115219971, 3187664453, 4021992190, 4246237180, 1971633162, 2144945539, 388172915, - 1125615727, 3158677395, 223946687, 859388069, 177786360, 4134747901, 4017781965, - 722598357, 783302897, 512784708, 1295821322, 3908594844, 1589703161, 2390608767, - 3481653941, 2111903071, 2713757719, 2520719089, 1385372463, 2948781360, 2576820949, - 638075690, 1427210741, 2251144849, 2829594348, 3832744490, 2871270139, 1257461820, - 73007125, 2746034685, 2078139227, 1974062189, 3065859797, 2729117517, 3004993826, - 2045843139, 772058252, 36092756, 3343221736, 4089112489, 1713441237, 2172680702, - 1504385586, 1920962856, 811825474, 4089105370, 1822881146, 4135459720, 2485576001, - 468898217, 1722559700, 3429640856, 1392696606, 2271486862, 549502184, 4040463514, - 3072935276, 2521060775, 3177940420, 2992525399, 2107053637, 1114377646, 4225425195, - 3596140613, 4259099320, 1862793542, 1439724658, 3752827533, 1357847339, 1488339939, - 4246544316, 1051119047, 270743120, 344033794, 154822619, 2595474066, 3143800435, - 986127123, 1849659590, 3153502700, 601607977, 2334441739, 437034992, 3559496104, - 1674050253, 1805619463, 455274178, 956939381, 1583197657, 141452045, 2383327493, - 1520979444, 3287498300, 3486106126, 2724274090, 1783957650, 2212706740, 3610364652, - 1939413704, 1145768764, 3628222572, 2719501850, 708259891, 2911760834, 3378423016, - 1443723222, 2678486648, 1498209005, 3599911248, 258077235, 3071875976, 513762712, - 3815218922, 4263328855, 91332032, 941306162, 3000922309, 3018835600, 1561559932, - 2356658919, 24726067, 3626352172, 368313673, 257301433, 3097408183, 461428710, - 2370224855, 3690066046, 3544357262, 907095008, 428073188, 2279237523, 2274317748, - 4112109825, 346999411, 990247709, 896290404, 1584471638, 1466763688, 1906935236, - 1592544037, 2387522308, 473837206, 3608227467, 1610069434, 1935303983, 4120978868, - 391171548, 825197245, 3614258069, 174125899, 2940674123, 2095676907, 167881680, - 4115219971, 3187664453, 456143482, 4246237180, 1971633162, 2144945539, 388172915, - 4041481385, 3158677395, 223946687, 859388069, 177786360, 3094936989, 4017781965, - 722598357, 783302897, 512784708, 4078350595, 2002159085, 3374931831, 1327513052, - 4231642441, 2398594140, 2750176655, 2377078716, 3051451207, 2923556938, 392203913, - 970480700, 1611278056, 1212903807, 85815670, 2398261756, 1052760308, 175807153, - 2617028873, 1862087601, 1824020594, 3770624867, 141863380, 2090619424, 3994019338, - 2363183556, 3095139522, 1792884692, 3026343485, 2320955816, 145789343, 214170401, - 2926373126, 3858640613, 2188241463, 459887603, 2117474937, 2514234285, 1454156613, - 1675396814, 4188979068, 1584843874, 3594779833, 563029256, 28681425, 446949770, - 3498545218, 435874305, 3448653884, 863509898, 2247299904, 4211345429, 971855563, - 1475394960, 3401692834, 167361776, 496249436, 1465278889, 780336162, 2108770597, - 1806981510, 3677875653, 1890122303, 16399665, 2747394159, 2098019492, 1597583332, - 1763649529, 1286079969, 1846278877, 1016796923, 959676917, 3091540766, 1626192266, - 780987350, 1102963422, 2507002232, 691766944, 193328868, 981596600, 2384820612, - 3149668778, 1691569420, 2852237957, 893819979, 2572584243, 216077070, 1267249886, - 2572508880, 1706489454, 2391561733, 2608477467, 209783612, 765896849, 3617020328, - 3488800100, 2237655981, 2095308189, 963275857, 3563488318, 1865487834, 480006810, - 18562439, 1025913188, 3368592397, 374648713, 2421713724, 2705651398, 3098059650, - 1109934605, 3085839620, 3184266772, 2359972463, 862934481, 3624479194, 3574284465, - 2700143837, 2468083868, 3798800988, 4116964911, 1832002264, 4276154871, 3256889524, - 4036954281, 697729046, 886223984, 2196986730, 1157617208, 1995907944, 398452318, - 3523714364, 613570866, 2962430983, 1408814780, 892117129, 4173164219, 3894076479, - 2721348430, 555734931, 1869034419, 336114876, 3142554871, 3349604636, 3450290892, - 955122895, 2202902910, 2558366468, 1701182712, 283197682, 1865942385, 2027648778, - 2285857699, 880475184, 958651279, 169534250, 3842420528, 1568559789, 2986618464, - 2568345525, 3081082692, - }, - { - 575494427, 2773243709, 4009191487, 3877909663, 2252044261, 1328043370, 1407136778, - 3204434425, 3881653592, 1481049819, 2939203697, 889352935, 628666312, 165199023, - 2949092155, 1116804589, 998930334, 4144153491, 4191022348, 9022505, 4033326555, - 2329569601, 824756145, 3501916851, 1481410328, 1970954319, 4022176157, 2356841052, - 3783173734, 3649102345, 3205430658, 1460938436, 280282398, 3262135457, 4055383786, - 28522973, 1100901182, 4048609665, 994490185, 2888527367, 3591919750, 65093467, - 399797207, 3377740861, 3103183487, 3696509979, 866353724, 3847992271, 2821933890, - 1491144079, 1702442928, 1271285504, 636444475, 2465430290, 2440306765, 2651443172, - 2895101023, 43843628, 518479547, 3708355608, 2313400729, 3786408564, 2823763904, - 3267560272, 524168411, 2580824843, 2687886610, 785942949, 2624395631, 3713348903, - 4104123478, 2234056629, 2683158959, 1805382347, 1645702909, 382688861, 2843792951, - 39122499, 2765954033, 3033237617, 784228054, 1680611136, 2306036746, 892707919, - 3825738103, 1289362844, 3462989616, 484526950, 178560970, 1863413515, 71290794, - 1716785670, 3881310302, 2826977504, 2312744076, 1000001815, 1580868938, 3808984884, - 2521899773, 738699928, 2244576791, 1833964269, 1361345793, 2934763305, 2944705940, - 2334116476, 674208214, 587191877, 271361277, 1639419136, 2742744205, 2556530506, - 3764115510, 861410771, 3473658359, 2879790483, 1497452846, 1101855458, 2268199923, - 1766359872, 480532790, 2926891626, 1366888524, 2262816900, 620045088, 2279182738, - 2479688463, 427385986, 271096497, 1999040724, 1980388138, 3104550456, 2496325717, - 2941450111, 1784373495, 4020221165, 2567325850, 2636190539, 2764516078, 2285887821, - 2395930109, 1867061176, 665795763, 3869868300, 4033135159, 2589983679, 682593183, - 1254600537, 1701095863, 3738080583, 369734429, 2231641462, 1866531599, 1317004965, - 466053171, 2320346625, 485850108, 1279183025, 423884362, 1878291714, 228799661, - 575494427, 2773243709, 4009191487, 3877909663, 1392246100, 1328043370, 1407136778, - 3204434425, 3881653592, 303018213, 2939203697, 889352935, 628666312, 165199023, - 691563049, 1116804589, 998930334, 4144153491, 4191022348, 2882458100, 4033326555, - 2329569601, 824756145, 3501916851, 3512382126, 1970954319, 4022176157, 2356841052, - 3783173734, 3277915742, 3205430658, 1460938436, 280282398, 3262135457, 416160861, - 28522973, 1100901182, 4048609665, 994490185, 2206150488, 3591919750, 65093467, - 399797207, 3377740861, 3954301001, 3696509979, 866353724, 3847992271, 2821933890, - 482325742, 1702442928, 1271285504, 636444475, 2465430290, 476965483, 2651443172, - 2895101023, 43843628, 518479547, 2354104222, 2313400729, 3786408564, 2823763904, - 3267560272, 1682576095, 2580824843, 2687886610, 785942949, 2624395631, 3219885224, - 4104123478, 2234056629, 2683158959, 1805382347, 4143809855, 382688861, 2843792951, - 39122499, 2765954033, 2870716981, 784228054, 1680611136, 2306036746, 892707919, - 2648492467, 1289362844, 3462989616, 484526950, 178560970, 3047404165, 71290794, - 1716785670, 3881310302, 2826977504, 2439325884, 1000001815, 1580868938, 3808984884, - 2521899773, 2222792732, 2244576791, 1833964269, 1361345793, 2934763305, 655108124, - 2334116476, 674208214, 587191877, 271361277, 1403491312, 2742744205, 2556530506, - 3764115510, 861410771, 2748819627, 2879790483, 1497452846, 1101855458, 2268199923, - 2646753562, 480532790, 2926891626, 1366888524, 2262816900, 691077353, 2279182738, - 2479688463, 427385986, 271096497, 357444234, 1980388138, 3104550456, 2496325717, - 2941450111, 717953620, 4020221165, 2567325850, 2636190539, 2764516078, 588189150, - 2395930109, 1867061176, 665795763, 3869868300, 2245339306, 2589983679, 682593183, - 1254600537, 1701095863, 3193417815, 369734429, 2231641462, 1866531599, 1317004965, - 1295326133, 2320346625, 485850108, 1279183025, 423884362, 1310342080, 228799661, - 575494427, 2773243709, 4009191487, 3178129190, 1392246100, 1328043370, 1407136778, - 3204434425, 558594993, 303018213, 2939203697, 889352935, 628666312, 3995857198, - 691563049, 1116804589, 998930334, 4144153491, 2375099047, 2882458100, 4033326555, - 2329569601, 824756145, 3031828205, 3512382126, 1970954319, 4022176157, 2356841052, - 1599294097, 3277915742, 3205430658, 1460938436, 280282398, 2438973535, 416160861, - 28522973, 1100901182, 4048609665, 2989609671, 2206150488, 3591919750, 65093467, - 399797207, 183644195, 3954301001, 3696509979, 866353724, 3847992271, 1244421011, - 482325742, 1702442928, 1271285504, 636444475, 3659422961, 476965483, 2651443172, - 2895101023, 43843628, 2230230933, 2354104222, 2313400729, 3786408564, 2823763904, - 4146329709, 1682576095, 2580824843, 2687886610, 785942949, 126345381, 3219885224, - 4104123478, 2234056629, 2683158959, 1734650983, 4143809855, 382688861, 2843792951, - 39122499, 3527484969, 2870716981, 784228054, 1680611136, 2306036746, 1606477743, - 2648492467, 1289362844, 3462989616, 484526950, 3730796296, 3047404165, 71290794, - 1716785670, 3881310302, 4233965062, 2439325884, 1000001815, 1580868938, 3808984884, - 1228341642, 2222792732, 2244576791, 1833964269, 1361345793, 3313812192, 655108124, - 2334116476, 674208214, 587191877, 1531247446, 1403491312, 2742744205, 2556530506, - 3764115510, 2419989900, 2748819627, 2879790483, 1497452846, 1101855458, 1430402656, - 2646753562, 480532790, 2926891626, 1366888524, 1848714433, 691077353, 2279182738, - 2479688463, 427385986, 3906690631, 357444234, 1980388138, 3104550456, 2496325717, - 2272350403, 717953620, 4020221165, 2567325850, 2636190539, 1950604113, 588189150, - 2395930109, 1867061176, 665795763, 1735147895, 2245339306, 2589983679, 682593183, - 1254600537, 1518037357, 3193417815, 369734429, 2231641462, 1866531599, 1751783137, - 1295326133, 2320346625, 485850108, 1279183025, 149835864, 1310342080, 228799661, - 575494427, 2773243709, 1505829825, 3178129190, 1392246100, 1328043370, 1407136778, - 856233019, 558594993, 303018213, 2939203697, 889352935, 625515593, 3995857198, - 691563049, 1116804589, 998930334, 3264640128, 2375099047, 2882458100, 4033326555, - 2329569601, 1824812377, 3031828205, 3512382126, 1970954319, 4022176157, 3682468973, - 1599294097, 3277915742, 3205430658, 1460938436, 2034940270, 2438973535, 416160861, - 28522973, 1100901182, 3534874298, 2989609671, 2206150488, 3591919750, 65093467, - 2231373121, 183644195, 3954301001, 3696509979, 866353724, 1479968372, 1244421011, - 482325742, 1702442928, 1271285504, 3834022401, 3659422961, 476965483, 2651443172, - 2895101023, 1042443120, 2230230933, 2354104222, 2313400729, 3786408564, 2940290545, - 4146329709, 1682576095, 2580824843, 2687886610, 895602439, 126345381, 3219885224, - 4104123478, 2234056629, 3633565082, 1734650983, 4143809855, 382688861, 2843792951, - 3076342354, 3527484969, 2870716981, 784228054, 1680611136, 3667923304, 1606477743, - 2648492467, 1289362844, 3462989616, 1338592032, 3730796296, 3047404165, 71290794, - 1716785670, 995728648, 4233965062, 2439325884, 1000001815, 1580868938, 1245957136, - 1228341642, 2222792732, 2244576791, 1833964269, 2899552190, 3313812192, 655108124, - 2334116476, 674208214, 1154789946, 1531247446, 1403491312, 2742744205, 2556530506, - 1668620496, 2419989900, 2748819627, 2879790483, 1497452846, 177853954, 1430402656, - 2646753562, 480532790, 2926891626, 3179057526, 1848714433, 691077353, 2279182738, - 2479688463, 1988854710, 3906690631, 357444234, 1980388138, 3104550456, 1772857305, - 2272350403, 717953620, 4020221165, 2567325850, 3129906484, 1950604113, 588189150, - 2395930109, 1867061176, 2248975336, 1735147895, 2245339306, 2589983679, 682593183, - 3087155398, 1518037357, 3193417815, 369734429, 2231641462, 1858424931, 1751783137, - 1295326133, 2320346625, 485850108, 2471611230, 107369761, 2623559579, 4256589070, - 2365810185, 907910243, 3901832478, 2068079364, 2072842987, 401440347, 1707255913, - 1450112231, 2618898012, 600446000, 788321632, 4119629235, 2648781584, 1927659116, - 171372782, 1789511950, 2648296999, 3558619514, 1819608632, 1392007708, 2918513974, - 2270003900, 784021820, 1379044539, 591935962, 1638390839, 10832053, 3946625290, - 2916913801, 2718331169, 1595482738, 1294279402, 19889234, 1374364843, 571354125, - 3357938719, 2337506269, 905453029, 2504232400, 258673393, 2590342355, 3308443353, - 3359617898, 2686453711, 932545954, 509832408, 820508114, 431186194, 3434866166, - 1108455121, 2802986572, 893446102, 3248197798, 1797985531, 3952804303, 558601278, - 1813674114, 311050994, 425175161, 1125527204, 1597986581, 2282580210, 1659733126, - 2080660004, 4121079137, 3373787661, 1902252724, 2669993847, 2450915273, 2155525933, - 2139535914, 274595185, 1890506924, 2631794527, 1423530517, 4027031002, 1085427968, - 2402514206, 3591455043, 2513094696, 2338347202, 1168222597, 3922339535, 3991725466, - 2774598759, 3478721168, 3676766916, 179748891, 2911159372, 191101265, 3389843262, - 3093358663, 2333576084, 1056514165, 2987497874, 2502331872, 2027710028, 2338525812, - 3904906078, 806669884, 596300960, 1993055778, 1541809402, 3578865742, 652348267, - 3332532764, 2656602623, 2037214047, 323260312, 3310408133, 4037617529, 137297627, - 1236501991, 495817051, 481150309, 3067841968, 3120347176, 714354848, 1554632062, - 2522324107, 4274051212, 2180914534, 1261686356, 3569290041, 1801431819, 4286755560, - 2749452442, 829235089, 2243153325, 2525168177, 1486881882, 585653228, 3288336688, - 2734161045, 30430534, 714492313, 2582732426, 595577790, 1463554287, 1949506865, - 4210942156, 2008105540, 4055753132, 2530320603, 319064177, 2305067982, 3825716413, - 1543867515, 108979478, 3089716545, 2921391708, 2403595525, 3783697766, 2313991047, - 3302598706, 1318323763, - }, - { - 1470380360, 3057428612, 2756676297, 1633786556, 4246459918, 2557524017, 1857180133, - 618903690, 2475611092, 2621430634, 2084292404, 1698607774, 1788956972, 3375072220, - 1499167056, 1218814632, 3699503479, 588281768, 3603925285, 1187721841, 1307962320, - 2562217840, 3882506958, 2387033730, 2097027049, 1593669125, 1899433035, 4039983902, - 1546854551, 1073191673, 3368453769, 3074694838, 534637095, 1860006723, 3416402670, - 802354899, 3998709605, 3944315555, 3454226397, 1648185195, 488532673, 3063734121, - 1318974867, 187087202, 200160693, 4170479404, 782764886, 4007973657, 1651636372, - 3084151528, 2085263921, 2424937940, 230704223, 3342587983, 1093085714, 683877298, - 3635026316, 3839461209, 2977567556, 3947448199, 3767172681, 1350679624, 3541409523, - 3975162472, 2459379316, 3287828387, 1565768431, 3149625429, 1328627497, 2156355750, - 112739894, 4052025045, 1396839113, 212349044, 110706825, 2185320852, 2540909191, - 2129623107, 3515174750, 2669147508, 1243549180, 3996575850, 149304348, 2755670869, - 930137412, 350687475, 1512442864, 3764389325, 3489308665, 276147411, 2268414314, - 30674096, 3202650841, 3446821592, 3341145621, 3749209259, 674361204, 1384681012, - 2716655878, 454169262, 289282175, 966029495, 3052791893, 3111969089, 1151599976, - 3620936019, 1877909034, 1953262994, 4240669039, 1857402256, 3337397349, 2392730459, - 1158928694, 1757447952, 2682284750, 2796982914, 1203210173, 797579212, 1645601877, - 3579805998, 797556690, 4106236617, 1379943929, 129105346, 3950170317, 723231430, - 88997404, 2591283275, 359831168, 306903531, 1987846974, 2654779951, 3724360049, - 1693615498, 1095306415, 3586751806, 2045807380, 2779363615, 2912940562, 1557518560, - 3620536996, 1723152132, 4087191232, 1042907094, 3210303305, 1536493323, 4094765090, - 575328723, 359319532, 2458971265, 3159207510, 387883436, 2521400838, 2359639886, - 261289463, 2094643916, 2269112547, 2387198764, 3619233779, 3019052785, 2910774311, - 1470380360, 3057428612, 2756676297, 1633786556, 386502519, 2557524017, 1857180133, - 618903690, 2475611092, 30080431, 2084292404, 1698607774, 1788956972, 3375072220, - 1158684464, 1218814632, 3699503479, 588281768, 3603925285, 238328161, 1307962320, - 2562217840, 3882506958, 2387033730, 3010587639, 1593669125, 1899433035, 4039983902, - 1546854551, 4192218972, 3368453769, 3074694838, 534637095, 1860006723, 652336168, - 802354899, 3998709605, 3944315555, 3454226397, 1926499185, 488532673, 3063734121, - 1318974867, 187087202, 1106075322, 4170479404, 782764886, 4007973657, 1651636372, - 2404132022, 2085263921, 2424937940, 230704223, 3342587983, 918664020, 683877298, - 3635026316, 3839461209, 2977567556, 1943458501, 3767172681, 1350679624, 3541409523, - 3975162472, 276593262, 3287828387, 1565768431, 3149625429, 1328627497, 1428675465, - 112739894, 4052025045, 1396839113, 212349044, 4056830215, 2185320852, 2540909191, - 2129623107, 3515174750, 1542171596, 1243549180, 3996575850, 149304348, 2755670869, - 3578672658, 350687475, 1512442864, 3764389325, 3489308665, 1546094236, 2268414314, - 30674096, 3202650841, 3446821592, 2954172575, 3749209259, 674361204, 1384681012, - 2716655878, 3784818668, 289282175, 966029495, 3052791893, 3111969089, 4157356036, - 3620936019, 1877909034, 1953262994, 4240669039, 558548232, 3337397349, 2392730459, - 1158928694, 1757447952, 2764253876, 2796982914, 1203210173, 797579212, 1645601877, - 1754284241, 797556690, 4106236617, 1379943929, 129105346, 1072954804, 723231430, - 88997404, 2591283275, 359831168, 3790749526, 1987846974, 2654779951, 3724360049, - 1693615498, 529478744, 3586751806, 2045807380, 2779363615, 2912940562, 3883779003, - 3620536996, 1723152132, 4087191232, 1042907094, 2510614710, 1536493323, 4094765090, - 575328723, 359319532, 4185709932, 3159207510, 387883436, 2521400838, 2359639886, - 143795416, 2094643916, 2269112547, 2387198764, 3619233779, 2856133500, 2910774311, - 1470380360, 3057428612, 2756676297, 1184346658, 386502519, 2557524017, 1857180133, - 618903690, 113530176, 30080431, 2084292404, 1698607774, 1788956972, 1446640841, - 1158684464, 1218814632, 3699503479, 588281768, 145530757, 238328161, 1307962320, - 2562217840, 3882506958, 2145494995, 3010587639, 1593669125, 1899433035, 4039983902, - 1668183055, 4192218972, 3368453769, 3074694838, 534637095, 1759744354, 652336168, - 802354899, 3998709605, 3944315555, 3058692249, 1926499185, 488532673, 3063734121, - 1318974867, 728549366, 1106075322, 4170479404, 782764886, 4007973657, 3270440405, - 2404132022, 2085263921, 2424937940, 230704223, 3329510499, 918664020, 683877298, - 3635026316, 3839461209, 79335966, 1943458501, 3767172681, 1350679624, 3541409523, - 925084463, 276593262, 3287828387, 1565768431, 3149625429, 3775346659, 1428675465, - 112739894, 4052025045, 1396839113, 865124022, 4056830215, 2185320852, 2540909191, - 2129623107, 408329043, 1542171596, 1243549180, 3996575850, 149304348, 3549625626, - 3578672658, 350687475, 1512442864, 3764389325, 2745315161, 1546094236, 2268414314, - 30674096, 3202650841, 1591955495, 2954172575, 3749209259, 674361204, 1384681012, - 4064148122, 3784818668, 289282175, 966029495, 3052791893, 1370867977, 4157356036, - 3620936019, 1877909034, 1953262994, 4021792514, 558548232, 3337397349, 2392730459, - 1158928694, 3155295174, 2764253876, 2796982914, 1203210173, 797579212, 3928348491, - 1754284241, 797556690, 4106236617, 1379943929, 535801204, 1072954804, 723231430, - 88997404, 2591283275, 3834650337, 3790749526, 1987846974, 2654779951, 3724360049, - 1042046499, 529478744, 3586751806, 2045807380, 2779363615, 1125934487, 3883779003, - 3620536996, 1723152132, 4087191232, 234512721, 2510614710, 1536493323, 4094765090, - 575328723, 3997395999, 4185709932, 3159207510, 387883436, 2521400838, 3125399953, - 143795416, 2094643916, 2269112547, 2387198764, 652167990, 2856133500, 2910774311, - 1470380360, 3057428612, 2132157457, 1184346658, 386502519, 2557524017, 1857180133, - 4131611047, 113530176, 30080431, 2084292404, 1698607774, 391246724, 1446640841, - 1158684464, 1218814632, 3699503479, 2411874184, 145530757, 238328161, 1307962320, - 2562217840, 2812151676, 2145494995, 3010587639, 1593669125, 1899433035, 2422208371, - 1668183055, 4192218972, 3368453769, 3074694838, 2148785858, 1759744354, 652336168, - 802354899, 3998709605, 1781938823, 3058692249, 1926499185, 488532673, 3063734121, - 3539633540, 728549366, 1106075322, 4170479404, 782764886, 2780824417, 3270440405, - 2404132022, 2085263921, 2424937940, 1908513596, 3329510499, 918664020, 683877298, - 3635026316, 2918953355, 79335966, 1943458501, 3767172681, 1350679624, 341369607, - 925084463, 276593262, 3287828387, 1565768431, 1957429498, 3775346659, 1428675465, - 112739894, 4052025045, 1847440090, 865124022, 4056830215, 2185320852, 2540909191, - 3477402775, 408329043, 1542171596, 1243549180, 3996575850, 179432054, 3549625626, - 3578672658, 350687475, 1512442864, 2118138924, 2745315161, 1546094236, 2268414314, - 30674096, 2317064191, 1591955495, 2954172575, 3749209259, 674361204, 3286542168, - 4064148122, 3784818668, 289282175, 966029495, 1327408800, 1370867977, 4157356036, - 3620936019, 1877909034, 405707683, 4021792514, 558548232, 3337397349, 2392730459, - 3244675609, 3155295174, 2764253876, 2796982914, 1203210173, 2274948223, 3928348491, - 1754284241, 797556690, 4106236617, 2665938417, 535801204, 1072954804, 723231430, - 88997404, 3006584290, 3834650337, 3790749526, 1987846974, 2654779951, 4271242910, - 1042046499, 529478744, 3586751806, 2045807380, 2283867237, 1125934487, 3883779003, - 3620536996, 1723152132, 1761178713, 234512721, 2510614710, 1536493323, 4094765090, - 2361030279, 3997395999, 4185709932, 3159207510, 387883436, 3979684113, 3125399953, - 143795416, 2094643916, 2269112547, 1499026790, 2673871071, 3817604600, 2996498142, - 1211396713, 4016438754, 992969238, 2196610884, 1333868752, 2722471337, 2178395143, - 533478044, 291720336, 3552502714, 1060260388, 1389737501, 3508724089, 3106493936, - 2013154532, 3169850047, 3773175439, 3604033115, 4234678017, 2903156223, 3832188501, - 2874956773, 4283805552, 3664062691, 1974738248, 925764827, 1750660924, 141239116, - 3273085573, 2427940522, 1962727892, 2493949152, 1043482688, 2345076260, 2209086707, - 3642865193, 3119873884, 571850463, 1599484831, 76923002, 3077572436, 4086821865, - 1523654720, 480304732, 476538774, 2169116383, 4033618691, 2819753414, 2856326003, - 747450871, 1851448547, 713503330, 3709263622, 781002495, 1968749577, 2933719965, - 4057398020, 3406593497, 689436820, 2935729647, 2030357428, 2075940397, 1830631914, - 1093330800, 1706624613, 1805612947, 4257097124, 3233604448, 159450674, 1050507045, - 566046625, 2253420120, 904902042, 1830037922, 4081490982, 1427186514, 2535536470, - 3869316947, 4097476542, 930420754, 2519255367, 49908928, 454325685, 888118139, - 3453892181, 1263601461, 1236190782, 674943665, 1648077470, 429399730, 2904879506, - 3718410520, 1802183310, 1872553091, 605480672, 774749173, 3200570514, 181210046, - 2560898144, 3947027625, 1535243167, 324801283, 4234744788, 746560316, 2456297875, - 3925756080, 533997731, 3919796086, 662975152, 864661066, 1070894403, 1020445801, - 1511298602, 4221508348, 3577952702, 4122306502, 2012051572, 1616168260, 2456901413, - 2717726537, 840264605, 2687215223, 2174960097, 1239122603, 2890231920, 3365350767, - 3998868598, 563137220, 893868530, 3400632172, 1538627830, 2812510298, 496662288, - 2317289974, 2252393722, 1221289032, 2418100559, 402670890, 1528570045, 3160531718, - 1806492066, 3211663975, 3617025598, 3664580463, 1338638297, 341637330, 2097019728, - 4031221207, 503636424, 3883416740, 1530237682, 1152125396, 2845384901, 332460372, - 457364876, 1738239808, - }, - { - 1118787884, 1884590246, 1007052798, 3717680750, 1609263052, 2486654530, 2761168910, - 163554565, 3928803020, 2632714628, 1386788970, 2621928183, 2855206157, 2989018213, - 1836814260, 4197635108, 1030118238, 2789863793, 2063944689, 1647608366, 255485979, - 3657534664, 1317185871, 2410074449, 3971156607, 907575923, 4132859581, 416269582, - 877554291, 633895348, 2236014545, 992386759, 3971362318, 2173597771, 1673339632, - 1371742490, 2033574313, 3809530180, 319182848, 1562235776, 463522324, 1482338913, - 1816432405, 3278626272, 1335179249, 171265751, 2249118654, 1153849045, 3013179633, - 1450352108, 1267908572, 1138658121, 623675874, 3608469129, 978093004, 1283228910, - 1810859539, 1179125634, 2939039286, 3862213960, 1168357273, 376788629, 314507445, - 219039712, 463080619, 2994990779, 1035692306, 2228303916, 1280244913, 1965417315, - 1815095408, 939691799, 3080056566, 3741305118, 1495905100, 65327713, 3884301346, - 2536445014, 1503280354, 3398924419, 3678532805, 2616964783, 3168581019, 3553322118, - 3023259169, 480342712, 451634742, 3562778450, 1943708078, 660077747, 434714388, - 2369278293, 2894425895, 1919542250, 2469130567, 551196237, 4193980239, 2952382875, - 3311173667, 2856797012, 2845888917, 1669184098, 3928626091, 2491577076, 3719464032, - 2151963814, 3474431449, 3971510537, 3695841119, 2215238146, 3668152847, 1974578319, - 2328185090, 2096356935, 3973692455, 3954842437, 422675402, 477894725, 3398641827, - 1366451030, 1354642198, 3029840461, 35700837, 2937170986, 1336296570, 3508313874, - 587724229, 2051237478, 3539754304, 1946154432, 2463932452, 144772179, 353408424, - 3493806256, 3782958493, 1957797444, 228084488, 192277278, 3612092522, 2235069734, - 467407503, 3391861572, 847810786, 1838763654, 2272109211, 3018265496, 4249218445, - 1722760791, 3484353162, 3906437663, 4208966227, 2352549740, 714311566, 1346246305, - 2865157059, 2989587005, 3946819548, 3109244860, 3885124598, 3314346978, 952826829, - 1118787884, 1884590246, 1007052798, 3717680750, 1521451317, 2486654530, 2761168910, - 163554565, 3928803020, 2299046195, 1386788970, 2621928183, 2855206157, 2989018213, - 3048269905, 4197635108, 1030118238, 2789863793, 2063944689, 1814057352, 255485979, - 3657534664, 1317185871, 2410074449, 4041610788, 907575923, 4132859581, 416269582, - 877554291, 2338964683, 2236014545, 992386759, 3971362318, 2173597771, 579340117, - 1371742490, 2033574313, 3809530180, 319182848, 3090313228, 463522324, 1482338913, - 1816432405, 3278626272, 2418220643, 171265751, 2249118654, 1153849045, 3013179633, - 2738647190, 1267908572, 1138658121, 623675874, 3608469129, 3096087202, 1283228910, - 1810859539, 1179125634, 2939039286, 2601862091, 1168357273, 376788629, 314507445, - 219039712, 1174181426, 2994990779, 1035692306, 2228303916, 1280244913, 752017703, - 1815095408, 939691799, 3080056566, 3741305118, 126135654, 65327713, 3884301346, - 2536445014, 1503280354, 955981361, 3678532805, 2616964783, 3168581019, 3553322118, - 3772187171, 480342712, 451634742, 3562778450, 1943708078, 1466950454, 434714388, - 2369278293, 2894425895, 1919542250, 317862862, 551196237, 4193980239, 2952382875, - 3311173667, 12728591, 2845888917, 1669184098, 3928626091, 2491577076, 2742989641, - 2151963814, 3474431449, 3971510537, 3695841119, 1005662613, 3668152847, 1974578319, - 2328185090, 2096356935, 3629684995, 3954842437, 422675402, 477894725, 3398641827, - 209352768, 1354642198, 3029840461, 35700837, 2937170986, 1660777984, 3508313874, - 587724229, 2051237478, 3539754304, 3631430985, 2463932452, 144772179, 353408424, - 3493806256, 3616422021, 1957797444, 228084488, 192277278, 3612092522, 3638977910, - 467407503, 3391861572, 847810786, 1838763654, 2427237699, 3018265496, 4249218445, - 1722760791, 3484353162, 2322365400, 4208966227, 2352549740, 714311566, 1346246305, - 954101391, 2989587005, 3946819548, 3109244860, 3885124598, 420941376, 952826829, - 1118787884, 1884590246, 1007052798, 539759724, 1521451317, 2486654530, 2761168910, - 163554565, 1954997983, 2299046195, 1386788970, 2621928183, 2855206157, 3104695189, - 3048269905, 4197635108, 1030118238, 2789863793, 3556473570, 1814057352, 255485979, - 3657534664, 1317185871, 3004205219, 4041610788, 907575923, 4132859581, 416269582, - 2980178044, 2338964683, 2236014545, 992386759, 3971362318, 2573125018, 579340117, - 1371742490, 2033574313, 3809530180, 766585731, 3090313228, 463522324, 1482338913, - 1816432405, 3101578277, 2418220643, 171265751, 2249118654, 1153849045, 2143267892, - 2738647190, 1267908572, 1138658121, 623675874, 2944231951, 3096087202, 1283228910, - 1810859539, 1179125634, 374714364, 2601862091, 1168357273, 376788629, 314507445, - 1710922505, 1174181426, 2994990779, 1035692306, 2228303916, 3222680885, 752017703, - 1815095408, 939691799, 3080056566, 1985366287, 126135654, 65327713, 3884301346, - 2536445014, 3002467868, 955981361, 3678532805, 2616964783, 3168581019, 2173417616, - 3772187171, 480342712, 451634742, 3562778450, 236095606, 1466950454, 434714388, - 2369278293, 2894425895, 1766257461, 317862862, 551196237, 4193980239, 2952382875, - 2416349742, 12728591, 2845888917, 1669184098, 3928626091, 2346338391, 2742989641, - 2151963814, 3474431449, 3971510537, 942354812, 1005662613, 3668152847, 1974578319, - 2328185090, 3234982376, 3629684995, 3954842437, 422675402, 477894725, 2931444539, - 209352768, 1354642198, 3029840461, 35700837, 3388567298, 1660777984, 3508313874, - 587724229, 2051237478, 1770178720, 3631430985, 2463932452, 144772179, 353408424, - 3783114255, 3616422021, 1957797444, 228084488, 192277278, 611095909, 3638977910, - 467407503, 3391861572, 847810786, 1413548572, 2427237699, 3018265496, 4249218445, - 1722760791, 1487262638, 2322365400, 4208966227, 2352549740, 714311566, 1378213368, - 954101391, 2989587005, 3946819548, 3109244860, 4183748384, 420941376, 952826829, - 1118787884, 1884590246, 2199811809, 539759724, 1521451317, 2486654530, 2761168910, - 1100080647, 1954997983, 2299046195, 1386788970, 2621928183, 916352763, 3104695189, - 3048269905, 4197635108, 1030118238, 369866139, 3556473570, 1814057352, 255485979, - 3657534664, 2916985473, 3004205219, 4041610788, 907575923, 4132859581, 3856599532, - 2980178044, 2338964683, 2236014545, 992386759, 3393662326, 2573125018, 579340117, - 1371742490, 2033574313, 1938766053, 766585731, 3090313228, 463522324, 1482338913, - 2122086302, 3101578277, 2418220643, 171265751, 2249118654, 952602228, 2143267892, - 2738647190, 1267908572, 1138658121, 1808026803, 2944231951, 3096087202, 1283228910, - 1810859539, 3881666794, 374714364, 2601862091, 1168357273, 376788629, 728738466, - 1710922505, 1174181426, 2994990779, 1035692306, 74930675, 3222680885, 752017703, - 1815095408, 939691799, 3404352271, 1985366287, 126135654, 65327713, 3884301346, - 1822629733, 3002467868, 955981361, 3678532805, 2616964783, 3865359567, 2173417616, - 3772187171, 480342712, 451634742, 1099609112, 236095606, 1466950454, 434714388, - 2369278293, 2671873359, 1766257461, 317862862, 551196237, 4193980239, 2006763654, - 2416349742, 12728591, 2845888917, 1669184098, 2492983893, 2346338391, 2742989641, - 2151963814, 3474431449, 2095232649, 942354812, 1005662613, 3668152847, 1974578319, - 1748794756, 3234982376, 3629684995, 3954842437, 422675402, 2291986911, 2931444539, - 209352768, 1354642198, 3029840461, 3772709822, 3388567298, 1660777984, 3508313874, - 587724229, 2759789003, 1770178720, 3631430985, 2463932452, 144772179, 1572181309, - 3783114255, 3616422021, 1957797444, 228084488, 4106643586, 611095909, 3638977910, - 467407503, 3391861572, 927151111, 1413548572, 2427237699, 3018265496, 4249218445, - 692575565, 1487262638, 2322365400, 4208966227, 2352549740, 1281886506, 1378213368, - 954101391, 2989587005, 3946819548, 1861811740, 1484768905, 359662140, 4058479705, - 1306547382, 514617018, 1685692791, 3370601554, 2920029077, 447798803, 3124262580, - 1841693810, 583764638, 853545489, 2614348705, 1445696741, 4226719361, 1299450005, - 7404137, 3158806368, 3487160245, 1410910965, 3697116584, 4272452035, 832215403, - 4190877996, 2360539465, 1011144434, 546018244, 613443074, 2523894977, 998991923, - 2569220540, 4221264346, 2627827148, 2606458015, 261584257, 4172552877, 1174774061, - 1040006970, 2378868955, 1539192255, 1322624483, 3221782707, 3352886416, 3634686692, - 65447704, 3962131218, 839088053, 4154193716, 1211888926, 319402483, 3922826413, - 3799829447, 623726612, 1586183272, 1853729462, 2621029589, 708558605, 1618007233, - 2784732545, 953859039, 921654620, 477148727, 3592256598, 2772318818, 1460772911, - 1309227716, 3484274262, 3425161241, 1677052569, 2238155114, 2828087292, 2361598991, - 4283732706, 1530059373, 1564048492, 243829114, 104328994, 3080249237, 2054985396, - 408961407, 2978652320, 2412674552, 3794618070, 3644862703, 2095186402, 3294126752, - 2970218740, 1800713612, 3806665216, 3990918051, 142666452, 531078813, 1079142774, - 3437358350, 635943961, 255576894, 2991317718, 1208676456, 247449774, 454879171, - 113230697, 3064123371, 336269028, 1137083842, 959568850, 2508623991, 3338418112, - 2660268938, 1318010299, 3950178561, 1078499199, 1176289535, 3875152821, 1984420952, - 1134199826, 2944539174, 3667625203, 2034152216, 1648355307, 2376447620, 2967418253, - 185143450, 889002925, 3999315013, 661455858, 4026799358, 3626504428, 3544795311, - 3642718771, 2467387138, 1034249749, 2051371333, 4251353248, 1575036366, 751400924, - 2906720214, 1210002606, 916508568, 1728487600, 2478884914, 3081526615, 1867135009, - 1955998382, 701713417, 512784398, 1255240210, 3665676113, 1771754697, 4000392442, - 3342268855, 2677221913, 369054145, 4011912082, 748537647, 1626721797, 852497405, - 168721778, 3091138383, - }, - { - 3781228998, 1787582256, 838267218, 2710632450, 690892139, 2484870604, 4151302318, - 1844787776, 727768263, 1075391038, 1842903369, 2927332301, 3246688068, 1234715005, - 2906526190, 3369636401, 3091858538, 3320767682, 920496809, 1406803705, 3163880457, - 1540551653, 2733620168, 2588558057, 147277542, 803170440, 821275940, 3897549272, - 151390608, 951639139, 904639695, 1106545578, 1514893712, 998760135, 2557458623, - 4109877399, 578824730, 2174064027, 3352513900, 3206168298, 911932439, 2030004973, - 3283902592, 3755877921, 250434692, 352122318, 977153640, 642640734, 2555395772, - 2307695537, 2593565626, 3738143618, 734614254, 3276420511, 2636087597, 4157371578, - 1082026387, 429736987, 3755125580, 1935957937, 3300547146, 3089498232, 4167244256, - 1619189426, 1094447351, 1061842570, 3666470174, 810916769, 2263633079, 3863543843, - 1804937521, 2774236887, 2858593613, 961498236, 1515309045, 1564424234, 2276602447, - 2540994858, 78621171, 3575132456, 2958793283, 387554009, 688827573, 3833764146, - 2611524056, 2296780370, 2411775612, 3790615886, 3399757437, 1385198595, 1005364336, - 2093159919, 2091827252, 1461775197, 4225171212, 1185831033, 12264437, 1313835999, - 556653278, 917105970, 1471530347, 2010243509, 3097827138, 1399987735, 273352191, - 2505795417, 1336824946, 3358720963, 2874295267, 2282349617, 3478581038, 4027859424, - 713597958, 4059691816, 2812811116, 2291324146, 932688463, 3001334051, 2028368589, - 830582457, 3964293916, 4276849132, 1828058403, 1351688755, 2113265048, 42517349, - 3100438883, 1137792178, 1479076106, 463377892, 3964913740, 2422362185, 436113863, - 2044139049, 4197323265, 3275185975, 2655265571, 1674107588, 1496360114, 3642050139, - 1739051417, 2393774399, 250035802, 10186306, 263338568, 3899157617, 3679157076, - 2258085991, 1407319575, 899008067, 3679828833, 711086272, 2952963707, 3373894808, - 445540851, 3405637490, 1343291195, 730888681, 507768703, 3473963321, 1779803564, - 3781228998, 1787582256, 838267218, 2710632450, 2431224659, 2484870604, 4151302318, - 1844787776, 727768263, 4012573268, 1842903369, 2927332301, 3246688068, 1234715005, - 3405161215, 3369636401, 3091858538, 3320767682, 920496809, 400609988, 3163880457, - 1540551653, 2733620168, 2588558057, 2137935937, 803170440, 821275940, 3897549272, - 151390608, 194431797, 904639695, 1106545578, 1514893712, 998760135, 62528087, - 4109877399, 578824730, 2174064027, 3352513900, 3495516649, 911932439, 2030004973, - 3283902592, 3755877921, 1774462108, 352122318, 977153640, 642640734, 2555395772, - 756528792, 2593565626, 3738143618, 734614254, 3276420511, 4086313763, 4157371578, - 1082026387, 429736987, 3755125580, 526056489, 3300547146, 3089498232, 4167244256, - 1619189426, 82235109, 1061842570, 3666470174, 810916769, 2263633079, 1110270726, - 1804937521, 2774236887, 2858593613, 961498236, 1840197918, 1564424234, 2276602447, - 2540994858, 78621171, 3690913528, 2958793283, 387554009, 688827573, 3833764146, - 3626285597, 2296780370, 2411775612, 3790615886, 3399757437, 1561545830, 1005364336, - 2093159919, 2091827252, 1461775197, 63358970, 1185831033, 12264437, 1313835999, - 556653278, 3918754976, 1471530347, 2010243509, 3097827138, 1399987735, 2767111911, - 2505795417, 1336824946, 3358720963, 2874295267, 902314853, 3478581038, 4027859424, - 713597958, 4059691816, 1462989647, 2291324146, 932688463, 3001334051, 2028368589, - 3594712587, 3964293916, 4276849132, 1828058403, 1351688755, 2571513800, 42517349, - 3100438883, 1137792178, 1479076106, 140519541, 3964913740, 2422362185, 436113863, - 2044139049, 226785542, 3275185975, 2655265571, 1674107588, 1496360114, 46428973, - 1739051417, 2393774399, 250035802, 10186306, 4118320101, 3899157617, 3679157076, - 2258085991, 1407319575, 4267866849, 3679828833, 711086272, 2952963707, 3373894808, - 3662249794, 3405637490, 1343291195, 730888681, 507768703, 2930510271, 1779803564, - 3781228998, 1787582256, 838267218, 1817693489, 2431224659, 2484870604, 4151302318, - 1844787776, 1788220652, 4012573268, 1842903369, 2927332301, 3246688068, 2050648011, - 3405161215, 3369636401, 3091858538, 3320767682, 241001958, 400609988, 3163880457, - 1540551653, 2733620168, 3857223520, 2137935937, 803170440, 821275940, 3897549272, - 1451986523, 194431797, 904639695, 1106545578, 1514893712, 4147878244, 62528087, - 4109877399, 578824730, 2174064027, 461571251, 3495516649, 911932439, 2030004973, - 3283902592, 1580354765, 1774462108, 352122318, 977153640, 642640734, 1019387737, - 756528792, 2593565626, 3738143618, 734614254, 999431451, 4086313763, 4157371578, - 1082026387, 429736987, 140091634, 526056489, 3300547146, 3089498232, 4167244256, - 3202763095, 82235109, 1061842570, 3666470174, 810916769, 3663992550, 1110270726, - 1804937521, 2774236887, 2858593613, 2203639366, 1840197918, 1564424234, 2276602447, - 2540994858, 978199281, 3690913528, 2958793283, 387554009, 688827573, 375113876, - 3626285597, 2296780370, 2411775612, 3790615886, 1277897939, 1561545830, 1005364336, - 2093159919, 2091827252, 1631078873, 63358970, 1185831033, 12264437, 1313835999, - 3872277948, 3918754976, 1471530347, 2010243509, 3097827138, 1291836608, 2767111911, - 2505795417, 1336824946, 3358720963, 3954754615, 902314853, 3478581038, 4027859424, - 713597958, 2198246306, 1462989647, 2291324146, 932688463, 3001334051, 2374736511, - 3594712587, 3964293916, 4276849132, 1828058403, 3619038368, 2571513800, 42517349, - 3100438883, 1137792178, 1146435746, 140519541, 3964913740, 2422362185, 436113863, - 3460540392, 226785542, 3275185975, 2655265571, 1674107588, 1288223861, 46428973, - 1739051417, 2393774399, 250035802, 1986226858, 4118320101, 3899157617, 3679157076, - 2258085991, 551117761, 4267866849, 3679828833, 711086272, 2952963707, 1667866621, - 3662249794, 3405637490, 1343291195, 730888681, 2381246695, 2930510271, 1779803564, - 3781228998, 1787582256, 1236367773, 1817693489, 2431224659, 2484870604, 4151302318, - 2902321811, 1788220652, 4012573268, 1842903369, 2927332301, 1185539274, 2050648011, - 3405161215, 3369636401, 3091858538, 4240555382, 241001958, 400609988, 3163880457, - 1540551653, 2539098607, 3857223520, 2137935937, 803170440, 821275940, 3485313735, - 1451986523, 194431797, 904639695, 1106545578, 1633417190, 4147878244, 62528087, - 4109877399, 578824730, 3671726812, 461571251, 3495516649, 911932439, 2030004973, - 2002341352, 1580354765, 1774462108, 352122318, 977153640, 170033402, 1019387737, - 756528792, 2593565626, 3738143618, 4160516213, 999431451, 4086313763, 4157371578, - 1082026387, 1423352480, 140091634, 526056489, 3300547146, 3089498232, 4266971502, - 3202763095, 82235109, 1061842570, 3666470174, 945994616, 3663992550, 1110270726, - 1804937521, 2774236887, 3776581315, 2203639366, 1840197918, 1564424234, 2276602447, - 928117829, 978199281, 3690913528, 2958793283, 387554009, 2817496615, 375113876, - 3626285597, 2296780370, 2411775612, 1346030561, 1277897939, 1561545830, 1005364336, - 2093159919, 821902776, 1631078873, 63358970, 1185831033, 12264437, 3192617499, - 3872277948, 3918754976, 1471530347, 2010243509, 4011062105, 1291836608, 2767111911, - 2505795417, 1336824946, 1593119272, 3954754615, 902314853, 3478581038, 4027859424, - 1163079365, 2198246306, 1462989647, 2291324146, 932688463, 4018333691, 2374736511, - 3594712587, 3964293916, 4276849132, 3902062310, 3619038368, 2571513800, 42517349, - 3100438883, 1645455709, 1146435746, 140519541, 3964913740, 2422362185, 3338363150, - 3460540392, 226785542, 3275185975, 2655265571, 3789582441, 1288223861, 46428973, - 1739051417, 2393774399, 2257001236, 1986226858, 4118320101, 3899157617, 3679157076, - 3707520907, 551117761, 4267866849, 3679828833, 711086272, 570153549, 1667866621, - 3662249794, 3405637490, 1343291195, 112368058, 2615115584, 2865130041, 357584504, - 528807633, 1816055434, 2854850066, 190222907, 1014915859, 3472967123, 2605782564, - 3353130066, 540430076, 2087143725, 1571283916, 1604766425, 934199876, 3359569795, - 4168578472, 1745876717, 277026333, 2679446726, 3582165485, 3954458991, 2615245404, - 2410035461, 3442004248, 2814474875, 1734556428, 2653422310, 4033890533, 2373774914, - 3011118469, 1276695464, 2995405818, 782363735, 2242531852, 4206829780, 1486885236, - 3764707851, 1945614253, 1147926733, 701960774, 3435251514, 3626050187, 3587799538, - 2399216643, 3217822006, 3600044386, 648239752, 2997947488, 1754097052, 4109638936, - 3413714077, 1038375790, 3394259389, 2284776380, 2711956471, 1278424040, 1272230764, - 3980809660, 1983901240, 894405781, 582621606, 1274260631, 763432985, 1862236664, - 10249416, 3838574116, 1912270458, 3491686662, 2696669149, 312119069, 1812714569, - 2729307370, 3045249652, 303684944, 503720764, 4029412414, 4101616421, 3484358948, - 1261027935, 145713434, 2918444923, 2099546237, 3173693583, 3498398823, 3769717769, - 2860220116, 2919562911, 1221047715, 1749384742, 1018968146, 2771587474, 2746107326, - 1182859751, 2403805226, 2206395932, 1500348209, 1762634532, 3017223998, 2043185588, - 2124568729, 1619852613, 3248258238, 3393223375, 644860154, 2465108160, 2358875673, - 3643741304, 1891106916, 416443047, 3298583974, 1030877276, 2839390034, 4181398645, - 1845333999, 3643365079, 1993116780, 1763857175, 1951718545, 3785659537, 4156412284, - 4138026128, 3480291142, 54280556, 4169041146, 3130638398, 3236816184, 3559898998, - 916420843, 938920758, 3425021599, 1528477728, 3597939783, 3516249439, 936528538, - 4174817780, 2541489033, 3962368135, 2054336507, 2610093970, 3613025255, 3583905994, - 2990129491, 332823408, 2505138276, 3811707598, 373987627, 4263703898, 1668946560, - 3213253899, 2673819338, 1631405099, 3127443274, 549232331, 21447814, 1647238011, - 3093799993, 1922712395, - }, - { - 4224788259, 3569487556, 1080137041, 2788623569, 856160888, 2195536417, 3030463035, - 2906439247, 896055051, 1967105456, 2093562169, 2919742950, 546374698, 1372591815, - 3773616637, 349073007, 1331102855, 3035367896, 1222622311, 2266618592, 74466398, - 1140488004, 855606859, 3803728487, 3589743162, 2748402856, 1044387368, 1494850922, - 2242660891, 3111566003, 2013737074, 163276737, 1526772858, 3047139947, 3150695453, - 2583795468, 3628272447, 305282258, 2151108134, 2905708853, 1052800761, 3354632338, - 1017036861, 2453680791, 2673902555, 1622154585, 2893733051, 3888482522, 306284440, - 3245137245, 3480776670, 2865396581, 3571456526, 3284891766, 1393584874, 1057867320, - 2888126310, 3302325443, 4135187530, 1770789166, 1615533805, 1438727397, 2921922012, - 3156703516, 435047591, 2999350446, 575044884, 1001339111, 625824120, 2489346227, - 2104489492, 2494528446, 1141458836, 4048430074, 2599022749, 2438694106, 1443850072, - 3321658999, 87870515, 958195816, 380666771, 3062272732, 4178548642, 4274603044, - 888566831, 3386636024, 1636806704, 2400069397, 3003029365, 1953620944, 3278772216, - 1562778171, 2767090642, 14436957, 913966574, 1724553886, 2015261135, 4191296122, - 1688939147, 110865735, 2913800286, 4131469475, 315962755, 1531174227, 1226678476, - 3446400266, 3896297836, 539834883, 2871306264, 3333932675, 2229436010, 1928458456, - 464682640, 1786180352, 162599143, 817038005, 3146256537, 1676400403, 2484731087, - 702610427, 4005124049, 1691076958, 1268494739, 4093608833, 3757213737, 2627839929, - 2884764386, 1548110665, 3361745333, 3955318088, 3264527857, 3969225726, 968269281, - 2630991382, 2716444139, 1071781623, 3704437685, 1511193802, 843840414, 1277966236, - 4141095880, 715016637, 1255888181, 1321941951, 1180174408, 1021629824, 3395369301, - 3912221525, 2611782663, 4038117717, 2253029302, 974431991, 347200257, 886823557, - 2275848777, 3732452739, 3708953729, 2688020866, 4185175489, 99605353, 2387945286, - 4224788259, 3569487556, 1080137041, 2788623569, 238715294, 2195536417, 3030463035, - 2906439247, 896055051, 3061240402, 2093562169, 2919742950, 546374698, 1372591815, - 851057115, 349073007, 1331102855, 3035367896, 1222622311, 3305595574, 74466398, - 1140488004, 855606859, 3803728487, 3838112757, 2748402856, 1044387368, 1494850922, - 2242660891, 1038286760, 2013737074, 163276737, 1526772858, 3047139947, 3518918891, - 2583795468, 3628272447, 305282258, 2151108134, 3555155951, 1052800761, 3354632338, - 1017036861, 2453680791, 2394691836, 1622154585, 2893733051, 3888482522, 306284440, - 2055552069, 3480776670, 2865396581, 3571456526, 3284891766, 1179339312, 1057867320, - 2888126310, 3302325443, 4135187530, 683364318, 1615533805, 1438727397, 2921922012, - 3156703516, 1333086260, 2999350446, 575044884, 1001339111, 625824120, 576119652, - 2104489492, 2494528446, 1141458836, 4048430074, 786660788, 2438694106, 1443850072, - 3321658999, 87870515, 457955380, 380666771, 3062272732, 4178548642, 4274603044, - 2256710588, 3386636024, 1636806704, 2400069397, 3003029365, 3733049985, 3278772216, - 1562778171, 2767090642, 14436957, 530062778, 1724553886, 2015261135, 4191296122, - 1688939147, 2981240708, 2913800286, 4131469475, 315962755, 1531174227, 2433363617, - 3446400266, 3896297836, 539834883, 2871306264, 2597546929, 2229436010, 1928458456, - 464682640, 1786180352, 1165821797, 817038005, 3146256537, 1676400403, 2484731087, - 3239493343, 4005124049, 1691076958, 1268494739, 4093608833, 2088690204, 2627839929, - 2884764386, 1548110665, 3361745333, 1075350364, 3264527857, 3969225726, 968269281, - 2630991382, 4103280359, 1071781623, 3704437685, 1511193802, 843840414, 1340474980, - 4141095880, 715016637, 1255888181, 1321941951, 2512565938, 1021629824, 3395369301, - 3912221525, 2611782663, 2287272047, 2253029302, 974431991, 347200257, 886823557, - 3775715445, 3732452739, 3708953729, 2688020866, 4185175489, 2151114047, 2387945286, - 4224788259, 3569487556, 1080137041, 879682447, 238715294, 2195536417, 3030463035, - 2906439247, 3975397430, 3061240402, 2093562169, 2919742950, 546374698, 1928060945, - 851057115, 349073007, 1331102855, 3035367896, 1148668613, 3305595574, 74466398, - 1140488004, 855606859, 917923571, 3838112757, 2748402856, 1044387368, 1494850922, - 995791756, 1038286760, 2013737074, 163276737, 1526772858, 1944370085, 3518918891, - 2583795468, 3628272447, 305282258, 685261037, 3555155951, 1052800761, 3354632338, - 1017036861, 1620076466, 2394691836, 1622154585, 2893733051, 3888482522, 4119309151, - 2055552069, 3480776670, 2865396581, 3571456526, 4008552940, 1179339312, 1057867320, - 2888126310, 3302325443, 2359989247, 683364318, 1615533805, 1438727397, 2921922012, - 2092991022, 1333086260, 2999350446, 575044884, 1001339111, 2406217399, 576119652, - 2104489492, 2494528446, 1141458836, 1856565466, 786660788, 2438694106, 1443850072, - 3321658999, 2752588925, 457955380, 380666771, 3062272732, 4178548642, 1354877973, - 2256710588, 3386636024, 1636806704, 2400069397, 2275777233, 3733049985, 3278772216, - 1562778171, 2767090642, 3438624166, 530062778, 1724553886, 2015261135, 4191296122, - 3842215040, 2981240708, 2913800286, 4131469475, 315962755, 2891870900, 2433363617, - 3446400266, 3896297836, 539834883, 1390877376, 2597546929, 2229436010, 1928458456, - 464682640, 1405678725, 1165821797, 817038005, 3146256537, 1676400403, 9522151, - 3239493343, 4005124049, 1691076958, 1268494739, 4076978821, 2088690204, 2627839929, - 2884764386, 1548110665, 3713129550, 1075350364, 3264527857, 3969225726, 968269281, - 2669129178, 4103280359, 1071781623, 3704437685, 1511193802, 2032747975, 1340474980, - 4141095880, 715016637, 1255888181, 1290704077, 2512565938, 1021629824, 3395369301, - 3912221525, 767420943, 2287272047, 2253029302, 974431991, 347200257, 940587649, - 3775715445, 3732452739, 3708953729, 2688020866, 1603856534, 2151114047, 2387945286, - 4224788259, 3569487556, 4060395365, 879682447, 238715294, 2195536417, 3030463035, - 774839173, 3975397430, 3061240402, 2093562169, 2919742950, 77503099, 1928060945, - 851057115, 349073007, 1331102855, 4216140027, 1148668613, 3305595574, 74466398, - 1140488004, 1728766104, 917923571, 3838112757, 2748402856, 1044387368, 1408900577, - 995791756, 1038286760, 2013737074, 163276737, 936142172, 1944370085, 3518918891, - 2583795468, 3628272447, 1701372078, 685261037, 3555155951, 1052800761, 3354632338, - 2951922777, 1620076466, 2394691836, 1622154585, 2893733051, 2494523614, 4119309151, - 2055552069, 3480776670, 2865396581, 3031455484, 4008552940, 1179339312, 1057867320, - 2888126310, 2970791558, 2359989247, 683364318, 1615533805, 1438727397, 3697460033, - 2092991022, 1333086260, 2999350446, 575044884, 2712063736, 2406217399, 576119652, - 2104489492, 2494528446, 1096189230, 1856565466, 786660788, 2438694106, 1443850072, - 3615481975, 2752588925, 457955380, 380666771, 3062272732, 2387056252, 1354877973, - 2256710588, 3386636024, 1636806704, 517188972, 2275777233, 3733049985, 3278772216, - 1562778171, 3436331606, 3438624166, 530062778, 1724553886, 2015261135, 1711407722, - 3842215040, 2981240708, 2913800286, 4131469475, 878455086, 2891870900, 2433363617, - 3446400266, 3896297836, 4251949215, 1390877376, 2597546929, 2229436010, 1928458456, - 719826541, 1405678725, 1165821797, 817038005, 3146256537, 3883590627, 9522151, - 3239493343, 4005124049, 1691076958, 893183073, 4076978821, 2088690204, 2627839929, - 2884764386, 3312769297, 3713129550, 1075350364, 3264527857, 3969225726, 4161107579, - 2669129178, 4103280359, 1071781623, 3704437685, 1400940789, 2032747975, 1340474980, - 4141095880, 715016637, 1705234794, 1290704077, 2512565938, 1021629824, 3395369301, - 2934074199, 767420943, 2287272047, 2253029302, 974431991, 3060035390, 940587649, - 3775715445, 3732452739, 3708953729, 3489160434, 3200799223, 340420813, 2539294182, - 2619616318, 456806966, 4272538790, 2994564124, 2757588894, 3493053179, 2946195469, - 1402305257, 2266356503, 3512914478, 273195440, 3579761455, 862317458, 1894959361, - 42596779, 376641729, 782820755, 716528645, 222675565, 4038035195, 311038326, - 395780597, 2025474869, 404396572, 4138962756, 2441107014, 3525378401, 947085768, - 3758218091, 3185789607, 638283508, 3802505926, 830259842, 1086400881, 3444485, - 142418107, 4283468141, 1669846189, 955065888, 3864384467, 73139517, 136809048, - 1444329434, 174974637, 3303183786, 282216656, 3114827080, 3811060015, 1610640996, - 3824096289, 1123437514, 3826582808, 39407702, 2437666463, 2454206642, 830758422, - 4190092654, 1941090912, 224373276, 3704201239, 3284012568, 4056152539, 1022047941, - 1077111803, 3028336675, 3207391465, 3459202233, 1991240724, 4184491520, 1851863093, - 1038639595, 1392247730, 2113875749, 1162388509, 2629935260, 3545260772, 991928712, - 4064775043, 4180493781, 2134685922, 642853690, 290065503, 1629968, 3150373868, - 3110755428, 2254306163, 421928533, 11426979, 3042809169, 786868170, 1287942583, - 1851107769, 1444903906, 4150950197, 3737798306, 2848738554, 505924220, 2944131627, - 2639930627, 1339887691, 2382166850, 2668971315, 3944739049, 2217612340, 4142682607, - 997824216, 123465626, 844518179, 1161486362, 2706162053, 2966530827, 4103639053, - 1837121393, 909648429, 298619078, 2057042454, 3613272637, 3609349032, 1664428748, - 1871510359, 58508710, 1079418100, 3278870121, 3821562746, 16654909, 2530580589, - 3361874982, 629910009, 2124761646, 2508133604, 1954315500, 3019833617, 141617625, - 1653192078, 1541695589, 1223978475, 3875963510, 3028691587, 3450826564, 2185849120, - 1956475624, 3053842172, 3550887830, 2672339803, 176823785, 913229929, 681399502, - 2256486297, 2881672598, 597153273, 2782767695, 1133158067, 4126077325, 3456027404, - 754062201, 4069172986, - }, - { - 2441935114, 3465447683, 2897229686, 3845380309, 1199633364, 495424232, 2490548037, - 581670528, 2467171733, 2200094863, 2163927790, 3895792830, 2097210789, 1606544633, - 1305562517, 4072525389, 3256142090, 349440478, 3920932491, 2462464051, 1075951496, - 2835763703, 1593198055, 2380945625, 543531323, 3182766507, 2927484354, 2877470578, - 4153923603, 2443156156, 1168544900, 888955615, 3605412824, 1336677864, 3256116974, - 2884036014, 4070749843, 2989661773, 1095584023, 1370834065, 3534389580, 312378113, - 3190819203, 1247574926, 2046019470, 3536918510, 1479030180, 847820646, 3992973956, - 3827223401, 4113429617, 3504933502, 295000614, 2238923504, 3485717254, 290246351, - 1064210816, 2848539559, 2617134888, 422213010, 2796674561, 3568250500, 2736237915, - 3950756060, 1527249993, 3603540278, 4115393386, 2851621193, 4230341156, 905168850, - 3916344126, 1496013046, 206343742, 2894205125, 1082918859, 2746480417, 3077328661, - 1209440053, 3258293856, 1032236533, 3043332566, 446879604, 587022214, 1614371566, - 3040899994, 3686422145, 937325128, 1968833679, 169086151, 4075432555, 1196046411, - 3101745581, 4228079966, 2942213563, 1195005323, 1673491641, 1762746534, 3641827252, - 694590905, 1828365460, 513716230, 3106485486, 2441593994, 4044462965, 3628121101, - 3957990629, 179764922, 579361186, 3474393871, 2474241006, 4031850878, 3120409532, - 4011587898, 3682942579, 3257272830, 3097029759, 2652540191, 1128762588, 1040256382, - 2743736716, 334893087, 1892049031, 2603159239, 3712772023, 2126593224, 3465793906, - 3180780589, 725740783, 3728108967, 573931936, 137996587, 110756053, 3984787930, - 3773232816, 3406981985, 1783088630, 2080089781, 195827466, 1409073281, 867635355, - 3049533211, 486687054, 2570137956, 527522011, 1084454084, 1019222771, 1415565066, - 650794786, 629618803, 1237709131, 1241899078, 2751644247, 2792313337, 649402117, - 275078659, 752459111, 2173220853, 3207031798, 821073585, 3005400729, 1085152012, - 2441935114, 3465447683, 2897229686, 3845380309, 3573898488, 495424232, 2490548037, - 581670528, 2467171733, 1208279791, 2163927790, 3895792830, 2097210789, 1606544633, - 2148733343, 4072525389, 3256142090, 349440478, 3920932491, 657289255, 1075951496, - 2835763703, 1593198055, 2380945625, 149487931, 3182766507, 2927484354, 2877470578, - 4153923603, 606130344, 1168544900, 888955615, 3605412824, 1336677864, 53448770, - 2884036014, 4070749843, 2989661773, 1095584023, 2766144383, 3534389580, 312378113, - 3190819203, 1247574926, 1530609481, 3536918510, 1479030180, 847820646, 3992973956, - 154171325, 4113429617, 3504933502, 295000614, 2238923504, 282708664, 290246351, - 1064210816, 2848539559, 2617134888, 36906646, 2796674561, 3568250500, 2736237915, - 3950756060, 3416260072, 3603540278, 4115393386, 2851621193, 4230341156, 448215287, - 3916344126, 1496013046, 206343742, 2894205125, 2420861244, 2746480417, 3077328661, - 1209440053, 3258293856, 2545287695, 3043332566, 446879604, 587022214, 1614371566, - 958587333, 3686422145, 937325128, 1968833679, 169086151, 154576725, 1196046411, - 3101745581, 4228079966, 2942213563, 2487464668, 1673491641, 1762746534, 3641827252, - 694590905, 3754606623, 513716230, 3106485486, 2441593994, 4044462965, 3064108377, - 3957990629, 179764922, 579361186, 3474393871, 2138270428, 4031850878, 3120409532, - 4011587898, 3682942579, 4015980199, 3097029759, 2652540191, 1128762588, 1040256382, - 3908621649, 334893087, 1892049031, 2603159239, 3712772023, 3291038350, 3465793906, - 3180780589, 725740783, 3728108967, 436976908, 137996587, 110756053, 3984787930, - 3773232816, 1000054791, 1783088630, 2080089781, 195827466, 1409073281, 3036813614, - 3049533211, 486687054, 2570137956, 527522011, 3669951690, 1019222771, 1415565066, - 650794786, 629618803, 4140569538, 1241899078, 2751644247, 2792313337, 649402117, - 2946582304, 752459111, 2173220853, 3207031798, 821073585, 1738142977, 1085152012, - 2441935114, 3465447683, 2897229686, 2707197334, 3573898488, 495424232, 2490548037, - 581670528, 2365865647, 1208279791, 2163927790, 3895792830, 2097210789, 3219551420, - 2148733343, 4072525389, 3256142090, 349440478, 3706519197, 657289255, 1075951496, - 2835763703, 1593198055, 2200084531, 149487931, 3182766507, 2927484354, 2877470578, - 2394288661, 606130344, 1168544900, 888955615, 3605412824, 1503975597, 53448770, - 2884036014, 4070749843, 2989661773, 243605110, 2766144383, 3534389580, 312378113, - 3190819203, 2398088088, 1530609481, 3536918510, 1479030180, 847820646, 2940281320, - 154171325, 4113429617, 3504933502, 295000614, 3078701806, 282708664, 290246351, - 1064210816, 2848539559, 3960345380, 36906646, 2796674561, 3568250500, 2736237915, - 2657034787, 3416260072, 3603540278, 4115393386, 2851621193, 3847740427, 448215287, - 3916344126, 1496013046, 206343742, 3419083433, 2420861244, 2746480417, 3077328661, - 1209440053, 3824237152, 2545287695, 3043332566, 446879604, 587022214, 506352928, - 958587333, 3686422145, 937325128, 1968833679, 1808935939, 154576725, 1196046411, - 3101745581, 4228079966, 709576348, 2487464668, 1673491641, 1762746534, 3641827252, - 3968332142, 3754606623, 513716230, 3106485486, 2441593994, 1453443785, 3064108377, - 3957990629, 179764922, 579361186, 1454621561, 2138270428, 4031850878, 3120409532, - 4011587898, 898119245, 4015980199, 3097029759, 2652540191, 1128762588, 1131456853, - 3908621649, 334893087, 1892049031, 2603159239, 4280222837, 3291038350, 3465793906, - 3180780589, 725740783, 1515867399, 436976908, 137996587, 110756053, 3984787930, - 1295994548, 1000054791, 1783088630, 2080089781, 195827466, 252558267, 3036813614, - 3049533211, 486687054, 2570137956, 786434419, 3669951690, 1019222771, 1415565066, - 650794786, 1316734597, 4140569538, 1241899078, 2751644247, 2792313337, 4014748337, - 2946582304, 752459111, 2173220853, 3207031798, 2903407363, 1738142977, 1085152012, - 2441935114, 3465447683, 1082984764, 2707197334, 3573898488, 495424232, 2490548037, - 240094068, 2365865647, 1208279791, 2163927790, 3895792830, 1107651215, 3219551420, - 2148733343, 4072525389, 3256142090, 681942656, 3706519197, 657289255, 1075951496, - 2835763703, 2172774506, 2200084531, 149487931, 3182766507, 2927484354, 3069592433, - 2394288661, 606130344, 1168544900, 888955615, 757163746, 1503975597, 53448770, - 2884036014, 4070749843, 1705538727, 243605110, 2766144383, 3534389580, 312378113, - 2256467250, 2398088088, 1530609481, 3536918510, 1479030180, 1360826079, 2940281320, - 154171325, 4113429617, 3504933502, 714934244, 3078701806, 282708664, 290246351, - 1064210816, 3694453051, 3960345380, 36906646, 2796674561, 3568250500, 3400481963, - 2657034787, 3416260072, 3603540278, 4115393386, 1466632735, 3847740427, 448215287, - 3916344126, 1496013046, 2893537514, 3419083433, 2420861244, 2746480417, 3077328661, - 2815979224, 3824237152, 2545287695, 3043332566, 446879604, 3719452721, 506352928, - 958587333, 3686422145, 937325128, 2653904510, 1808935939, 154576725, 1196046411, - 3101745581, 425411544, 709576348, 2487464668, 1673491641, 1762746534, 1960605594, - 3968332142, 3754606623, 513716230, 3106485486, 2881551071, 1453443785, 3064108377, - 3957990629, 179764922, 1408218536, 1454621561, 2138270428, 4031850878, 3120409532, - 3700386494, 898119245, 4015980199, 3097029759, 2652540191, 2181464767, 1131456853, - 3908621649, 334893087, 1892049031, 4220220071, 4280222837, 3291038350, 3465793906, - 3180780589, 1737123182, 1515867399, 436976908, 137996587, 110756053, 1360813614, - 1295994548, 1000054791, 1783088630, 2080089781, 1019367341, 252558267, 3036813614, - 3049533211, 486687054, 387915679, 786434419, 3669951690, 1019222771, 1415565066, - 4267042909, 1316734597, 4140569538, 1241899078, 2751644247, 3622120385, 4014748337, - 2946582304, 752459111, 2173220853, 1128460687, 2268047031, 239933818, 4141570430, - 1318816940, 2378987660, 731877825, 3950952879, 2975574698, 2938375136, 431933385, - 154404673, 2020658234, 846815781, 822137193, 1057315444, 3632584082, 3263363094, - 942201956, 2704683551, 1768107067, 4009446092, 3090701064, 701246680, 3548419575, - 3873366129, 1639833080, 2401253373, 66597794, 2515774132, 516246524, 4232115668, - 34426096, 2206423458, 3628832867, 2776950121, 2782943544, 2058958317, 1805852726, - 2151415233, 2940074103, 2318397273, 3067676663, 3127709351, 71509976, 115529187, - 1841252918, 2217805156, 733917373, 2432474677, 1416887641, 1895320369, 2779694586, - 510547269, 2614743018, 759552691, 2264773752, 305497497, 1082013785, 1681067734, - 1085957001, 846460632, 2824079919, 1820633139, 3686495295, 3978521319, 1734452426, - 4105472656, 1771256166, 1578071897, 1972844727, 2048372515, 3002132226, 1889169118, - 2932142799, 2166712623, 592016143, 1116895096, 889321536, 375621825, 2935845994, - 1982459859, 3336799370, 294519309, 2661638345, 1089335942, 227150969, 1454919198, - 3780503305, 1862290968, 1491836299, 766546986, 3638407467, 925906735, 208891816, - 236714698, 2853181150, 3889751556, 2161215392, 853579433, 2131555681, 1396396345, - 1088128136, 978252562, 2134024308, 2429920974, 1159468871, 2395949266, 1441791888, - 916521377, 3950270431, 2663319810, 3873120593, 2080989388, 2896532502, 3176181708, - 1736685126, 4081767288, 3515770288, 1371473598, 1491850178, 4284949727, 2774513541, - 1541596000, 3948112869, 2114538326, 2641532252, 1837244955, 2292505300, 3179787565, - 639953781, 785902378, 3852544833, 553508260, 23014564, 106722100, 2705412979, - 3449440367, 950636401, 870804158, 629831074, 424163855, 373653940, 2739378330, - 377730945, 418426029, 267367218, 554678849, 4222664331, 3346048120, 1870226737, - 2435616108, 3747040233, 698046507, 1671346285, 4127293033, 568612264, 3467142937, - 1627988025, 1305525598, - }, - { - 2246605826, 215030128, 871645668, 3402612852, 423273439, 316965236, 47416561, - 1470716454, 2288582385, 2021890755, 2148091363, 167227868, 3085506034, 3365950545, - 1170282137, 1345986409, 197195155, 2644113318, 2491271090, 2597072003, 170335901, - 2540851884, 2584420407, 3609142920, 3052130502, 4018095157, 2850805299, 2777821400, - 110647395, 3262987676, 1447103309, 3632575579, 3243210595, 1892770504, 4214485953, - 38676169, 2431628817, 2836918800, 272023527, 2825888902, 2794421955, 2354379386, - 452404203, 584718212, 1915053836, 1455821656, 4264066935, 1150980581, 3792433350, - 3104909316, 441521402, 3807587668, 275969953, 3970844623, 3323695518, 3909107329, - 290225599, 957520066, 4048181850, 2623778463, 1957371891, 540091753, 3072448879, - 2386916346, 392549194, 1261391184, 4137605148, 314807135, 2916930821, 3168561018, - 2332027308, 1967082817, 1849256214, 1141134412, 1206824012, 2088102210, 4170914605, - 3399892824, 59190648, 1657183299, 1314626253, 500606287, 413229420, 1245395908, - 664681, 2726979120, 3408998445, 2318397638, 1882820077, 2073055266, 4262833629, - 1348801932, 229857331, 3086071450, 1327801028, 812015573, 2214355282, 2232635690, - 3162540418, 2049877621, 470752564, 2527480795, 1285499716, 220173566, 4239277569, - 788168494, 3748855859, 1360707769, 449512212, 1238219398, 2880205975, 2755133627, - 372409230, 411800575, 2455333195, 4080817864, 3556684908, 2857940866, 1969081563, - 2526852668, 1026062474, 1849785784, 3552290093, 4214448, 460332681, 30890894, - 1108618048, 272438799, 3339891045, 1512685591, 1310038443, 2431938882, 1478442144, - 2804640700, 3426381347, 861206186, 290322827, 2736623609, 327318125, 1922859957, - 1939922519, 3539608908, 3442377433, 3868710131, 2244493875, 47774461, 3858864626, - 3294523981, 1798515481, 565017248, 2633378137, 811307482, 1743357106, 419676111, - 1688841846, 1799884674, 1720546272, 3900863156, 3506303345, 1719438472, 576775454, - 2246605826, 215030128, 871645668, 3402612852, 619000856, 316965236, 47416561, - 1470716454, 2288582385, 3464704266, 2148091363, 167227868, 3085506034, 3365950545, - 901169164, 1345986409, 197195155, 2644113318, 2491271090, 3243741640, 170335901, - 2540851884, 2584420407, 3609142920, 2051834116, 4018095157, 2850805299, 2777821400, - 110647395, 2822981113, 1447103309, 3632575579, 3243210595, 1892770504, 1947501555, - 38676169, 2431628817, 2836918800, 272023527, 4010280501, 2794421955, 2354379386, - 452404203, 584718212, 3991257933, 1455821656, 4264066935, 1150980581, 3792433350, - 2151631692, 441521402, 3807587668, 275969953, 3970844623, 3965914153, 3909107329, - 290225599, 957520066, 4048181850, 4011285909, 1957371891, 540091753, 3072448879, - 2386916346, 1347453316, 1261391184, 4137605148, 314807135, 2916930821, 840822698, - 2332027308, 1967082817, 1849256214, 1141134412, 960593185, 2088102210, 4170914605, - 3399892824, 59190648, 2261593014, 1314626253, 500606287, 413229420, 1245395908, - 3401527918, 2726979120, 3408998445, 2318397638, 1882820077, 1683077666, 4262833629, - 1348801932, 229857331, 3086071450, 3363644507, 812015573, 2214355282, 2232635690, - 3162540418, 3579858747, 470752564, 2527480795, 1285499716, 220173566, 2294101261, - 788168494, 3748855859, 1360707769, 449512212, 28595866, 2880205975, 2755133627, - 372409230, 411800575, 1905311140, 4080817864, 3556684908, 2857940866, 1969081563, - 148561593, 1026062474, 1849785784, 3552290093, 4214448, 2237247821, 30890894, - 1108618048, 272438799, 3339891045, 169576507, 1310038443, 2431938882, 1478442144, - 2804640700, 4119485855, 861206186, 290322827, 2736623609, 327318125, 3408620608, - 1939922519, 3539608908, 3442377433, 3868710131, 1188056275, 47774461, 3858864626, - 3294523981, 1798515481, 1228896851, 2633378137, 811307482, 1743357106, 419676111, - 3111013241, 1799884674, 1720546272, 3900863156, 3506303345, 1474164586, 576775454, - 2246605826, 215030128, 871645668, 2968519387, 619000856, 316965236, 47416561, - 1470716454, 9648980, 3464704266, 2148091363, 167227868, 3085506034, 1505294373, - 901169164, 1345986409, 197195155, 2644113318, 1227359150, 3243741640, 170335901, - 2540851884, 2584420407, 1205921163, 2051834116, 4018095157, 2850805299, 2777821400, - 2967529310, 2822981113, 1447103309, 3632575579, 3243210595, 532996977, 1947501555, - 38676169, 2431628817, 2836918800, 1761031313, 4010280501, 2794421955, 2354379386, - 452404203, 1222630846, 3991257933, 1455821656, 4264066935, 1150980581, 2344548386, - 2151631692, 441521402, 3807587668, 275969953, 963889269, 3965914153, 3909107329, - 290225599, 957520066, 4176220201, 4011285909, 1957371891, 540091753, 3072448879, - 1810164615, 1347453316, 1261391184, 4137605148, 314807135, 2672526663, 840822698, - 2332027308, 1967082817, 1849256214, 734862208, 960593185, 2088102210, 4170914605, - 3399892824, 2471507530, 2261593014, 1314626253, 500606287, 413229420, 970185057, - 3401527918, 2726979120, 3408998445, 2318397638, 708987193, 1683077666, 4262833629, - 1348801932, 229857331, 749849397, 3363644507, 812015573, 2214355282, 2232635690, - 2901095495, 3579858747, 470752564, 2527480795, 1285499716, 941862108, 2294101261, - 788168494, 3748855859, 1360707769, 3818227212, 28595866, 2880205975, 2755133627, - 372409230, 570110534, 1905311140, 4080817864, 3556684908, 2857940866, 2253777974, - 148561593, 1026062474, 1849785784, 3552290093, 1525559608, 2237247821, 30890894, - 1108618048, 272438799, 3996203631, 169576507, 1310038443, 2431938882, 1478442144, - 2857841871, 4119485855, 861206186, 290322827, 2736623609, 1184217272, 3408620608, - 1939922519, 3539608908, 3442377433, 1263700272, 1188056275, 47774461, 3858864626, - 3294523981, 2611619, 1228896851, 2633378137, 811307482, 1743357106, 1930089302, - 3111013241, 1799884674, 1720546272, 3900863156, 2370003471, 1474164586, 576775454, - 2246605826, 215030128, 540197019, 2968519387, 619000856, 316965236, 47416561, - 3585128733, 9648980, 3464704266, 2148091363, 167227868, 509283324, 1505294373, - 901169164, 1345986409, 197195155, 3983525470, 1227359150, 3243741640, 170335901, - 2540851884, 2812935262, 1205921163, 2051834116, 4018095157, 2850805299, 2798430304, - 2967529310, 2822981113, 1447103309, 3632575579, 389184524, 532996977, 1947501555, - 38676169, 2431628817, 1055068556, 1761031313, 4010280501, 2794421955, 2354379386, - 965687576, 1222630846, 3991257933, 1455821656, 4264066935, 1551000086, 2344548386, - 2151631692, 441521402, 3807587668, 3701529910, 963889269, 3965914153, 3909107329, - 290225599, 1771599976, 4176220201, 4011285909, 1957371891, 540091753, 1670159873, - 1810164615, 1347453316, 1261391184, 4137605148, 4191698993, 2672526663, 840822698, - 2332027308, 1967082817, 3098515331, 734862208, 960593185, 2088102210, 4170914605, - 2470055060, 2471507530, 2261593014, 1314626253, 500606287, 1100764382, 970185057, - 3401527918, 2726979120, 3408998445, 4100198161, 708987193, 1683077666, 4262833629, - 1348801932, 3744209503, 749849397, 3363644507, 812015573, 2214355282, 3217409412, - 2901095495, 3579858747, 470752564, 2527480795, 552979949, 941862108, 2294101261, - 788168494, 3748855859, 2355231228, 3818227212, 28595866, 2880205975, 2755133627, - 833553378, 570110534, 1905311140, 4080817864, 3556684908, 4124102038, 2253777974, - 148561593, 1026062474, 1849785784, 656329297, 1525559608, 2237247821, 30890894, - 1108618048, 1464443032, 3996203631, 169576507, 1310038443, 2431938882, 2100788071, - 2857841871, 4119485855, 861206186, 290322827, 3653047356, 1184217272, 3408620608, - 1939922519, 3539608908, 4267170500, 1263700272, 1188056275, 47774461, 3858864626, - 1046565728, 2611619, 1228896851, 2633378137, 811307482, 1312393456, 1930089302, - 3111013241, 1799884674, 1720546272, 1199041144, 2406753856, 2108495166, 2126345981, - 1524975128, 1269232392, 3162531748, 3076707658, 1736955170, 1036221745, 1232435193, - 3945348482, 1057631163, 520376289, 4154435769, 1280565077, 1865705876, 1030078366, - 1140849319, 1769263412, 1161866807, 2768552980, 561022685, 2712685799, 1501252058, - 3608433719, 3138564149, 4093654128, 1218455911, 892700607, 2012017510, 3568315757, - 4002239824, 1754440379, 2641708101, 1027390781, 199831087, 1261208885, 2058433786, - 2101649235, 220966013, 3445375335, 1100438514, 4075559840, 4244062658, 3417249884, - 150102478, 3337395219, 2464869101, 3720375949, 93353579, 2329780067, 777826834, - 2745626035, 2984812746, 568848158, 1593919595, 1166619196, 96177504, 305329591, - 4271176854, 3829149188, 1551058535, 2828280993, 1367551996, 4208083082, 2260803683, - 3118708147, 434935608, 702805370, 3544156958, 792712531, 231019757, 136272259, - 4049968615, 2722527811, 603697698, 2891035509, 4270409302, 1220615076, 1932569338, - 1084454986, 468729683, 2377913518, 2068946556, 530579176, 1422294615, 4032799503, - 2065706770, 604700228, 98049660, 3182511353, 935830212, 1938107848, 1266035034, - 957505506, 2758220503, 1805223938, 3393041584, 3958541336, 2695487012, 3355668819, - 276889675, 3098939423, 415941187, 180737121, 2638873657, 1103150707, 4255168358, - 2736183195, 1275942292, 2687807236, 538129710, 3337005391, 3941968393, 1113153386, - 3813628384, 1775835369, 296314749, 1697642748, 3614403315, 1953056095, 2102878063, - 3161706344, 2207159580, 3078233525, 3836286614, 886914072, 1884037075, 4135819784, - 1616380780, 1672616998, 3879848699, 2277472209, 3933249848, 2428044648, 2876076879, - 165724720, 2277165385, 1984963196, 1456923194, 2406217222, 3388886718, 47522558, - 1903557801, 1959641458, 2325355446, 3251147398, 2266553941, 2243962024, 1420017618, - 1791159474, 1793406225, 601509698, 3207357979, 1189285184, 148538800, 2077251302, - 3267239327, 2851475997, - }, - { - 2628162153, 3861478870, 2769884494, 3423483820, 1118276924, 536776894, 3742490940, - 550084334, 2441329856, 2604618499, 2308745810, 1178166365, 1345165241, 4039508109, - 1246601384, 3843182157, 2200144237, 91750284, 4290064840, 3363597477, 3243492274, - 4271100308, 4186328336, 2291901989, 1834723222, 372220743, 2190417067, 2624886324, - 3567647862, 1591175369, 2278087682, 2461678432, 232820452, 2714694382, 3070258434, - 2412655444, 2667664607, 249083056, 4166379751, 1360927521, 2247816079, 3253689753, - 1563674427, 1914999382, 2101454952, 1067816947, 1098201917, 4054175236, 1805828534, - 1815913104, 738357340, 2597170030, 1689737432, 2004663483, 1160995461, 1008175050, - 2004702919, 4258654415, 938972594, 2121583885, 2208729114, 276726877, 3973538591, - 2991069145, 2345655326, 2980162173, 1915611444, 2332104940, 2382102873, 2324437093, - 2640563452, 2680619359, 3413490949, 2140843463, 2424016743, 3735508133, 3421831326, - 4037977349, 3721506282, 510431975, 1014707294, 1378686477, 1939678832, 2223101760, - 2067687989, 309274614, 276596103, 3757624719, 1212251468, 2649271847, 4140361758, - 2634738350, 2029358730, 3205861896, 3090549771, 3775019657, 2018542036, 3675805680, - 3946144023, 331655838, 326568491, 1867863527, 1550945400, 3087000670, 2342003578, - 3949479453, 586483056, 147951307, 503062740, 3823927166, 2789767841, 3121654578, - 634238762, 4084629478, 3878778788, 435990088, 1724770389, 1403031256, 1334135626, - 1096780503, 3288769545, 2793293893, 80675548, 1637232257, 1856565474, 2675485635, - 1961165681, 1647512786, 4190102851, 4081320784, 2853183400, 3812341867, 278236392, - 1700614299, 2765246084, 3846866009, 1220806787, 3655684157, 1133921183, 2779125219, - 523552281, 703813725, 3110126767, 823843890, 290243102, 821297176, 364959993, - 3381862130, 2305271841, 356059263, 2558018765, 3235968999, 1070598970, 2444411636, - 3636221117, 4275517214, 4035198865, 3339014315, 2911872812, 4049586122, 4211583637, - 2628162153, 3861478870, 2769884494, 3423483820, 3254616321, 536776894, 3742490940, - 550084334, 2441329856, 1909596092, 2308745810, 1178166365, 1345165241, 4039508109, - 1349347043, 3843182157, 2200144237, 91750284, 4290064840, 803098068, 3243492274, - 4271100308, 4186328336, 2291901989, 2575673198, 372220743, 2190417067, 2624886324, - 3567647862, 132569424, 2278087682, 2461678432, 232820452, 2714694382, 3490648253, - 2412655444, 2667664607, 249083056, 4166379751, 3503294711, 2247816079, 3253689753, - 1563674427, 1914999382, 3121933565, 1067816947, 1098201917, 4054175236, 1805828534, - 816420552, 738357340, 2597170030, 1689737432, 2004663483, 397934907, 1008175050, - 2004702919, 4258654415, 938972594, 156733019, 2208729114, 276726877, 3973538591, - 2991069145, 2470446383, 2980162173, 1915611444, 2332104940, 2382102873, 3265195583, - 2640563452, 2680619359, 3413490949, 2140843463, 142464483, 3735508133, 3421831326, - 4037977349, 3721506282, 1898668265, 1014707294, 1378686477, 1939678832, 2223101760, - 4085776926, 309274614, 276596103, 3757624719, 1212251468, 1116423339, 4140361758, - 2634738350, 2029358730, 3205861896, 880658361, 3775019657, 2018542036, 3675805680, - 3946144023, 839516623, 326568491, 1867863527, 1550945400, 3087000670, 420309880, - 3949479453, 586483056, 147951307, 503062740, 416618471, 2789767841, 3121654578, - 634238762, 4084629478, 1120413065, 435990088, 1724770389, 1403031256, 1334135626, - 240966420, 3288769545, 2793293893, 80675548, 1637232257, 1785064235, 2675485635, - 1961165681, 1647512786, 4190102851, 2775407492, 2853183400, 3812341867, 278236392, - 1700614299, 2439624528, 3846866009, 1220806787, 3655684157, 1133921183, 366933679, - 523552281, 703813725, 3110126767, 823843890, 132468066, 821297176, 364959993, - 3381862130, 2305271841, 1048450041, 2558018765, 3235968999, 1070598970, 2444411636, - 1699430013, 4275517214, 4035198865, 3339014315, 2911872812, 324524850, 4211583637, - 2628162153, 3861478870, 2769884494, 1995585079, 3254616321, 536776894, 3742490940, - 550084334, 2121458511, 1909596092, 2308745810, 1178166365, 1345165241, 3067877274, - 1349347043, 3843182157, 2200144237, 91750284, 1246148630, 803098068, 3243492274, - 4271100308, 4186328336, 2932236493, 2575673198, 372220743, 2190417067, 2624886324, - 3945294599, 132569424, 2278087682, 2461678432, 232820452, 3341915918, 3490648253, - 2412655444, 2667664607, 249083056, 2307336284, 3503294711, 2247816079, 3253689753, - 1563674427, 1717494311, 3121933565, 1067816947, 1098201917, 4054175236, 971917867, - 816420552, 738357340, 2597170030, 1689737432, 243915062, 397934907, 1008175050, - 2004702919, 4258654415, 1807067458, 156733019, 2208729114, 276726877, 3973538591, - 1909483753, 2470446383, 2980162173, 1915611444, 2332104940, 3454651559, 3265195583, - 2640563452, 2680619359, 3413490949, 462852932, 142464483, 3735508133, 3421831326, - 4037977349, 1372088341, 1898668265, 1014707294, 1378686477, 1939678832, 752503486, - 4085776926, 309274614, 276596103, 3757624719, 4193030119, 1116423339, 4140361758, - 2634738350, 2029358730, 1725105892, 880658361, 3775019657, 2018542036, 3675805680, - 3496508290, 839516623, 326568491, 1867863527, 1550945400, 2685835387, 420309880, - 3949479453, 586483056, 147951307, 1639139280, 416618471, 2789767841, 3121654578, - 634238762, 3622035469, 1120413065, 435990088, 1724770389, 1403031256, 3548817929, - 240966420, 3288769545, 2793293893, 80675548, 3119506726, 1785064235, 2675485635, - 1961165681, 1647512786, 4019542081, 2775407492, 2853183400, 3812341867, 278236392, - 3487875111, 2439624528, 3846866009, 1220806787, 3655684157, 3303554633, 366933679, - 523552281, 703813725, 3110126767, 2477354049, 132468066, 821297176, 364959993, - 3381862130, 4065162466, 1048450041, 2558018765, 3235968999, 1070598970, 191819556, - 1699430013, 4275517214, 4035198865, 3339014315, 3588518026, 324524850, 4211583637, - 2628162153, 3861478870, 3361198093, 1995585079, 3254616321, 536776894, 3742490940, - 3912424229, 2121458511, 1909596092, 2308745810, 1178166365, 1882174246, 3067877274, - 1349347043, 3843182157, 2200144237, 1210030640, 1246148630, 803098068, 3243492274, - 4271100308, 402141998, 2932236493, 2575673198, 372220743, 2190417067, 1883679642, - 3945294599, 132569424, 2278087682, 2461678432, 708189294, 3341915918, 3490648253, - 2412655444, 2667664607, 2871800434, 2307336284, 3503294711, 2247816079, 3253689753, - 2113837945, 1717494311, 3121933565, 1067816947, 1098201917, 1041869160, 971917867, - 816420552, 738357340, 2597170030, 2306273930, 243915062, 397934907, 1008175050, - 2004702919, 2345434637, 1807067458, 156733019, 2208729114, 276726877, 2452083872, - 1909483753, 2470446383, 2980162173, 1915611444, 2043489400, 3454651559, 3265195583, - 2640563452, 2680619359, 2845757473, 462852932, 142464483, 3735508133, 3421831326, - 25103542, 1372088341, 1898668265, 1014707294, 1378686477, 2680788341, 752503486, - 4085776926, 309274614, 276596103, 3663266970, 4193030119, 1116423339, 4140361758, - 2634738350, 453005903, 1725105892, 880658361, 3775019657, 2018542036, 2601909713, - 3496508290, 839516623, 326568491, 1867863527, 3474340574, 2685835387, 420309880, - 3949479453, 586483056, 297934218, 1639139280, 416618471, 2789767841, 3121654578, - 958889718, 3622035469, 1120413065, 435990088, 1724770389, 2589603756, 3548817929, - 240966420, 3288769545, 2793293893, 972899860, 3119506726, 1785064235, 2675485635, - 1961165681, 2576799764, 4019542081, 2775407492, 2853183400, 3812341867, 159345352, - 3487875111, 2439624528, 3846866009, 1220806787, 3367080935, 3303554633, 366933679, - 523552281, 703813725, 1717395617, 2477354049, 132468066, 821297176, 364959993, - 1088290332, 4065162466, 1048450041, 2558018765, 3235968999, 285340039, 191819556, - 1699430013, 4275517214, 4035198865, 3544133220, 285121978, 1175302919, 4101282768, - 513236580, 890655666, 3051849972, 2315486379, 3067287276, 3134806925, 3926373006, - 2502825498, 461387883, 770459119, 3121636621, 1243065093, 1612354797, 659033930, - 621176955, 214256518, 371573588, 1168438671, 1233027650, 1984255965, 659404177, - 1218841419, 1226193512, 4247589702, 334814687, 980422670, 2518384561, 4041002302, - 1203659320, 509643440, 2528499450, 1512213710, 4052651069, 1378025938, 3436277168, - 2797728577, 463383787, 1184681947, 283482187, 2421891582, 3200080903, 373817869, - 452807139, 2002545143, 1068199574, 3390998240, 377559317, 1548403713, 1580741080, - 253591624, 759280679, 2174360733, 1687952097, 1325235423, 3856575909, 652218568, - 4130230594, 3757998028, 1349431618, 2870775414, 229741978, 1900794007, 201310771, - 4075023260, 3390078853, 3572716207, 1959949436, 1000128498, 1636575064, 241058867, - 2075461870, 1819342070, 619233032, 3164328001, 4280892071, 4219074185, 2719764611, - 3827656652, 4062556527, 621515766, 2542375627, 3901998596, 2295087430, 2880672054, - 2940372823, 2318642706, 914614262, 2549699597, 2907475284, 3901259809, 2663167002, - 3775306719, 2212887565, 1271873285, 3673659531, 3856609875, 1195785209, 1204338358, - 2785362544, 2398696803, 3038377816, 4288025143, 262511310, 4151907455, 924716723, - 3298769960, 2065938273, 3277412030, 122636766, 2164055077, 1000638739, 2044933533, - 2935604716, 2772787255, 3727331409, 1315627932, 2610657438, 832931652, 452359900, - 681035792, 3312648046, 1059435047, 1489639114, 3647631796, 417952902, 731020350, - 2847472725, 2779076784, 2674295324, 487600023, 2925909449, 3997011591, 3697231318, - 967300591, 2310856069, 684710043, 811911286, 4174732177, 1010656728, 702780279, - 920081774, 1578296057, 944734808, 2884038169, 2885919611, 2633474915, 2508946673, - 3579216621, 656143887, 426108406, 2166202683, 991797657, 706498590, 561168186, - 1144619335, 3136206425, - }, - { - 3600072515, 651444872, 2348224675, 1684848433, 1913333701, 3413467790, 1567802204, - 2125206188, 2463158656, 2251055204, 4132590383, 3192977084, 3718261822, 3431519430, - 3506690867, 1313208797, 637811069, 12802085, 3456408080, 166617386, 1764224523, - 4016338923, 2225367442, 2461647273, 3137989854, 373730087, 3013524828, 242949418, - 3443491410, 3671816408, 2391000148, 3964107377, 716535366, 1884597979, 3917515811, - 3441985401, 2472173593, 4034695117, 2486526143, 1658764329, 1873516415, 884116165, - 814992460, 1069506245, 3797556389, 838088473, 2279863068, 1002637017, 4174541774, - 644478743, 4138151954, 4030442072, 297710349, 3507828614, 1403493362, 3132267322, - 227377796, 388148240, 2760904473, 352998924, 1603734504, 1528807885, 2283620218, - 737730350, 2761342715, 809367801, 1667936422, 1510238771, 3762862328, 1171532060, - 647580587, 1460988169, 3944640945, 2331043627, 1965076564, 2913596196, 2960957119, - 1316491503, 3086954934, 3471945989, 2485431762, 692294537, 3148362914, 3371415765, - 2990795967, 706771848, 3734467362, 2768750385, 2061275631, 3935582473, 1449841372, - 1239527551, 592595530, 1685341001, 3352323357, 4147988039, 4003871917, 4035869533, - 3022833195, 1266052547, 1429645393, 565106475, 327014810, 348739711, 3262918351, - 915509292, 397356303, 3248246752, 1122821778, 2373765260, 1795464380, 3485315196, - 1731529670, 86888382, 2789587372, 850847993, 1794523220, 577288126, 1996569530, - 909222664, 2601642298, 1469035973, 2727135938, 3467853736, 633292505, 756260381, - 41782389, 226724724, 3633968708, 1695315503, 1846857904, 3185630605, 823108172, - 3609336496, 3422558797, 2865413534, 564221408, 591845835, 2498463433, 3573926554, - 1336639597, 4180084026, 3195588503, 2822864841, 1916459886, 2073158796, 56968669, - 1234765864, 2456093821, 3500058416, 3146725645, 3295822468, 4135196531, 628000231, - 745509757, 4143543278, 1941480444, 3607603517, 2288239329, 1991437813, 4081693775, - 3600072515, 651444872, 2348224675, 1684848433, 3748890341, 3413467790, 1567802204, - 2125206188, 2463158656, 1516568259, 4132590383, 3192977084, 3718261822, 3431519430, - 461466951, 1313208797, 637811069, 12802085, 3456408080, 3444149988, 1764224523, - 4016338923, 2225367442, 2461647273, 2594402002, 373730087, 3013524828, 242949418, - 3443491410, 2740782133, 2391000148, 3964107377, 716535366, 1884597979, 3161911677, - 3441985401, 2472173593, 4034695117, 2486526143, 3623045141, 1873516415, 884116165, - 814992460, 1069506245, 1053106195, 838088473, 2279863068, 1002637017, 4174541774, - 1806935386, 4138151954, 4030442072, 297710349, 3507828614, 2328331779, 3132267322, - 227377796, 388148240, 2760904473, 3654577129, 1603734504, 1528807885, 2283620218, - 737730350, 2134741424, 809367801, 1667936422, 1510238771, 3762862328, 4084104273, - 647580587, 1460988169, 3944640945, 2331043627, 3458437694, 2913596196, 2960957119, - 1316491503, 3086954934, 2404530503, 2485431762, 692294537, 3148362914, 3371415765, - 3697728317, 706771848, 3734467362, 2768750385, 2061275631, 1337146928, 1449841372, - 1239527551, 592595530, 1685341001, 3121493408, 4147988039, 4003871917, 4035869533, - 3022833195, 2709537023, 1429645393, 565106475, 327014810, 348739711, 1278935671, - 915509292, 397356303, 3248246752, 1122821778, 1086107506, 1795464380, 3485315196, - 1731529670, 86888382, 3645735256, 850847993, 1794523220, 577288126, 1996569530, - 1126950, 2601642298, 1469035973, 2727135938, 3467853736, 3668777652, 756260381, - 41782389, 226724724, 3633968708, 738274780, 1846857904, 3185630605, 823108172, - 3609336496, 3371270228, 2865413534, 564221408, 591845835, 2498463433, 4157618574, - 1336639597, 4180084026, 3195588503, 2822864841, 3844986377, 2073158796, 56968669, - 1234765864, 2456093821, 1001761927, 3146725645, 3295822468, 4135196531, 628000231, - 541676954, 4143543278, 1941480444, 3607603517, 2288239329, 1068806322, 4081693775, - 3600072515, 651444872, 2348224675, 47991343, 3748890341, 3413467790, 1567802204, - 2125206188, 2662653600, 1516568259, 4132590383, 3192977084, 3718261822, 2554440323, - 461466951, 1313208797, 637811069, 12802085, 982676468, 3444149988, 1764224523, - 4016338923, 2225367442, 451503008, 2594402002, 373730087, 3013524828, 242949418, - 1086137206, 2740782133, 2391000148, 3964107377, 716535366, 731470002, 3161911677, - 3441985401, 2472173593, 4034695117, 44456710, 3623045141, 1873516415, 884116165, - 814992460, 4004771121, 1053106195, 838088473, 2279863068, 1002637017, 1587145121, - 1806935386, 4138151954, 4030442072, 297710349, 2570695340, 2328331779, 3132267322, - 227377796, 388148240, 3570998746, 3654577129, 1603734504, 1528807885, 2283620218, - 188017185, 2134741424, 809367801, 1667936422, 1510238771, 1503613101, 4084104273, - 647580587, 1460988169, 3944640945, 3301866374, 3458437694, 2913596196, 2960957119, - 1316491503, 2674694926, 2404530503, 2485431762, 692294537, 3148362914, 1645995464, - 3697728317, 706771848, 3734467362, 2768750385, 670964862, 1337146928, 1449841372, - 1239527551, 592595530, 4204421245, 3121493408, 4147988039, 4003871917, 4035869533, - 3652555523, 2709537023, 1429645393, 565106475, 327014810, 2716443687, 1278935671, - 915509292, 397356303, 3248246752, 204830047, 1086107506, 1795464380, 3485315196, - 1731529670, 662578255, 3645735256, 850847993, 1794523220, 577288126, 4237140216, - 1126950, 2601642298, 1469035973, 2727135938, 92392213, 3668777652, 756260381, - 41782389, 226724724, 1123105466, 738274780, 1846857904, 3185630605, 823108172, - 2880110296, 3371270228, 2865413534, 564221408, 591845835, 2356214088, 4157618574, - 1336639597, 4180084026, 3195588503, 4266261353, 3844986377, 2073158796, 56968669, - 1234765864, 3166457679, 1001761927, 3146725645, 3295822468, 4135196531, 496099322, - 541676954, 4143543278, 1941480444, 3607603517, 2578543796, 1068806322, 4081693775, - 3600072515, 651444872, 1131603264, 47991343, 3748890341, 3413467790, 1567802204, - 2823058381, 2662653600, 1516568259, 4132590383, 3192977084, 4247798474, 2554440323, - 461466951, 1313208797, 637811069, 2744898822, 982676468, 3444149988, 1764224523, - 4016338923, 2845667517, 451503008, 2594402002, 373730087, 3013524828, 3442521115, - 1086137206, 2740782133, 2391000148, 3964107377, 4060067791, 731470002, 3161911677, - 3441985401, 2472173593, 4227407417, 44456710, 3623045141, 1873516415, 884116165, - 2550700713, 4004771121, 1053106195, 838088473, 2279863068, 1296332348, 1587145121, - 1806935386, 4138151954, 4030442072, 2552496880, 2570695340, 2328331779, 3132267322, - 227377796, 3887816270, 3570998746, 3654577129, 1603734504, 1528807885, 3365552060, - 188017185, 2134741424, 809367801, 1667936422, 1358744245, 1503613101, 4084104273, - 647580587, 1460988169, 2318828416, 3301866374, 3458437694, 2913596196, 2960957119, - 49464436, 2674694926, 2404530503, 2485431762, 692294537, 1803418945, 1645995464, - 3697728317, 706771848, 3734467362, 2407932841, 670964862, 1337146928, 1449841372, - 1239527551, 1124552917, 4204421245, 3121493408, 4147988039, 4003871917, 3542256025, - 3652555523, 2709537023, 1429645393, 565106475, 2063548817, 2716443687, 1278935671, - 915509292, 397356303, 1049916999, 204830047, 1086107506, 1795464380, 3485315196, - 2183256184, 662578255, 3645735256, 850847993, 1794523220, 2943700388, 4237140216, - 1126950, 2601642298, 1469035973, 535075238, 92392213, 3668777652, 756260381, - 41782389, 1043025574, 1123105466, 738274780, 1846857904, 3185630605, 3324487649, - 2880110296, 3371270228, 2865413534, 564221408, 2528599862, 2356214088, 4157618574, - 1336639597, 4180084026, 592094844, 4266261353, 3844986377, 2073158796, 56968669, - 629503707, 3166457679, 1001761927, 3146725645, 3295822468, 2725304934, 496099322, - 541676954, 4143543278, 1941480444, 3557859116, 31832949, 3805791401, 4056283801, - 242812250, 4072988068, 2316479446, 2260433816, 2211372380, 2039672698, 2947948280, - 4106140026, 342600216, 98745656, 2541799209, 926067404, 2733213159, 3163537903, - 2800370126, 2099121446, 1279545581, 3699822446, 3764095615, 690503808, 3799637505, - 1000641330, 242588257, 3657834529, 824791208, 2529299371, 4081898575, 2120338882, - 1273883107, 1680877886, 1253060582, 1760259553, 2250763915, 31780198, 2511451445, - 3102141340, 861489797, 105854693, 70927387, 2725671050, 688282241, 2622257646, - 3466254816, 1905008219, 2980966436, 2154356718, 1075686806, 1966147415, 2357249256, - 2684600972, 400926709, 523449509, 2891602783, 673425710, 3766475216, 2319843954, - 3471794777, 13838840, 1908374660, 3839606132, 3829795513, 3403561639, 1369780874, - 4276407916, 3217619, 1284482371, 2020138237, 2804427294, 1194369854, 1094800747, - 2119081501, 726494474, 490750173, 1117517565, 3498786968, 2163060528, 696718831, - 2780121254, 1286646297, 1594539045, 411215116, 1407268753, 2759136967, 2179483407, - 2088977769, 2737453188, 2411478102, 3112688013, 4112484868, 429293789, 426390687, - 3158027863, 2601897382, 1546855515, 4258208908, 3691263847, 2394986813, 1986623921, - 2632462203, 3551311099, 3309482741, 2632571927, 1200010240, 554555739, 4119397989, - 622818813, 3116222066, 1801867255, 2738500841, 1452697246, 733457482, 1680421668, - 1035766144, 468847991, 3606474156, 2612692123, 730556693, 859096521, 4005878655, - 1138273887, 2182363629, 2710579590, 3345140092, 2562710857, 3859276724, 2318176233, - 3964665794, 3295219265, 3037789445, 371545704, 3434130670, 3686032092, 19964088, - 340386179, 2147090894, 1446742483, 3083526520, 561888846, 2903328518, 1524465288, - 360120037, 2031515996, 1516035872, 2752848969, 1094251072, 984159948, 369999653, - 864602622, 2402584241, 3028363830, 252580667, 480470405, 3201548259, 2739036185, - 2198549891, 1978812013, - }, - { - 2546657140, 2771792972, 3371698159, 1137313111, 2399264952, 1204642544, 2090179262, - 2948712987, 2908027331, 498636511, 2292804841, 1480836858, 2826016727, 196495965, - 2168559184, 3910150715, 320076735, 3144753899, 3199094529, 1165806050, 728308199, - 2322528104, 2891334400, 561853019, 4161870615, 1348321971, 2461357166, 1216229488, - 1392766290, 3060494848, 3282469664, 1866493654, 2351421557, 4195620347, 1512242723, - 478174598, 1087303780, 471631659, 2599553643, 791527994, 563537164, 1238109907, - 3218421602, 133222502, 4182363220, 305688802, 2666439314, 2408520958, 787389550, - 4226450542, 4107143646, 4103547035, 1840887424, 2686247491, 334267386, 3772035402, - 3436827662, 1411515743, 2193739735, 1892746640, 4163192062, 2921191805, 1011310614, - 2178118214, 33647321, 1121452997, 507942677, 2542792587, 351339975, 1586639416, - 1918003826, 2513357034, 2747854573, 606238275, 1132105249, 574593993, 2655425816, - 1680556547, 1831942411, 2587194016, 90710116, 4291431098, 1899367028, 3251152898, - 3297078396, 2712235924, 1546135008, 897753268, 1619454780, 938130143, 1828916640, - 3620488958, 1822437033, 172584228, 1853048226, 3659288522, 3623450763, 1893292786, - 851522142, 3411705687, 4106341088, 4109830348, 1193339049, 878885723, 2964062476, - 2320209608, 1777678953, 2886897705, 3856938396, 252913914, 3648685154, 544382669, - 2631141468, 1524405364, 1848509666, 580646927, 2451560151, 181916967, 1426301928, - 1652422182, 2625099169, 176664750, 1582626255, 1675120608, 2571617898, 2096572277, - 2471745846, 419906507, 886861124, 1974832558, 3157060904, 216000225, 746978071, - 1424984058, 1457979883, 809822177, 3833178010, 3926414726, 1423462846, 3024443248, - 4067020014, 2881559869, 1376840097, 548130303, 1118013762, 1309103114, 2227304261, - 4205319357, 228947246, 2167410411, 620496852, 2724112116, 705259153, 3499686911, - 3085999115, 2447267299, 4190122199, 1091465954, 1233728238, 39711865, 1076751044, - 2546657140, 2771792972, 3371698159, 1137313111, 3857150586, 1204642544, 2090179262, - 2948712987, 2908027331, 368199414, 2292804841, 1480836858, 2826016727, 196495965, - 3235583934, 3910150715, 320076735, 3144753899, 3199094529, 1374597050, 728308199, - 2322528104, 2891334400, 561853019, 1515915224, 1348321971, 2461357166, 1216229488, - 1392766290, 15252704, 3282469664, 1866493654, 2351421557, 4195620347, 192355609, - 478174598, 1087303780, 471631659, 2599553643, 1725604263, 563537164, 1238109907, - 3218421602, 133222502, 305098282, 305688802, 2666439314, 2408520958, 787389550, - 3195522899, 4107143646, 4103547035, 1840887424, 2686247491, 1565529892, 3772035402, - 3436827662, 1411515743, 2193739735, 1848198417, 4163192062, 2921191805, 1011310614, - 2178118214, 3474206203, 1121452997, 507942677, 2542792587, 351339975, 3599278861, - 1918003826, 2513357034, 2747854573, 606238275, 446979745, 574593993, 2655425816, - 1680556547, 1831942411, 3338512802, 90710116, 4291431098, 1899367028, 3251152898, - 1006512939, 2712235924, 1546135008, 897753268, 1619454780, 1429190743, 1828916640, - 3620488958, 1822437033, 172584228, 2529855020, 3659288522, 3623450763, 1893292786, - 851522142, 1417935793, 4106341088, 4109830348, 1193339049, 878885723, 1886400637, - 2320209608, 1777678953, 2886897705, 3856938396, 1813134786, 3648685154, 544382669, - 2631141468, 1524405364, 687661410, 580646927, 2451560151, 181916967, 1426301928, - 1463347373, 2625099169, 176664750, 1582626255, 1675120608, 3387060344, 2096572277, - 2471745846, 419906507, 886861124, 4209699955, 3157060904, 216000225, 746978071, - 1424984058, 3063941448, 809822177, 3833178010, 3926414726, 1423462846, 750559587, - 4067020014, 2881559869, 1376840097, 548130303, 4056763004, 1309103114, 2227304261, - 4205319357, 228947246, 774411056, 620496852, 2724112116, 705259153, 3499686911, - 2486247387, 2447267299, 4190122199, 1091465954, 1233728238, 54639263, 1076751044, - 2546657140, 2771792972, 3371698159, 1152150303, 3857150586, 1204642544, 2090179262, - 2948712987, 452427847, 368199414, 2292804841, 1480836858, 2826016727, 1929008184, - 3235583934, 3910150715, 320076735, 3144753899, 895636897, 1374597050, 728308199, - 2322528104, 2891334400, 1871824871, 1515915224, 1348321971, 2461357166, 1216229488, - 3170568098, 15252704, 3282469664, 1866493654, 2351421557, 4253216490, 192355609, - 478174598, 1087303780, 471631659, 4230260400, 1725604263, 563537164, 1238109907, - 3218421602, 960481514, 305098282, 305688802, 2666439314, 2408520958, 242741163, - 3195522899, 4107143646, 4103547035, 1840887424, 2768321503, 1565529892, 3772035402, - 3436827662, 1411515743, 545362965, 1848198417, 4163192062, 2921191805, 1011310614, - 1196775493, 3474206203, 1121452997, 507942677, 2542792587, 1948892535, 3599278861, - 1918003826, 2513357034, 2747854573, 4172793632, 446979745, 574593993, 2655425816, - 1680556547, 2986869736, 3338512802, 90710116, 4291431098, 1899367028, 3376952160, - 1006512939, 2712235924, 1546135008, 897753268, 2061577225, 1429190743, 1828916640, - 3620488958, 1822437033, 4221327184, 2529855020, 3659288522, 3623450763, 1893292786, - 16446898, 1417935793, 4106341088, 4109830348, 1193339049, 2895194326, 1886400637, - 2320209608, 1777678953, 2886897705, 117861450, 1813134786, 3648685154, 544382669, - 2631141468, 1105253905, 687661410, 580646927, 2451560151, 181916967, 1605087684, - 1463347373, 2625099169, 176664750, 1582626255, 1993431057, 3387060344, 2096572277, - 2471745846, 419906507, 3219719670, 4209699955, 3157060904, 216000225, 746978071, - 3304126047, 3063941448, 809822177, 3833178010, 3926414726, 4061584738, 750559587, - 4067020014, 2881559869, 1376840097, 973425409, 4056763004, 1309103114, 2227304261, - 4205319357, 939664759, 774411056, 620496852, 2724112116, 705259153, 176172666, - 2486247387, 2447267299, 4190122199, 1091465954, 300145620, 54639263, 1076751044, - 2546657140, 2771792972, 188149161, 1152150303, 3857150586, 1204642544, 2090179262, - 626100323, 452427847, 368199414, 2292804841, 1480836858, 2700509669, 1929008184, - 3235583934, 3910150715, 320076735, 1715326239, 895636897, 1374597050, 728308199, - 2322528104, 2356051490, 1871824871, 1515915224, 1348321971, 2461357166, 243332180, - 3170568098, 15252704, 3282469664, 1866493654, 4079212881, 4253216490, 192355609, - 478174598, 1087303780, 3787911270, 4230260400, 1725604263, 563537164, 1238109907, - 1147223471, 960481514, 305098282, 305688802, 2666439314, 1503870433, 242741163, - 3195522899, 4107143646, 4103547035, 4041516761, 2768321503, 1565529892, 3772035402, - 3436827662, 3952861918, 545362965, 1848198417, 4163192062, 2921191805, 793561655, - 1196775493, 3474206203, 1121452997, 507942677, 3788690254, 1948892535, 3599278861, - 1918003826, 2513357034, 3301940062, 4172793632, 446979745, 574593993, 2655425816, - 667233719, 2986869736, 3338512802, 90710116, 4291431098, 2027122085, 3376952160, - 1006512939, 2712235924, 1546135008, 2609276017, 2061577225, 1429190743, 1828916640, - 3620488958, 1603195641, 4221327184, 2529855020, 3659288522, 3623450763, 2313432963, - 16446898, 1417935793, 4106341088, 4109830348, 4106013120, 2895194326, 1886400637, - 2320209608, 1777678953, 1952597964, 117861450, 1813134786, 3648685154, 544382669, - 3108229631, 1105253905, 687661410, 580646927, 2451560151, 1160575897, 1605087684, - 1463347373, 2625099169, 176664750, 1998534134, 1993431057, 3387060344, 2096572277, - 2471745846, 2246406696, 3219719670, 4209699955, 3157060904, 216000225, 902956869, - 3304126047, 3063941448, 809822177, 3833178010, 815366736, 4061584738, 750559587, - 4067020014, 2881559869, 350775477, 973425409, 4056763004, 1309103114, 2227304261, - 2047915817, 939664759, 774411056, 620496852, 2724112116, 3593903529, 176172666, - 2486247387, 2447267299, 4190122199, 1450746791, 1521739409, 272699299, 4113952664, - 1408743622, 4082014187, 2454446462, 1401621236, 2050232096, 4204834821, 2413497685, - 1032465253, 4276089655, 1737267711, 3335718398, 1924071395, 1560525661, 3064183869, - 1775038231, 89761304, 489201378, 1236489133, 2774076159, 822652970, 1583752702, - 1781766972, 2238480533, 3428349870, 3344555477, 2251934941, 2533404243, 3651295253, - 2359372862, 704049384, 3238382362, 2405156187, 2572833624, 531907732, 2240111412, - 4102445586, 849739856, 3649572083, 3317634415, 1141345331, 1118528358, 1664181643, - 648360156, 1364897187, 289264571, 1625825195, 1075970578, 3925373833, 2780782646, - 727038162, 2824687935, 3844230994, 2070739238, 2437298873, 1837327520, 4248571219, - 183041221, 3759390508, 3881974011, 658115161, 560642175, 32860408, 1321227669, - 1380454450, 1676524786, 476585241, 4034481274, 1110506516, 815601591, 2009522227, - 2168306897, 1856639149, 1328281664, 2710915389, 1886116025, 2074502324, 23109943, - 670045122, 2926671795, 4269143768, 2688621201, 1618605914, 1541217762, 4273045819, - 1029546542, 3663663567, 1402692384, 109336276, 2446546057, 2225682064, 3535545430, - 3847123891, 369718877, 3411726117, 703735748, 3139527634, 22388546, 998860697, - 2532911305, 1532808237, 4170332196, 1131906845, 1814343609, 4161931326, 1185668213, - 1903273604, 3466154373, 3988139604, 1079368270, 991305574, 898158502, 2898908951, - 651161128, 1952607949, 1221528540, 29979722, 3006846808, 2911550178, 2569412437, - 1460616937, 2127921978, 3689931108, 950505297, 3469337654, 3180457017, 2316433735, - 1464678429, 2867173456, 391248106, 3622065314, 2143251073, 860219584, 323835636, - 340886643, 1805485977, 109344001, 1537119779, 1795626099, 2568079633, 3048040562, - 1204069532, 2488753091, 2160014198, 3132782711, 1266102795, 91252225, 2018366053, - 39675212, 979320891, 343397131, 814470367, 366655857, 3287033048, 3379301026, - 1566381433, 3431153818, - }, - { - 2234324389, 1682296894, 3526681456, 3988544681, 1315506584, 1754723911, 3607564438, - 3764062195, 3408328234, 2385116969, 3827569659, 4104590721, 2612634189, 1762747544, - 1676800931, 1814546108, 2684685172, 1659194343, 3381624140, 2286640580, 688245437, - 2593335056, 1657668516, 1161309746, 3390664973, 2460564382, 2811435329, 2169200311, - 2768093584, 4288309691, 1341061221, 1361417084, 3060155336, 2526021346, 1037055386, - 890124736, 2185462193, 765141735, 1841745804, 3562499272, 1437907207, 2127475991, - 2845453063, 4007976206, 4160093314, 2717704308, 4193767498, 1667876711, 3477753188, - 3150367681, 3224086539, 231347764, 2737121599, 1230656103, 4168131490, 1463860373, - 2760968409, 2579133178, 2309591728, 2958907244, 1041094855, 685134804, 3861095208, - 1088109135, 815655228, 2618003265, 3454840568, 1668276240, 1668403077, 663034899, - 4020374281, 1896863688, 677285319, 4047674693, 4098535894, 2038783953, 236635760, - 3641273565, 3568356824, 3405704765, 186484522, 3626346451, 3653227559, 281949942, - 1847600066, 4168753288, 1723123703, 3600798445, 4267802363, 2947454105, 468768748, - 2745777741, 26635454, 837186232, 206931043, 2601865569, 2021732453, 3171165636, - 786833002, 116631308, 1604778670, 437644814, 2437761489, 3573139998, 2637030522, - 972076738, 4075927397, 1427554739, 597414077, 559325169, 1774857312, 224593737, - 3697511293, 3905126277, 2446278950, 1847061846, 333176687, 2988562696, 3623938567, - 2389910304, 4273100167, 1673622334, 2163644598, 3666601063, 3971760462, 4176957983, - 565952761, 566996714, 103136762, 3648349163, 115456167, 3265051494, 2826313040, - 1898888678, 3921049266, 1276809956, 4051866478, 959265349, 851980436, 3105565302, - 2905096898, 342438530, 3428101638, 912389587, 2306839396, 3613297213, 200159550, - 3406974927, 832121231, 2998593393, 1242069873, 1464281204, 1828082526, 2620095350, - 3727900009, 986958825, 3332332947, 1610600284, 3193282615, 1873987353, 537698841, - 2234324389, 1682296894, 3526681456, 3988544681, 1112334635, 1754723911, 3607564438, - 3764062195, 3408328234, 2702680798, 3827569659, 4104590721, 2612634189, 1762747544, - 1596420149, 1814546108, 2684685172, 1659194343, 3381624140, 2424233156, 688245437, - 2593335056, 1657668516, 1161309746, 260803614, 2460564382, 2811435329, 2169200311, - 2768093584, 1426048416, 1341061221, 1361417084, 3060155336, 2526021346, 688976997, - 890124736, 2185462193, 765141735, 1841745804, 1113361455, 1437907207, 2127475991, - 2845453063, 4007976206, 1719248425, 2717704308, 4193767498, 1667876711, 3477753188, - 449353539, 3224086539, 231347764, 2737121599, 1230656103, 2122699205, 1463860373, - 2760968409, 2579133178, 2309591728, 4017154219, 1041094855, 685134804, 3861095208, - 1088109135, 3954527144, 2618003265, 3454840568, 1668276240, 1668403077, 3235241899, - 4020374281, 1896863688, 677285319, 4047674693, 4043186819, 2038783953, 236635760, - 3641273565, 3568356824, 3946220303, 186484522, 3626346451, 3653227559, 281949942, - 1896524045, 4168753288, 1723123703, 3600798445, 4267802363, 412498526, 468768748, - 2745777741, 26635454, 837186232, 1473941762, 2601865569, 2021732453, 3171165636, - 786833002, 3461566768, 1604778670, 437644814, 2437761489, 3573139998, 306196591, - 972076738, 4075927397, 1427554739, 597414077, 2401305323, 1774857312, 224593737, - 3697511293, 3905126277, 1527832817, 1847061846, 333176687, 2988562696, 3623938567, - 2731158470, 4273100167, 1673622334, 2163644598, 3666601063, 1991088422, 4176957983, - 565952761, 566996714, 103136762, 1639884175, 115456167, 3265051494, 2826313040, - 1898888678, 2976556877, 1276809956, 4051866478, 959265349, 851980436, 2482970929, - 2905096898, 342438530, 3428101638, 912389587, 2716490551, 3613297213, 200159550, - 3406974927, 832121231, 2865829307, 1242069873, 1464281204, 1828082526, 2620095350, - 3671861666, 986958825, 3332332947, 1610600284, 3193282615, 164496953, 537698841, - 2234324389, 1682296894, 3526681456, 486931321, 1112334635, 1754723911, 3607564438, - 3764062195, 898439171, 2702680798, 3827569659, 4104590721, 2612634189, 1703436382, - 1596420149, 1814546108, 2684685172, 1659194343, 3421607784, 2424233156, 688245437, - 2593335056, 1657668516, 362342820, 260803614, 2460564382, 2811435329, 2169200311, - 4248717010, 1426048416, 1341061221, 1361417084, 3060155336, 2693026827, 688976997, - 890124736, 2185462193, 765141735, 2445632748, 1113361455, 1437907207, 2127475991, - 2845453063, 1830953748, 1719248425, 2717704308, 4193767498, 1667876711, 2469362144, - 449353539, 3224086539, 231347764, 2737121599, 2917779591, 2122699205, 1463860373, - 2760968409, 2579133178, 2600345316, 4017154219, 1041094855, 685134804, 3861095208, - 3682591427, 3954527144, 2618003265, 3454840568, 1668276240, 988400088, 3235241899, - 4020374281, 1896863688, 677285319, 2749516227, 4043186819, 2038783953, 236635760, - 3641273565, 4073317913, 3946220303, 186484522, 3626346451, 3653227559, 872336642, - 1896524045, 4168753288, 1723123703, 3600798445, 524095357, 412498526, 468768748, - 2745777741, 26635454, 840544541, 1473941762, 2601865569, 2021732453, 3171165636, - 1058640324, 3461566768, 1604778670, 437644814, 2437761489, 3615438045, 306196591, - 972076738, 4075927397, 1427554739, 2369367008, 2401305323, 1774857312, 224593737, - 3697511293, 4186564433, 1527832817, 1847061846, 333176687, 2988562696, 4039340326, - 2731158470, 4273100167, 1673622334, 2163644598, 307949376, 1991088422, 4176957983, - 565952761, 566996714, 4159448552, 1639884175, 115456167, 3265051494, 2826313040, - 2698725478, 2976556877, 1276809956, 4051866478, 959265349, 293029699, 2482970929, - 2905096898, 342438530, 3428101638, 4172766741, 2716490551, 3613297213, 200159550, - 3406974927, 3723281866, 2865829307, 1242069873, 1464281204, 1828082526, 3304191156, - 3671861666, 986958825, 3332332947, 1610600284, 2370407607, 164496953, 537698841, - 2234324389, 1682296894, 826891606, 486931321, 1112334635, 1754723911, 3607564438, - 3598993552, 898439171, 2702680798, 3827569659, 4104590721, 1421852097, 1703436382, - 1596420149, 1814546108, 2684685172, 4090587429, 3421607784, 2424233156, 688245437, - 2593335056, 4151905751, 362342820, 260803614, 2460564382, 2811435329, 2402832015, - 4248717010, 1426048416, 1341061221, 1361417084, 1629089021, 2693026827, 688976997, - 890124736, 2185462193, 303105066, 2445632748, 1113361455, 1437907207, 2127475991, - 62024604, 1830953748, 1719248425, 2717704308, 4193767498, 667433630, 2469362144, - 449353539, 3224086539, 231347764, 3918249451, 2917779591, 2122699205, 1463860373, - 2760968409, 4274016442, 2600345316, 4017154219, 1041094855, 685134804, 643006688, - 3682591427, 3954527144, 2618003265, 3454840568, 4180665518, 988400088, 3235241899, - 4020374281, 1896863688, 3678687414, 2749516227, 4043186819, 2038783953, 236635760, - 2880089648, 4073317913, 3946220303, 186484522, 3626346451, 2454620114, 872336642, - 1896524045, 4168753288, 1723123703, 2692406059, 524095357, 412498526, 468768748, - 2745777741, 918726515, 840544541, 1473941762, 2601865569, 2021732453, 3534238020, - 1058640324, 3461566768, 1604778670, 437644814, 2894699005, 3615438045, 306196591, - 972076738, 4075927397, 3468671461, 2369367008, 2401305323, 1774857312, 224593737, - 2734827022, 4186564433, 1527832817, 1847061846, 333176687, 2437714719, 4039340326, - 2731158470, 4273100167, 1673622334, 196072958, 307949376, 1991088422, 4176957983, - 565952761, 847200194, 4159448552, 1639884175, 115456167, 3265051494, 2503079777, - 2698725478, 2976556877, 1276809956, 4051866478, 2731665893, 293029699, 2482970929, - 2905096898, 342438530, 581060953, 4172766741, 2716490551, 3613297213, 200159550, - 4222335623, 3723281866, 2865829307, 1242069873, 1464281204, 1080647953, 3304191156, - 3671861666, 986958825, 3332332947, 920422540, 3656094274, 4036161427, 2157099981, - 1855437762, 1385781426, 199192882, 489599802, 3472601685, 717544078, 2241742884, - 3951326913, 3590866192, 1087524220, 3517385549, 360484251, 2718513148, 1386577185, - 1833613127, 2926418589, 1652463225, 548895720, 1343026759, 1797789098, 3229783023, - 1745843414, 200554865, 2442780740, 2359926428, 2970332116, 3097392757, 134294482, - 936225458, 1968264650, 64868134, 3821668262, 2502175363, 1623767635, 2936073062, - 1991791011, 2971174068, 3142195911, 2874818345, 2192526584, 496586185, 2491564144, - 2415210641, 314307270, 2936737494, 557604388, 1067914024, 3270690738, 375601880, - 962749065, 3610467620, 402112984, 1432929499, 3872957776, 3971384069, 2223968592, - 407083609, 2178236674, 1806303230, 3397564470, 12158764, 415570813, 4033667395, - 3687406137, 801878150, 953500350, 3667783172, 1203668106, 902418194, 779786150, - 774683730, 2870261992, 509192460, 1961621392, 1064906432, 3665710891, 1733725153, - 1887608856, 1314631523, 4097239005, 29074501, 3472521950, 4040841657, 532128023, - 2333441401, 1671717886, 1678544416, 1218347584, 3680929567, 4025753853, 2810948711, - 1846100306, 3377469279, 3144481747, 2625781306, 730632118, 3162408393, 3423660386, - 1364968369, 4270900402, 1075484840, 2892932277, 3700635052, 3853022563, 281755151, - 1530909868, 2364069707, 2361723426, 738500028, 1401903990, 1543704261, 2442916222, - 1076190609, 1882477803, 740024557, 1591015439, 2730909167, 2723330839, 1637373491, - 3777799860, 2921269571, 3698591972, 3997463570, 3877862147, 1912888417, 3365137165, - 3465700492, 771243134, 4037723169, 1715894739, 1025821874, 1924958945, 3382242859, - 121591031, 483980724, 546523388, 2446882279, 856267778, 578739009, 2978085488, - 480884914, 966764808, 457039953, 3817520708, 1113646451, 2503896910, 3507840816, - 717151671, 4149352573, 1568869830, 395015863, 773165995, 1853682362, 2861368846, - 1884368812, 1250092101, - }, - { - 916910638, 961623451, 1193013401, 1016438484, 4091279871, 287282633, 8590725, - 3575333670, 324340905, 3133751747, 2840894649, 2980503178, 1111215768, 2783846375, - 72516413, 4158424384, 2184094569, 2305724254, 4057093054, 1407652993, 3105191537, - 768505376, 298782270, 993926164, 2694730042, 1479658113, 2376490281, 2767906402, - 1619969256, 3256472015, 2563843533, 2974784738, 2529307107, 4289918826, 3105587575, - 3748950898, 2182744253, 431888679, 3780324902, 2525978209, 54545903, 1688749940, - 2394884334, 3477656171, 263834270, 1562965459, 804704330, 4185729868, 138898835, - 2113063150, 327612841, 1252226275, 935318076, 2956823075, 4095101181, 1510586062, - 156282440, 3386839706, 2294393752, 1306167091, 4005033667, 651716500, 4115192738, - 123027719, 3873547487, 2910637335, 2571924586, 3489608656, 956791985, 2467423726, - 3214531645, 2054232851, 49634692, 377192215, 1865068750, 2479252980, 3481787748, - 3243507737, 605491073, 4062466752, 988602517, 1539348794, 1555068617, 2657884010, - 460334294, 4240766479, 3639800790, 253377117, 3969136265, 488705329, 1722560286, - 2289159295, 1025876008, 2927117896, 767521707, 2047999999, 4260853571, 2079302241, - 2409677301, 1087552976, 2363907365, 2574464321, 2606273241, 3716086457, 26053603, - 3162779415, 14843078, 2614076143, 1157531920, 2773275636, 2338825066, 435472225, - 1399711137, 1224374788, 2154533280, 560135209, 935800607, 1940258814, 3826959530, - 3423217355, 3704934971, 3815248829, 3878175339, 1395508015, 3295101527, 177901558, - 4167531389, 1375148189, 3125377631, 557218961, 4088880299, 3478859071, 3687276754, - 2845114223, 1713171361, 1756507633, 3160807894, 2375334470, 843542578, 1907952570, - 1544844563, 2294372007, 3336681376, 734347193, 102566945, 2311037104, 4294750194, - 3572240326, 732958152, 263733314, 2087890678, 331542297, 3549110380, 2073894939, - 2104101380, 3670791368, 3122901693, 3799823891, 3783548253, 1102633864, 44327348, - 916910638, 961623451, 1193013401, 1016438484, 1873779640, 287282633, 8590725, - 3575333670, 324340905, 1144671533, 2840894649, 2980503178, 1111215768, 2783846375, - 2000673937, 4158424384, 2184094569, 2305724254, 4057093054, 533488413, 3105191537, - 768505376, 298782270, 993926164, 2015456740, 1479658113, 2376490281, 2767906402, - 1619969256, 3120736988, 2563843533, 2974784738, 2529307107, 4289918826, 729503771, - 3748950898, 2182744253, 431888679, 3780324902, 373638396, 54545903, 1688749940, - 2394884334, 3477656171, 1083764681, 1562965459, 804704330, 4185729868, 138898835, - 823405282, 327612841, 1252226275, 935318076, 2956823075, 899234846, 1510586062, - 156282440, 3386839706, 2294393752, 2769934879, 4005033667, 651716500, 4115192738, - 123027719, 3729538641, 2910637335, 2571924586, 3489608656, 956791985, 139360134, - 3214531645, 2054232851, 49634692, 377192215, 2754746969, 2479252980, 3481787748, - 3243507737, 605491073, 732155706, 988602517, 1539348794, 1555068617, 2657884010, - 3753733088, 4240766479, 3639800790, 253377117, 3969136265, 3848735787, 1722560286, - 2289159295, 1025876008, 2927117896, 3661948694, 2047999999, 4260853571, 2079302241, - 2409677301, 3421911122, 2363907365, 2574464321, 2606273241, 3716086457, 2064343322, - 3162779415, 14843078, 2614076143, 1157531920, 826449637, 2338825066, 435472225, - 1399711137, 1224374788, 3770340198, 560135209, 935800607, 1940258814, 3826959530, - 2963586762, 3704934971, 3815248829, 3878175339, 1395508015, 3721612680, 177901558, - 4167531389, 1375148189, 3125377631, 1023552290, 4088880299, 3478859071, 3687276754, - 2845114223, 3831557301, 1756507633, 3160807894, 2375334470, 843542578, 2798365898, - 1544844563, 2294372007, 3336681376, 734347193, 1856808621, 2311037104, 4294750194, - 3572240326, 732958152, 1999195012, 2087890678, 331542297, 3549110380, 2073894939, - 3115936764, 3670791368, 3122901693, 3799823891, 3783548253, 132796150, 44327348, - 916910638, 961623451, 1193013401, 1753944196, 1873779640, 287282633, 8590725, - 3575333670, 1447720209, 1144671533, 2840894649, 2980503178, 1111215768, 1211945983, - 2000673937, 4158424384, 2184094569, 2305724254, 402617261, 533488413, 3105191537, - 768505376, 298782270, 2915553159, 2015456740, 1479658113, 2376490281, 2767906402, - 3473761811, 3120736988, 2563843533, 2974784738, 2529307107, 737859212, 729503771, - 3748950898, 2182744253, 431888679, 2013420163, 373638396, 54545903, 1688749940, - 2394884334, 675998523, 1083764681, 1562965459, 804704330, 4185729868, 1165431355, - 823405282, 327612841, 1252226275, 935318076, 2420680216, 899234846, 1510586062, - 156282440, 3386839706, 2101339651, 2769934879, 4005033667, 651716500, 4115192738, - 112049740, 3729538641, 2910637335, 2571924586, 3489608656, 305695595, 139360134, - 3214531645, 2054232851, 49634692, 1073828255, 2754746969, 2479252980, 3481787748, - 3243507737, 3392719169, 732155706, 988602517, 1539348794, 1555068617, 3246776527, - 3753733088, 4240766479, 3639800790, 253377117, 872273450, 3848735787, 1722560286, - 2289159295, 1025876008, 4168154213, 3661948694, 2047999999, 4260853571, 2079302241, - 2380420842, 3421911122, 2363907365, 2574464321, 2606273241, 3881916078, 2064343322, - 3162779415, 14843078, 2614076143, 473288515, 826449637, 2338825066, 435472225, - 1399711137, 3068538992, 3770340198, 560135209, 935800607, 1940258814, 1469655183, - 2963586762, 3704934971, 3815248829, 3878175339, 2410602840, 3721612680, 177901558, - 4167531389, 1375148189, 1367577763, 1023552290, 4088880299, 3478859071, 3687276754, - 678224549, 3831557301, 1756507633, 3160807894, 2375334470, 2884561721, 2798365898, - 1544844563, 2294372007, 3336681376, 1938834658, 1856808621, 2311037104, 4294750194, - 3572240326, 2786764913, 1999195012, 2087890678, 331542297, 3549110380, 3597797341, - 3115936764, 3670791368, 3122901693, 3799823891, 1271317799, 132796150, 44327348, - 916910638, 961623451, 2427821332, 1753944196, 1873779640, 287282633, 8590725, - 1244012658, 1447720209, 1144671533, 2840894649, 2980503178, 3548902577, 1211945983, - 2000673937, 4158424384, 2184094569, 2152623453, 402617261, 533488413, 3105191537, - 768505376, 1095141108, 2915553159, 2015456740, 1479658113, 2376490281, 337998873, - 3473761811, 3120736988, 2563843533, 2974784738, 3087228498, 737859212, 729503771, - 3748950898, 2182744253, 2140410733, 2013420163, 373638396, 54545903, 1688749940, - 528290088, 675998523, 1083764681, 1562965459, 804704330, 2536362875, 1165431355, - 823405282, 327612841, 1252226275, 4037635314, 2420680216, 899234846, 1510586062, - 156282440, 2012335895, 2101339651, 2769934879, 4005033667, 651716500, 2552583570, - 112049740, 3729538641, 2910637335, 2571924586, 2436645403, 305695595, 139360134, - 3214531645, 2054232851, 2384286326, 1073828255, 2754746969, 2479252980, 3481787748, - 1948315585, 3392719169, 732155706, 988602517, 1539348794, 4110558494, 3246776527, - 3753733088, 4240766479, 3639800790, 3627363812, 872273450, 3848735787, 1722560286, - 2289159295, 4122430477, 4168154213, 3661948694, 2047999999, 4260853571, 1767882442, - 2380420842, 3421911122, 2363907365, 2574464321, 2778622726, 3881916078, 2064343322, - 3162779415, 14843078, 1513897109, 473288515, 826449637, 2338825066, 435472225, - 322954918, 3068538992, 3770340198, 560135209, 935800607, 345602050, 1469655183, - 2963586762, 3704934971, 3815248829, 3508249920, 2410602840, 3721612680, 177901558, - 4167531389, 2161244150, 1367577763, 1023552290, 4088880299, 3478859071, 1108183104, - 678224549, 3831557301, 1756507633, 3160807894, 2551630811, 2884561721, 2798365898, - 1544844563, 2294372007, 2520267760, 1938834658, 1856808621, 2311037104, 4294750194, - 2310096003, 2786764913, 1999195012, 2087890678, 331542297, 1205238749, 3597797341, - 3115936764, 3670791368, 3122901693, 2008141679, 2018425028, 3435073328, 1452813805, - 1628661138, 1323367156, 1062553693, 4029321700, 2772685842, 3798388850, 1315172209, - 3930983291, 3816791373, 529176017, 3419610188, 3331589216, 4016977274, 2047089790, - 3892571923, 2363414008, 1144631948, 3004954882, 2558739305, 19774033, 2525079911, - 3774885821, 2817837373, 986111566, 1446678953, 3238485630, 3993748600, 1601954599, - 3100591537, 2098009380, 3935971261, 4202546603, 3713465083, 3845664764, 2466365355, - 1452340065, 2003576531, 1013434822, 2254608933, 783902023, 3129770529, 129130612, - 821418228, 350036483, 3473671510, 4128495167, 2773832518, 683262085, 2143353417, - 256251732, 1719056536, 2670223618, 328467339, 1564657740, 451231672, 2788353006, - 882900088, 3255241056, 3198073758, 2541070985, 1941509325, 674933160, 207753676, - 2605303964, 1681335994, 1143520001, 448872632, 302917879, 1100138495, 2058770021, - 3116955098, 2081754747, 3734924767, 1916718058, 3873335960, 2740460398, 2171157007, - 27677949, 2364721928, 175851655, 1468083950, 3162369526, 2441504540, 556978295, - 2372096172, 3181101116, 2582850132, 1101292643, 862643740, 2095546242, 3261953801, - 748040658, 3970037674, 819116843, 3594523650, 1597423019, 4109336883, 1198282420, - 2905230517, 1729529596, 3230132814, 3640242164, 1899059108, 1944906555, 3426510495, - 3035188107, 6448083, 1093882965, 2867500469, 3626379157, 1849073068, 897616501, - 604221668, 1020676159, 4083635798, 1716022041, 3671877965, 1738820843, 30077467, - 729231767, 3413193248, 207000406, 3854363185, 3302747326, 3293643267, 2101250157, - 460131091, 4159442595, 1133391045, 1031215443, 4195487944, 45931575, 2922629291, - 789302543, 3024994662, 442525623, 2850119076, 838309503, 2585361734, 1020449164, - 1623631007, 955374631, 2932467671, 3713639221, 3019179416, 977970472, 1817244230, - 3856774853, 1140530868, 886199600, 1218509766, 4001537244, 2840913665, 2133254364, - 3332344608, 475291624, - }, - { - 1854921599, 2655519695, 3124573588, 319882484, 603545603, 4175512633, 141286453, - 1183670252, 1789500145, 37351733, 3190829323, 2782782009, 493805446, 1228958246, - 2672482554, 2274981421, 2935438833, 3625733677, 3679506394, 687805550, 134516308, - 3576789728, 965007022, 1056542222, 2319405423, 3944221200, 950102624, 3848192810, - 3205299696, 82033760, 1241913280, 1360146137, 1675732327, 2164452797, 3920498715, - 2226452641, 3172047212, 1569171738, 2631589480, 2889660225, 2030783667, 2237381973, - 2706217212, 3143638386, 1733174225, 1166820137, 3818389960, 193959252, 2793509934, - 316291605, 2502743884, 1963136977, 3739017448, 25754513, 1590156485, 1856291967, - 4143674472, 2538785911, 2159135699, 1908446793, 3303325234, 2589568800, 1193586059, - 77481069, 789413194, 2556570543, 162987300, 1960844609, 2973799047, 4253906178, - 315868734, 2542622968, 3949539136, 1479106582, 4225431384, 1235059630, 1533374854, - 847792023, 4031286530, 4194276632, 164541100, 1010135841, 143302319, 1335585015, - 1237311692, 20896020, 344974153, 2576803233, 3430251730, 984163376, 2680612471, - 1276425436, 2400671554, 1628640140, 2161048926, 2109177634, 998215324, 3127793500, - 1759998050, 3105138908, 2583746384, 2126302368, 3258602104, 1262742375, 3565617377, - 3726060195, 157069329, 390662438, 3800994052, 2007694482, 377281730, 3251789121, - 236703173, 122782596, 775407411, 3394010206, 4232159202, 468321553, 2704615220, - 1332411375, 2978494251, 989230484, 3122841814, 2348872707, 731335994, 541354422, - 223117443, 2225009071, 4230058949, 1875162926, 3897048544, 3550177883, 2461273592, - 1046820583, 1333727817, 1378024753, 3686775275, 4230752590, 64834458, 1281467967, - 729116355, 3886390916, 65029451, 3478506446, 1387684482, 1172004841, 2525409243, - 1677678908, 1704646757, 930937262, 1088384271, 689357059, 1754542213, 702963842, - 2864311668, 1960202673, 1009675673, 3742350158, 3751269215, 3166659283, 9090161, - 1854921599, 2655519695, 3124573588, 319882484, 1422536794, 4175512633, 141286453, - 1183670252, 1789500145, 850391877, 3190829323, 2782782009, 493805446, 1228958246, - 837232655, 2274981421, 2935438833, 3625733677, 3679506394, 955772620, 134516308, - 3576789728, 965007022, 1056542222, 874117013, 3944221200, 950102624, 3848192810, - 3205299696, 543679720, 1241913280, 1360146137, 1675732327, 2164452797, 1169030022, - 2226452641, 3172047212, 1569171738, 2631589480, 3783543297, 2030783667, 2237381973, - 2706217212, 3143638386, 1560162209, 1166820137, 3818389960, 193959252, 2793509934, - 4258046618, 2502743884, 1963136977, 3739017448, 25754513, 1204846712, 1856291967, - 4143674472, 2538785911, 2159135699, 3889946075, 3303325234, 2589568800, 1193586059, - 77481069, 969912041, 2556570543, 162987300, 1960844609, 2973799047, 427583517, - 315868734, 2542622968, 3949539136, 1479106582, 92839917, 1235059630, 1533374854, - 847792023, 4031286530, 1147875681, 164541100, 1010135841, 143302319, 1335585015, - 368616909, 20896020, 344974153, 2576803233, 3430251730, 1078575783, 2680612471, - 1276425436, 2400671554, 1628640140, 4149623645, 2109177634, 998215324, 3127793500, - 1759998050, 3525419965, 2583746384, 2126302368, 3258602104, 1262742375, 1996113346, - 3726060195, 157069329, 390662438, 3800994052, 982000497, 377281730, 3251789121, - 236703173, 122782596, 2303768414, 3394010206, 4232159202, 468321553, 2704615220, - 681592492, 2978494251, 989230484, 3122841814, 2348872707, 4089094260, 541354422, - 223117443, 2225009071, 4230058949, 2754981128, 3897048544, 3550177883, 2461273592, - 1046820583, 668143612, 1378024753, 3686775275, 4230752590, 64834458, 3765910650, - 729116355, 3886390916, 65029451, 3478506446, 3419111947, 1172004841, 2525409243, - 1677678908, 1704646757, 155635560, 1088384271, 689357059, 1754542213, 702963842, - 2712009967, 1960202673, 1009675673, 3742350158, 3751269215, 129749802, 9090161, - 1854921599, 2655519695, 3124573588, 809557750, 1422536794, 4175512633, 141286453, - 1183670252, 1739311360, 850391877, 3190829323, 2782782009, 493805446, 1738527771, - 837232655, 2274981421, 2935438833, 3625733677, 1858071296, 955772620, 134516308, - 3576789728, 965007022, 3367712327, 874117013, 3944221200, 950102624, 3848192810, - 2420548306, 543679720, 1241913280, 1360146137, 1675732327, 176019367, 1169030022, - 2226452641, 3172047212, 1569171738, 76544055, 3783543297, 2030783667, 2237381973, - 2706217212, 3283985735, 1560162209, 1166820137, 3818389960, 193959252, 346134252, - 4258046618, 2502743884, 1963136977, 3739017448, 3887005605, 1204846712, 1856291967, - 4143674472, 2538785911, 366578749, 3889946075, 3303325234, 2589568800, 1193586059, - 2917569085, 969912041, 2556570543, 162987300, 1960844609, 61311938, 427583517, - 315868734, 2542622968, 3949539136, 2278526422, 92839917, 1235059630, 1533374854, - 847792023, 1361054176, 1147875681, 164541100, 1010135841, 143302319, 1348709332, - 368616909, 20896020, 344974153, 2576803233, 3290873783, 1078575783, 2680612471, - 1276425436, 2400671554, 628790408, 4149623645, 2109177634, 998215324, 3127793500, - 2019336900, 3525419965, 2583746384, 2126302368, 3258602104, 2858154034, 1996113346, - 3726060195, 157069329, 390662438, 2250549235, 982000497, 377281730, 3251789121, - 236703173, 3487415996, 2303768414, 3394010206, 4232159202, 468321553, 2773608982, - 681592492, 2978494251, 989230484, 3122841814, 3647638215, 4089094260, 541354422, - 223117443, 2225009071, 2829509947, 2754981128, 3897048544, 3550177883, 2461273592, - 282627696, 668143612, 1378024753, 3686775275, 4230752590, 1105868822, 3765910650, - 729116355, 3886390916, 65029451, 328554604, 3419111947, 1172004841, 2525409243, - 1677678908, 1395036942, 155635560, 1088384271, 689357059, 1754542213, 1076601715, - 2712009967, 1960202673, 1009675673, 3742350158, 2581225953, 129749802, 9090161, - 1854921599, 2655519695, 1393282220, 809557750, 1422536794, 4175512633, 141286453, - 2211497169, 1739311360, 850391877, 3190829323, 2782782009, 2694871802, 1738527771, - 837232655, 2274981421, 2935438833, 3145832503, 1858071296, 955772620, 134516308, - 3576789728, 4045354759, 3367712327, 874117013, 3944221200, 950102624, 3562634568, - 2420548306, 543679720, 1241913280, 1360146137, 3644280343, 176019367, 1169030022, - 2226452641, 3172047212, 3927720006, 76544055, 3783543297, 2030783667, 2237381973, - 1497233808, 3283985735, 1560162209, 1166820137, 3818389960, 2344066681, 346134252, - 4258046618, 2502743884, 1963136977, 79988846, 3887005605, 1204846712, 1856291967, - 4143674472, 3967952414, 366578749, 3889946075, 3303325234, 2589568800, 2193179011, - 2917569085, 969912041, 2556570543, 162987300, 52882655, 61311938, 427583517, - 315868734, 2542622968, 1575831590, 2278526422, 92839917, 1235059630, 1533374854, - 2397068791, 1361054176, 1147875681, 164541100, 1010135841, 2586368032, 1348709332, - 368616909, 20896020, 344974153, 3445652232, 3290873783, 1078575783, 2680612471, - 1276425436, 3682156544, 628790408, 4149623645, 2109177634, 998215324, 4049708298, - 2019336900, 3525419965, 2583746384, 2126302368, 1627944270, 2858154034, 1996113346, - 3726060195, 157069329, 1481222640, 2250549235, 982000497, 377281730, 3251789121, - 3564274539, 3487415996, 2303768414, 3394010206, 4232159202, 3509025997, 2773608982, - 681592492, 2978494251, 989230484, 980252048, 3647638215, 4089094260, 541354422, - 223117443, 543970497, 2829509947, 2754981128, 3897048544, 3550177883, 2736782140, - 282627696, 668143612, 1378024753, 3686775275, 2728601425, 1105868822, 3765910650, - 729116355, 3886390916, 1866378660, 328554604, 3419111947, 1172004841, 2525409243, - 1506924008, 1395036942, 155635560, 1088384271, 689357059, 3587092123, 1076601715, - 2712009967, 1960202673, 1009675673, 4292715891, 2465250857, 3267969665, 2459570573, - 3644463083, 1637197500, 684559293, 3520611957, 2976084366, 1512112440, 1778285193, - 1849742417, 3144801412, 3009052859, 820829188, 1382783871, 3373481539, 3777016406, - 266942530, 1792334422, 4109859515, 1468149634, 1356457853, 623893785, 1301686542, - 441704877, 3377795902, 879822753, 329462927, 543858304, 2221828617, 2996486613, - 981774202, 1032220084, 1066536452, 1004068806, 1336694798, 3744375323, 3802436665, - 3366526577, 418696462, 1776559103, 1291965608, 1623030339, 1443628607, 572114324, - 899621592, 332121275, 3637616671, 457287722, 3803043476, 408472701, 660940326, - 1209169008, 1202511620, 2906900959, 2600414642, 2015874468, 2931389161, 1760773669, - 2601299639, 543821664, 3426280682, 1337602255, 3334593650, 1320885980, 3857269540, - 2548321029, 2250001180, 673341051, 1900184720, 731675831, 2461790412, 2593291320, - 1640301250, 863529987, 91627443, 2437824309, 2834231475, 4093270720, 1474594761, - 4186662839, 1683556862, 1302286991, 806676270, 703274107, 3756759580, 674737904, - 912015048, 1823306025, 1509430520, 3128952761, 290841833, 3917789380, 1022040580, - 1810054038, 334998864, 1009274987, 310979037, 606749827, 546291081, 3438438313, - 1840081424, 1950680845, 4217236364, 1814584903, 2814353208, 194196981, 1540331253, - 3135937654, 773351497, 1878220007, 3097009802, 1252607159, 1378821846, 2741884614, - 178612659, 3656860395, 1259606652, 3942111545, 488406826, 3640897405, 3419000480, - 353909713, 2996208477, 2862593073, 108483327, 648472258, 1060249632, 1049865483, - 430087518, 1364157854, 3367631180, 251313827, 2374149836, 2109357086, 479172068, - 464775113, 1806677787, 3488082411, 356035738, 3080424395, 4134646749, 369528743, - 1031004516, 2525336414, 4189798138, 3928909462, 568714397, 1681832820, 1753328641, - 827357673, 1651960551, 1798317455, 737101952, 3257553606, 400882781, 1473208110, - 4134183873, 2193420912, - }, - { - 2483976489, 2790651795, 3298324523, 3508205426, 2236819708, 917494217, 769620837, - 3411018785, 2391335000, 1627061280, 3356773416, 1288706527, 4178910717, 3636299534, - 4221874052, 3674654381, 537787012, 4271656840, 185820273, 1160533598, 1862365049, - 2550353307, 1392072847, 1870891365, 1517453821, 524666025, 3645751565, 2415020247, - 3691419894, 2580450642, 2130267479, 3636103610, 562446539, 750696587, 97137475, - 3894066051, 2239638596, 3256181120, 3981041836, 774947039, 451287677, 3618957054, - 4236303539, 1027744929, 1497195372, 498574915, 2164122779, 582902291, 3040883311, - 1626221455, 1853378, 2125490000, 3185055972, 1607660025, 432884530, 779476209, - 124284956, 2488937128, 2521389012, 107485781, 2873055013, 1171872946, 3130489952, - 4273333914, 646240524, 3970896645, 942009076, 4069926418, 3129385884, 3470469370, - 388702536, 450999415, 2995728716, 1687173264, 3049352827, 2648078738, 190663705, - 486809970, 424002670, 2421764946, 2941043524, 3841512738, 119077561, 1801381572, - 2208680167, 2502730219, 9899015, 2455199230, 3755314209, 3958460021, 3846398898, - 1405136244, 2870563334, 821846618, 2790899812, 863647562, 629585032, 958925512, - 1190540209, 57251233, 2109551995, 2294881622, 2603370255, 3839518646, 123838650, - 3436270690, 1637121394, 3761101432, 954001192, 759760236, 3268295908, 2313083096, - 630164216, 2367213191, 3992059381, 3292952769, 2040774258, 1420209005, 527547730, - 1222399440, 1515078401, 2005580991, 645585788, 2256370254, 3057235502, 2870727428, - 2785498804, 333440916, 1873686678, 2489794553, 3726728164, 3405629071, 3869328595, - 3081963448, 2122133003, 1428788181, 4141962679, 41030733, 183716455, 36316501, - 1430796327, 1884066707, 1216957106, 3455082673, 1092665987, 535070834, 3873372533, - 175757671, 3414803303, 791028991, 3436610906, 2950895946, 977680845, 4224715886, - 2809442211, 4044727083, 3035532020, 4253187882, 969203959, 2539482914, 813880136, - 2483976489, 2790651795, 3298324523, 3508205426, 49280479, 917494217, 769620837, - 3411018785, 2391335000, 3036738936, 3356773416, 1288706527, 4178910717, 3636299534, - 2294957038, 3674654381, 537787012, 4271656840, 185820273, 2622722506, 1862365049, - 2550353307, 1392072847, 1870891365, 2838104933, 524666025, 3645751565, 2415020247, - 3691419894, 1295777418, 2130267479, 3636103610, 562446539, 750696587, 249830932, - 3894066051, 2239638596, 3256181120, 3981041836, 3217398876, 451287677, 3618957054, - 4236303539, 1027744929, 1724964245, 498574915, 2164122779, 582902291, 3040883311, - 3101287841, 1853378, 2125490000, 3185055972, 1607660025, 1128474163, 779476209, - 124284956, 2488937128, 2521389012, 338597864, 2873055013, 1171872946, 3130489952, - 4273333914, 1557892392, 3970896645, 942009076, 4069926418, 3129385884, 2688433076, - 388702536, 450999415, 2995728716, 1687173264, 157685189, 2648078738, 190663705, - 486809970, 424002670, 979986388, 2941043524, 3841512738, 119077561, 1801381572, - 2668625968, 2502730219, 9899015, 2455199230, 3755314209, 2699515741, 3846398898, - 1405136244, 2870563334, 821846618, 505633792, 863647562, 629585032, 958925512, - 1190540209, 2067402799, 2109551995, 2294881622, 2603370255, 3839518646, 2688067120, - 3436270690, 1637121394, 3761101432, 954001192, 3206166733, 3268295908, 2313083096, - 630164216, 2367213191, 3007494680, 3292952769, 2040774258, 1420209005, 527547730, - 4047406592, 1515078401, 2005580991, 645585788, 2256370254, 13805572, 2870727428, - 2785498804, 333440916, 1873686678, 1928222740, 3726728164, 3405629071, 3869328595, - 3081963448, 2971423693, 1428788181, 4141962679, 41030733, 183716455, 4064095256, - 1430796327, 1884066707, 1216957106, 3455082673, 985592757, 535070834, 3873372533, - 175757671, 3414803303, 2159028553, 3436610906, 2950895946, 977680845, 4224715886, - 345462057, 4044727083, 3035532020, 4253187882, 969203959, 984166534, 813880136, - 2483976489, 2790651795, 3298324523, 1080001158, 49280479, 917494217, 769620837, - 3411018785, 3216598401, 3036738936, 3356773416, 1288706527, 4178910717, 3311472057, - 2294957038, 3674654381, 537787012, 4271656840, 220045511, 2622722506, 1862365049, - 2550353307, 1392072847, 3057632678, 2838104933, 524666025, 3645751565, 2415020247, - 252304106, 1295777418, 2130267479, 3636103610, 562446539, 80437039, 249830932, - 3894066051, 2239638596, 3256181120, 117173223, 3217398876, 451287677, 3618957054, - 4236303539, 1986849360, 1724964245, 498574915, 2164122779, 582902291, 288631030, - 3101287841, 1853378, 2125490000, 3185055972, 824635664, 1128474163, 779476209, - 124284956, 2488937128, 1231646648, 338597864, 2873055013, 1171872946, 3130489952, - 708957725, 1557892392, 3970896645, 942009076, 4069926418, 2286522565, 2688433076, - 388702536, 450999415, 2995728716, 2523361978, 157685189, 2648078738, 190663705, - 486809970, 151444406, 979986388, 2941043524, 3841512738, 119077561, 3762447035, - 2668625968, 2502730219, 9899015, 2455199230, 3532439568, 2699515741, 3846398898, - 1405136244, 2870563334, 2242036665, 505633792, 863647562, 629585032, 958925512, - 2618618630, 2067402799, 2109551995, 2294881622, 2603370255, 2461404010, 2688067120, - 3436270690, 1637121394, 3761101432, 1076814097, 3206166733, 3268295908, 2313083096, - 630164216, 12196305, 3007494680, 3292952769, 2040774258, 1420209005, 2609377752, - 4047406592, 1515078401, 2005580991, 645585788, 865985176, 13805572, 2870727428, - 2785498804, 333440916, 3735553268, 1928222740, 3726728164, 3405629071, 3869328595, - 501640466, 2971423693, 1428788181, 4141962679, 41030733, 97561214, 4064095256, - 1430796327, 1884066707, 1216957106, 3840122090, 985592757, 535070834, 3873372533, - 175757671, 3856277268, 2159028553, 3436610906, 2950895946, 977680845, 3313441827, - 345462057, 4044727083, 3035532020, 4253187882, 3468811573, 984166534, 813880136, - 2483976489, 2790651795, 3733649754, 1080001158, 49280479, 917494217, 769620837, - 3969566450, 3216598401, 3036738936, 3356773416, 1288706527, 2444128005, 3311472057, - 2294957038, 3674654381, 537787012, 4166109669, 220045511, 2622722506, 1862365049, - 2550353307, 2552992760, 3057632678, 2838104933, 524666025, 3645751565, 664164441, - 252304106, 1295777418, 2130267479, 3636103610, 3227561061, 80437039, 249830932, - 3894066051, 2239638596, 1071536668, 117173223, 3217398876, 451287677, 3618957054, - 3066415327, 1986849360, 1724964245, 498574915, 2164122779, 3541914330, 288631030, - 3101287841, 1853378, 2125490000, 2207189978, 824635664, 1128474163, 779476209, - 124284956, 2117633906, 1231646648, 338597864, 2873055013, 1171872946, 891038594, - 708957725, 1557892392, 3970896645, 942009076, 42952651, 2286522565, 2688433076, - 388702536, 450999415, 2986730356, 2523361978, 157685189, 2648078738, 190663705, - 3058267870, 151444406, 979986388, 2941043524, 3841512738, 1844101292, 3762447035, - 2668625968, 2502730219, 9899015, 2599582093, 3532439568, 2699515741, 3846398898, - 1405136244, 811001941, 2242036665, 505633792, 863647562, 629585032, 2722320710, - 2618618630, 2067402799, 2109551995, 2294881622, 1820862072, 2461404010, 2688067120, - 3436270690, 1637121394, 3642978005, 1076814097, 3206166733, 3268295908, 2313083096, - 1900318020, 12196305, 3007494680, 3292952769, 2040774258, 520848705, 2609377752, - 4047406592, 1515078401, 2005580991, 2530251392, 865985176, 13805572, 2870727428, - 2785498804, 2878984912, 3735553268, 1928222740, 3726728164, 3405629071, 2717736455, - 501640466, 2971423693, 1428788181, 4141962679, 3704214873, 97561214, 4064095256, - 1430796327, 1884066707, 1721732760, 3840122090, 985592757, 535070834, 3873372533, - 770732059, 3856277268, 2159028553, 3436610906, 2950895946, 33753949, 3313441827, - 345462057, 4044727083, 3035532020, 4166506071, 2719759982, 1025532659, 3811323959, - 713457907, 1577198020, 1719946821, 3963262337, 1719605451, 703663722, 1943886497, - 2916371044, 1655862745, 109438187, 195575943, 2572727533, 2421761970, 1796539813, - 2020762515, 1191344316, 2492085516, 2778033179, 4002316684, 1571080685, 1157340389, - 3859584731, 3403766082, 2292873365, 2032258920, 1749575450, 848549431, 1893685820, - 3510068298, 3308906564, 1193936308, 2561670234, 1043148718, 2611815896, 3832995202, - 2436487998, 3377369330, 1174818128, 796514731, 1985886833, 88296218, 3032898657, - 4101301361, 1486994584, 237792475, 1029399834, 1708840018, 2934039708, 1496674948, - 4243234983, 3896751668, 1726119825, 2706068825, 1900013134, 2639641919, 1433377392, - 2962655166, 1870954268, 3873603462, 1778084630, 2393311756, 4135022799, 3669603001, - 811404758, 784379778, 4283689136, 405168660, 3873488622, 486946690, 347427153, - 2139072474, 1143349522, 3780264455, 2938731842, 3864001470, 3497981827, 2703917008, - 3222236962, 2604106616, 1281570367, 175937153, 433252852, 3232065906, 1111895932, - 1027363895, 2435093744, 4232690481, 1940855209, 2844613991, 2095175619, 3479946852, - 393314401, 3625733631, 1073779513, 2884072879, 4089630675, 3614205484, 1379809260, - 3980251795, 3914556410, 3633356126, 3030204458, 1654727861, 3765074811, 959734060, - 842315676, 353688341, 145655006, 1972100601, 1456042517, 3767579955, 4282066379, - 498998655, 4123310742, 1801424182, 777808179, 655425670, 588715641, 2136252742, - 1283378143, 639191135, 3132375783, 276649124, 2036776039, 3352396498, 3893441746, - 3298373918, 1024178230, 2623051553, 1956117442, 2955394456, 2478945776, 3904945720, - 769232312, 2168822980, 3715831945, 453874622, 3351529191, 3256151193, 808042625, - 1700919462, 1008305347, 1518733915, 3194328753, 2228970756, 2604658038, 1376476152, - 2147167203, 2585867511, 445717950, 3595016420, 3673970127, 3640614546, 494944945, - 152508312, 4160926899, - }, - { - 3225674336, 827428943, 2858523441, 2447266124, 1539223637, 2299756421, 776912458, - 279091824, 1152725492, 3903457284, 3987010398, 3996115574, 839506039, 3052513014, - 28550291, 2597814974, 2328446377, 1961600298, 3695276714, 1334932648, 1141381380, - 3025370440, 997698792, 931473445, 3091440507, 820119215, 3586778616, 1993126242, - 4252838072, 3033829531, 2120026924, 65722921, 746724958, 461423533, 1582298542, - 1564918930, 3710935369, 419349792, 3914061713, 2279209938, 770031171, 2062767935, - 3373230309, 3582372364, 2025682996, 3352859025, 1262632952, 3140021482, 501370035, - 2554730117, 352450195, 1002557127, 2813224858, 2808406559, 290476252, 4216846311, - 1187381982, 3131323304, 1094330039, 2646234280, 655242013, 1152156402, 3658526705, - 3565043535, 693375321, 2120064836, 3726555752, 97387177, 546586686, 1013492636, - 3874404446, 440995849, 1929251266, 95137166, 564969023, 3559119399, 3855477390, - 2439885481, 2492213232, 2611214170, 2054191666, 2778642234, 2267416277, 2194315209, - 1360165075, 1018128176, 2841084399, 3028189871, 3631770575, 541021087, 1091467742, - 2743780329, 3566538467, 1277066122, 279582475, 2712119598, 3296319359, 4187226385, - 1468994750, 2946664285, 2284913307, 740953233, 3351500634, 1791054313, 3355533193, - 610062694, 3089981426, 3469441840, 3225672476, 2223653903, 2593994385, 548784340, - 549871569, 865468702, 1593939385, 645229999, 1412095765, 2814231763, 3619658094, - 877462820, 2198765077, 1845119421, 4144145546, 1356681209, 848707034, 4144513299, - 3231318896, 3382035479, 693621410, 2821661683, 4236142563, 680649431, 3290999942, - 200856634, 617766412, 3194332974, 4102392657, 2776797278, 2932808060, 793967937, - 2149374605, 3736514467, 3547689148, 3744888920, 98278184, 1497045279, 2945126332, - 4285864315, 2791068812, 1939995011, 56752862, 864909862, 625377571, 2266362085, - 1050287398, 925722519, 1008109592, 2819528345, 3573068613, 1915083884, 1536828870, - 3225674336, 827428943, 2858523441, 2447266124, 2186287936, 2299756421, 776912458, - 279091824, 1152725492, 1271286102, 3987010398, 3996115574, 839506039, 3052513014, - 1036957208, 2597814974, 2328446377, 1961600298, 3695276714, 2395157917, 1141381380, - 3025370440, 997698792, 931473445, 2727078785, 820119215, 3586778616, 1993126242, - 4252838072, 1171102868, 2120026924, 65722921, 746724958, 461423533, 2335086228, - 1564918930, 3710935369, 419349792, 3914061713, 1136716661, 770031171, 2062767935, - 3373230309, 3582372364, 4100328450, 3352859025, 1262632952, 3140021482, 501370035, - 2579000299, 352450195, 1002557127, 2813224858, 2808406559, 2642514897, 4216846311, - 1187381982, 3131323304, 1094330039, 3092488663, 655242013, 1152156402, 3658526705, - 3565043535, 3280658482, 2120064836, 3726555752, 97387177, 546586686, 584864345, - 3874404446, 440995849, 1929251266, 95137166, 823950215, 3559119399, 3855477390, - 2439885481, 2492213232, 2297040376, 2054191666, 2778642234, 2267416277, 2194315209, - 573807317, 1018128176, 2841084399, 3028189871, 3631770575, 2747338726, 1091467742, - 2743780329, 3566538467, 1277066122, 1715139924, 2712119598, 3296319359, 4187226385, - 1468994750, 3361368810, 2284913307, 740953233, 3351500634, 1791054313, 4290564545, - 610062694, 3089981426, 3469441840, 3225672476, 1010959310, 2593994385, 548784340, - 549871569, 865468702, 1825306744, 645229999, 1412095765, 2814231763, 3619658094, - 3792219969, 2198765077, 1845119421, 4144145546, 1356681209, 268197516, 4144513299, - 3231318896, 3382035479, 693621410, 2786831464, 4236142563, 680649431, 3290999942, - 200856634, 3822069622, 3194332974, 4102392657, 2776797278, 2932808060, 525501162, - 2149374605, 3736514467, 3547689148, 3744888920, 3219948462, 1497045279, 2945126332, - 4285864315, 2791068812, 2678467476, 56752862, 864909862, 625377571, 2266362085, - 2258093843, 925722519, 1008109592, 2819528345, 3573068613, 2743241289, 1536828870, - 3225674336, 827428943, 2858523441, 992128922, 2186287936, 2299756421, 776912458, - 279091824, 2108721702, 1271286102, 3987010398, 3996115574, 839506039, 1315622698, - 1036957208, 2597814974, 2328446377, 1961600298, 3098343478, 2395157917, 1141381380, - 3025370440, 997698792, 1317753106, 2727078785, 820119215, 3586778616, 1993126242, - 2295599934, 1171102868, 2120026924, 65722921, 746724958, 3999203443, 2335086228, - 1564918930, 3710935369, 419349792, 1662083910, 1136716661, 770031171, 2062767935, - 3373230309, 3271761171, 4100328450, 3352859025, 1262632952, 3140021482, 3981040854, - 2579000299, 352450195, 1002557127, 2813224858, 1064251076, 2642514897, 4216846311, - 1187381982, 3131323304, 2077640887, 3092488663, 655242013, 1152156402, 3658526705, - 548941006, 3280658482, 2120064836, 3726555752, 97387177, 4112878213, 584864345, - 3874404446, 440995849, 1929251266, 227230803, 823950215, 3559119399, 3855477390, - 2439885481, 610498128, 2297040376, 2054191666, 2778642234, 2267416277, 518192832, - 573807317, 1018128176, 2841084399, 3028189871, 2512871059, 2747338726, 1091467742, - 2743780329, 3566538467, 386661563, 1715139924, 2712119598, 3296319359, 4187226385, - 2508754324, 3361368810, 2284913307, 740953233, 3351500634, 1296305541, 4290564545, - 610062694, 3089981426, 3469441840, 148510865, 1010959310, 2593994385, 548784340, - 549871569, 124676809, 1825306744, 645229999, 1412095765, 2814231763, 2540745278, - 3792219969, 2198765077, 1845119421, 4144145546, 3966655401, 268197516, 4144513299, - 3231318896, 3382035479, 1674022032, 2786831464, 4236142563, 680649431, 3290999942, - 4065303704, 3822069622, 3194332974, 4102392657, 2776797278, 3735376922, 525501162, - 2149374605, 3736514467, 3547689148, 2064870756, 3219948462, 1497045279, 2945126332, - 4285864315, 2389978045, 2678467476, 56752862, 864909862, 625377571, 2308006661, - 2258093843, 925722519, 1008109592, 2819528345, 2927186231, 2743241289, 1536828870, - 3225674336, 827428943, 1583633720, 992128922, 2186287936, 2299756421, 776912458, - 298217241, 2108721702, 1271286102, 3987010398, 3996115574, 1041730366, 1315622698, - 1036957208, 2597814974, 2328446377, 1386688725, 3098343478, 2395157917, 1141381380, - 3025370440, 2292273773, 1317753106, 2727078785, 820119215, 3586778616, 206996196, - 2295599934, 1171102868, 2120026924, 65722921, 3271158508, 3999203443, 2335086228, - 1564918930, 3710935369, 3305544914, 1662083910, 1136716661, 770031171, 2062767935, - 4244195826, 3271761171, 4100328450, 3352859025, 1262632952, 3581040310, 3981040854, - 2579000299, 352450195, 1002557127, 1789606594, 1064251076, 2642514897, 4216846311, - 1187381982, 1519386238, 2077640887, 3092488663, 655242013, 1152156402, 3732146227, - 548941006, 3280658482, 2120064836, 3726555752, 2631398817, 4112878213, 584864345, - 3874404446, 440995849, 3541162446, 227230803, 823950215, 3559119399, 3855477390, - 560704260, 610498128, 2297040376, 2054191666, 2778642234, 1614756373, 518192832, - 573807317, 1018128176, 2841084399, 927011949, 2512871059, 2747338726, 1091467742, - 2743780329, 566198434, 386661563, 1715139924, 2712119598, 3296319359, 520529825, - 2508754324, 3361368810, 2284913307, 740953233, 2414584088, 1296305541, 4290564545, - 610062694, 3089981426, 120496553, 148510865, 1010959310, 2593994385, 548784340, - 3206664898, 124676809, 1825306744, 645229999, 1412095765, 821445348, 2540745278, - 3792219969, 2198765077, 1845119421, 3434574619, 3966655401, 268197516, 4144513299, - 3231318896, 3856935910, 1674022032, 2786831464, 4236142563, 680649431, 3331403374, - 4065303704, 3822069622, 3194332974, 4102392657, 2194924932, 3735376922, 525501162, - 2149374605, 3736514467, 2041458481, 2064870756, 3219948462, 1497045279, 2945126332, - 3515890044, 2389978045, 2678467476, 56752862, 864909862, 1009125580, 2308006661, - 2258093843, 925722519, 1008109592, 4166824654, 3399481064, 3848337172, 841675162, - 2388734555, 3373081217, 1627287001, 1958651480, 1771323855, 2126620758, 3879967947, - 1885140905, 806066092, 2168342987, 3778265278, 943582962, 3895768303, 337928214, - 3677576461, 1884088203, 2629440785, 2357038005, 2362450760, 2080907681, 2644383608, - 4153875040, 794977307, 2675637463, 2655426076, 3481699657, 2262369403, 1038608931, - 4210267953, 2376694315, 2661705117, 3994997027, 2994346963, 4074343171, 833108024, - 3562046155, 1113632369, 3087093963, 2115712884, 2778607581, 2702162487, 1347693590, - 4271098334, 2746712394, 1629623802, 1932973152, 3077074108, 1338011180, 848785806, - 1834095770, 4017238, 2661097500, 2935787683, 1214195119, 3099491937, 3868451396, - 1063740008, 2768962809, 2554721244, 695479209, 2634119800, 1379839034, 2653377927, - 921934002, 3586936843, 3035369677, 769283110, 2417935220, 3330084607, 2020519519, - 2546176786, 1523223165, 3654065096, 1835059231, 2776263618, 3837173427, 3236141295, - 1184415634, 157448610, 2474336972, 3313035876, 309195150, 2288837115, 548743307, - 528342914, 1527562212, 554918643, 2739291918, 2630873849, 155419923, 226845272, - 1343735931, 3106346884, 4177975386, 2515480406, 2049734808, 2802879609, 1805234272, - 317920918, 745796250, 3816657414, 4198378080, 3057334192, 503016924, 2027816790, - 579332504, 3037999504, 2857298788, 911046668, 1170775701, 2369720, 3364839261, - 1462383461, 181600856, 1315241696, 2861043792, 3549404088, 3974245218, 4141518566, - 1195336199, 2291064152, 3287203016, 3867432937, 2593766219, 2114273192, 3716228986, - 410286941, 2497285113, 1338500439, 3748757692, 2315519304, 545570554, 1519868916, - 679216320, 3264840479, 4083041163, 261878334, 2370312122, 1408058272, 1287635274, - 3433241543, 3923613754, 2423502603, 3948993135, 1418484161, 230113502, 1766447938, - 3101286974, 917358979, 2836128279, 2859079881, 3162688352, 2158281644, 154509481, - 2409785274, 3096379437, - }, - { - 2456954827, 2895978734, 1621803157, 230462381, 4046364119, 716597790, 2031510641, - 2208319977, 1107910846, 3379950723, 3628284249, 1393263274, 3842378742, 768116962, - 1782906996, 3022943801, 510040722, 2180373447, 1294989632, 1659724107, 953774117, - 500296619, 2269873184, 3215368465, 3933601613, 2401810535, 3568992417, 617528376, - 2437412983, 2921242388, 2311040363, 3695847323, 1609309841, 3222455492, 1108155620, - 397599239, 3344183623, 1159383441, 81554651, 4223302962, 2812031899, 2613176831, - 2967803832, 3560382993, 670173062, 2361031672, 1745444335, 2067906079, 3019908371, - 2662226130, 2962440272, 3053411095, 92212044, 1041941495, 4116546365, 2094375399, - 3992554702, 216246182, 2271736480, 1006434362, 4104644208, 2543874803, 1310440964, - 1956002873, 1485192936, 3027546418, 448786402, 749040342, 406538664, 2522826782, - 3681979470, 3941253886, 672615054, 3655479714, 392178376, 1619518340, 1639889010, - 666186812, 4094569743, 2947917117, 3308938954, 1572886498, 1065510431, 2158389109, - 1583642689, 1763046973, 3578310229, 4106948216, 58659757, 691952777, 3394715763, - 3333944006, 3020203798, 3598064251, 3151881711, 2071056894, 1263790655, 4188233031, - 4230429856, 3088041549, 3031631424, 3912597408, 1768734847, 1975027092, 3976493733, - 2376551740, 1137628506, 535767974, 3105256806, 15427398, 2472341690, 685997424, - 1374644561, 2446214061, 2844847931, 1058649390, 1581230869, 1725846082, 3062699842, - 1116400547, 1095426642, 2940190462, 4276038488, 2091764667, 4074059985, 98163536, - 4157153745, 32476821, 2354284775, 752663757, 2987293678, 1119786914, 3019442904, - 111185876, 3569592548, 3991775183, 3161418733, 3973354577, 1650454973, 426129509, - 3659038742, 1387393667, 543731583, 781586523, 917315276, 832142534, 3911092159, - 325250500, 2735441676, 163564958, 1002098855, 337936437, 1869530240, 2233969733, - 4108076124, 3255026725, 4072134049, 2083771067, 1559589006, 1845121907, 466036013, - 2456954827, 2895978734, 1621803157, 230462381, 2022150409, 716597790, 2031510641, - 2208319977, 1107910846, 1838834877, 3628284249, 1393263274, 3842378742, 768116962, - 2303040715, 3022943801, 510040722, 2180373447, 1294989632, 3074858415, 953774117, - 500296619, 2269873184, 3215368465, 3531413908, 2401810535, 3568992417, 617528376, - 2437412983, 1730632320, 2311040363, 3695847323, 1609309841, 3222455492, 3189359980, - 397599239, 3344183623, 1159383441, 81554651, 1933731121, 2812031899, 2613176831, - 2967803832, 3560382993, 758113139, 2361031672, 1745444335, 2067906079, 3019908371, - 3537991495, 2962440272, 3053411095, 92212044, 1041941495, 2653519981, 2094375399, - 3992554702, 216246182, 2271736480, 695350220, 4104644208, 2543874803, 1310440964, - 1956002873, 3373048130, 3027546418, 448786402, 749040342, 406538664, 4081844472, - 3681979470, 3941253886, 672615054, 3655479714, 197563239, 1619518340, 1639889010, - 666186812, 4094569743, 2518320719, 3308938954, 1572886498, 1065510431, 2158389109, - 3320483696, 1763046973, 3578310229, 4106948216, 58659757, 3412172826, 3394715763, - 3333944006, 3020203798, 3598064251, 1693717788, 2071056894, 1263790655, 4188233031, - 4230429856, 2564478937, 3031631424, 3912597408, 1768734847, 1975027092, 3546175061, - 2376551740, 1137628506, 535767974, 3105256806, 450760279, 2472341690, 685997424, - 1374644561, 2446214061, 1873063065, 1058649390, 1581230869, 1725846082, 3062699842, - 813496775, 1095426642, 2940190462, 4276038488, 2091764667, 3857233976, 98163536, - 4157153745, 32476821, 2354284775, 3115605568, 2987293678, 1119786914, 3019442904, - 111185876, 996447434, 3991775183, 3161418733, 3973354577, 1650454973, 1089784804, - 3659038742, 1387393667, 543731583, 781586523, 2711412312, 832142534, 3911092159, - 325250500, 2735441676, 3563501139, 1002098855, 337936437, 1869530240, 2233969733, - 1156926454, 3255026725, 4072134049, 2083771067, 1559589006, 3832870112, 466036013, - 2456954827, 2895978734, 1621803157, 2340808859, 2022150409, 716597790, 2031510641, - 2208319977, 1823993818, 1838834877, 3628284249, 1393263274, 3842378742, 2489609764, - 2303040715, 3022943801, 510040722, 2180373447, 4204167795, 3074858415, 953774117, - 500296619, 2269873184, 2320314628, 3531413908, 2401810535, 3568992417, 617528376, - 712451843, 1730632320, 2311040363, 3695847323, 1609309841, 3224192365, 3189359980, - 397599239, 3344183623, 1159383441, 758272390, 1933731121, 2812031899, 2613176831, - 2967803832, 3986798661, 758113139, 2361031672, 1745444335, 2067906079, 3814344052, - 3537991495, 2962440272, 3053411095, 92212044, 817573506, 2653519981, 2094375399, - 3992554702, 216246182, 2456924809, 695350220, 4104644208, 2543874803, 1310440964, - 1151286621, 3373048130, 3027546418, 448786402, 749040342, 637572176, 4081844472, - 3681979470, 3941253886, 672615054, 3038758846, 197563239, 1619518340, 1639889010, - 666186812, 4254608071, 2518320719, 3308938954, 1572886498, 1065510431, 3100620860, - 3320483696, 1763046973, 3578310229, 4106948216, 403923766, 3412172826, 3394715763, - 3333944006, 3020203798, 1859724785, 1693717788, 2071056894, 1263790655, 4188233031, - 2908736862, 2564478937, 3031631424, 3912597408, 1768734847, 966714666, 3546175061, - 2376551740, 1137628506, 535767974, 1561255376, 450760279, 2472341690, 685997424, - 1374644561, 3122124160, 1873063065, 1058649390, 1581230869, 1725846082, 3791666219, - 813496775, 1095426642, 2940190462, 4276038488, 2802023399, 3857233976, 98163536, - 4157153745, 32476821, 1640659450, 3115605568, 2987293678, 1119786914, 3019442904, - 4278091706, 996447434, 3991775183, 3161418733, 3973354577, 3398421232, 1089784804, - 3659038742, 1387393667, 543731583, 1694361696, 2711412312, 832142534, 3911092159, - 325250500, 166035542, 3563501139, 1002098855, 337936437, 1869530240, 1306446339, - 1156926454, 3255026725, 4072134049, 2083771067, 61899937, 3832870112, 466036013, - 2456954827, 2895978734, 767569205, 2340808859, 2022150409, 716597790, 2031510641, - 1690074863, 1823993818, 1838834877, 3628284249, 1393263274, 546011580, 2489609764, - 2303040715, 3022943801, 510040722, 825252468, 4204167795, 3074858415, 953774117, - 500296619, 1952242515, 2320314628, 3531413908, 2401810535, 3568992417, 4254767597, - 712451843, 1730632320, 2311040363, 3695847323, 2393864919, 3224192365, 3189359980, - 397599239, 3344183623, 1759399025, 758272390, 1933731121, 2812031899, 2613176831, - 2809078783, 3986798661, 758113139, 2361031672, 1745444335, 1223235915, 3814344052, - 3537991495, 2962440272, 3053411095, 3711100000, 817573506, 2653519981, 2094375399, - 3992554702, 2987412942, 2456924809, 695350220, 4104644208, 2543874803, 2746231792, - 1151286621, 3373048130, 3027546418, 448786402, 801157439, 637572176, 4081844472, - 3681979470, 3941253886, 975875511, 3038758846, 197563239, 1619518340, 1639889010, - 3137491209, 4254608071, 2518320719, 3308938954, 1572886498, 631178204, 3100620860, - 3320483696, 1763046973, 3578310229, 3338308117, 403923766, 3412172826, 3394715763, - 3333944006, 37220448, 1859724785, 1693717788, 2071056894, 1263790655, 228419012, - 2908736862, 2564478937, 3031631424, 3912597408, 3862306448, 966714666, 3546175061, - 2376551740, 1137628506, 1114919961, 1561255376, 450760279, 2472341690, 685997424, - 2456661198, 3122124160, 1873063065, 1058649390, 1581230869, 2996925693, 3791666219, - 813496775, 1095426642, 2940190462, 1642720015, 2802023399, 3857233976, 98163536, - 4157153745, 1578965959, 1640659450, 3115605568, 2987293678, 1119786914, 1748408698, - 4278091706, 996447434, 3991775183, 3161418733, 4123935663, 3398421232, 1089784804, - 3659038742, 1387393667, 770706529, 1694361696, 2711412312, 832142534, 3911092159, - 335435644, 166035542, 3563501139, 1002098855, 337936437, 2961857543, 1306446339, - 1156926454, 3255026725, 4072134049, 1717290230, 1323146393, 2156340433, 2065716367, - 2597996276, 3402032152, 779574284, 2369501052, 2316224856, 2720986136, 3016786025, - 2916554213, 3476215746, 1132150235, 2619889920, 1279664685, 679206534, 4014394509, - 3624968312, 1480455625, 725015758, 707677352, 3764409715, 1938306480, 2171474419, - 3379664161, 684262379, 2142433069, 43407198, 1398850259, 2059135843, 240266749, - 3788738212, 118513026, 820245055, 1152812311, 1398373423, 3188977726, 872620936, - 2084649448, 807979538, 819501992, 615447916, 3393148006, 1765623964, 2514767257, - 3711360450, 2941886951, 3739102698, 4022385962, 2306039667, 3321267290, 2179238310, - 3192652502, 2118792870, 2571142127, 761776508, 873010906, 1609627751, 4260021041, - 1747852747, 960771906, 2647903291, 77475681, 1282566533, 4022186916, 2681128032, - 1554542462, 3181701944, 1168469070, 74236514, 2806532232, 3981048887, 1888842784, - 2888607878, 1763028723, 701886756, 4124077776, 3738147505, 4066663138, 3816449863, - 921061872, 2956972182, 3159072916, 3337110888, 3552795700, 2281281091, 671098116, - 1282750020, 1008618197, 2363767765, 1812013295, 1854965999, 131027176, 666394000, - 2062217824, 1763334218, 551118598, 1277961175, 3523893635, 1855881150, 2067903393, - 2590963277, 3214508854, 1604911832, 1906690475, 389417851, 2711591984, 427723436, - 1039703630, 639602991, 444779318, 2722002973, 3927985419, 1297446054, 298277450, - 656022205, 134304205, 3847728042, 3339100423, 407022043, 1282443442, 3173884578, - 1417906094, 2364502739, 2158353472, 2402775649, 1807696073, 2837535198, 705887737, - 2129202688, 3853676283, 1388329793, 875153687, 2367465660, 2763058233, 2500632304, - 2196920062, 491306883, 277753357, 3868415380, 324867643, 3654474955, 2569410351, - 1128175417, 1853572398, 1133201743, 662085935, 2263514999, 3077768113, 3309730620, - 3602394176, 3747458070, 188422725, 813812450, 1502276531, 3909138356, 2766044599, - 3760928321, 573108836, - }, - { - 1240264181, 1624064648, 3039823158, 2013985253, 1473300299, 2762062141, 3273470484, - 1889745445, 2516996174, 3190376531, 996186898, 3893981177, 1268272590, 3226095713, - 153038465, 2184871198, 3224094011, 2526518401, 1738960059, 1187560605, 4194384320, - 2837011297, 3638232350, 367907454, 574009898, 1948901330, 60430044, 1569835584, - 3160561697, 321792583, 3179087993, 1936928378, 412346905, 4020812489, 2603392174, - 3499496781, 1499441233, 1062415256, 1347130973, 1823246794, 3411391800, 4253618056, - 1507733072, 1605629518, 1503312494, 8035741, 4038904206, 2408545792, 969543501, - 954847087, 956553276, 3096241999, 2566194741, 84678421, 3882676079, 2483934330, - 3673546814, 2461422466, 620385599, 898325340, 2145883445, 3653728520, 3744850294, - 2441124935, 904854507, 3216304963, 2373268568, 2354362010, 1245572787, 2894748714, - 2889136188, 3716879184, 1766013949, 1305712667, 1227530310, 4051221847, 925440190, - 1508686692, 1104647879, 1496666754, 3300504219, 127787091, 1528394637, 1739640835, - 2475711496, 3792639955, 1450796299, 1634217367, 3289785095, 2149949989, 811612039, - 1750779366, 1157474938, 514004414, 2264909096, 3730411668, 3308882513, 1834571716, - 378288317, 3800023701, 763396788, 1597708317, 983953861, 94566098, 1548157668, - 3755427117, 1646496505, 3748241449, 3439805936, 2321644449, 3805706235, 4220083901, - 1069923823, 2984004391, 3824885361, 1967477766, 218978249, 348955028, 3188651823, - 1008338679, 2331688720, 1562995454, 1837179689, 3033872688, 3007293665, 1759522678, - 319754369, 2763991927, 1983149629, 1353197132, 1489552694, 2990539062, 3244609108, - 669775440, 886127995, 1636688014, 1251222487, 2351883247, 3261502906, 3139614137, - 3203790139, 2777648095, 3693390579, 3540514982, 3200191735, 750726325, 1014534145, - 2091792357, 3931704474, 1383925867, 2038878506, 2247134268, 2840132188, 61137652, - 1162051299, 399657268, 1682018695, 2640231287, 1733438115, 3611823506, 2077891037, - 1240264181, 1624064648, 3039823158, 2013985253, 4188888201, 2762062141, 3273470484, - 1889745445, 2516996174, 2621448256, 996186898, 3893981177, 1268272590, 3226095713, - 952803645, 2184871198, 3224094011, 2526518401, 1738960059, 738368399, 4194384320, - 2837011297, 3638232350, 367907454, 3772812520, 1948901330, 60430044, 1569835584, - 3160561697, 1655622513, 3179087993, 1936928378, 412346905, 4020812489, 3754224996, - 3499496781, 1499441233, 1062415256, 1347130973, 1167581269, 3411391800, 4253618056, - 1507733072, 1605629518, 1867781671, 8035741, 4038904206, 2408545792, 969543501, - 3189323143, 956553276, 3096241999, 2566194741, 84678421, 996778900, 2483934330, - 3673546814, 2461422466, 620385599, 3129088144, 2145883445, 3653728520, 3744850294, - 2441124935, 4230756652, 3216304963, 2373268568, 2354362010, 1245572787, 1600525238, - 2889136188, 3716879184, 1766013949, 1305712667, 59908073, 4051221847, 925440190, - 1508686692, 1104647879, 2931214731, 3300504219, 127787091, 1528394637, 1739640835, - 62963469, 3792639955, 1450796299, 1634217367, 3289785095, 667987389, 811612039, - 1750779366, 1157474938, 514004414, 2737193098, 3730411668, 3308882513, 1834571716, - 378288317, 3452657469, 763396788, 1597708317, 983953861, 94566098, 2752347916, - 3755427117, 1646496505, 3748241449, 3439805936, 4222757079, 3805706235, 4220083901, - 1069923823, 2984004391, 3887639520, 1967477766, 218978249, 348955028, 3188651823, - 4168456281, 2331688720, 1562995454, 1837179689, 3033872688, 814903833, 1759522678, - 319754369, 2763991927, 1983149629, 3818528075, 1489552694, 2990539062, 3244609108, - 669775440, 1004789460, 1636688014, 1251222487, 2351883247, 3261502906, 4143823654, - 3203790139, 2777648095, 3693390579, 3540514982, 153421222, 750726325, 1014534145, - 2091792357, 3931704474, 4018591985, 2038878506, 2247134268, 2840132188, 61137652, - 1455028838, 399657268, 1682018695, 2640231287, 1733438115, 1853142849, 2077891037, - 1240264181, 1624064648, 3039823158, 2235369076, 4188888201, 2762062141, 3273470484, - 1889745445, 3627876603, 2621448256, 996186898, 3893981177, 1268272590, 2687846008, - 952803645, 2184871198, 3224094011, 2526518401, 861379413, 738368399, 4194384320, - 2837011297, 3638232350, 3753321702, 3772812520, 1948901330, 60430044, 1569835584, - 581506474, 1655622513, 3179087993, 1936928378, 412346905, 2710043900, 3754224996, - 3499496781, 1499441233, 1062415256, 2704745463, 1167581269, 3411391800, 4253618056, - 1507733072, 4215403465, 1867781671, 8035741, 4038904206, 2408545792, 3252742933, - 3189323143, 956553276, 3096241999, 2566194741, 1865159158, 996778900, 2483934330, - 3673546814, 2461422466, 3123557619, 3129088144, 2145883445, 3653728520, 3744850294, - 21840044, 4230756652, 3216304963, 2373268568, 2354362010, 1934462999, 1600525238, - 2889136188, 3716879184, 1766013949, 2822794708, 59908073, 4051221847, 925440190, - 1508686692, 2938291976, 2931214731, 3300504219, 127787091, 1528394637, 1914923136, - 62963469, 3792639955, 1450796299, 1634217367, 257322213, 667987389, 811612039, - 1750779366, 1157474938, 3083649350, 2737193098, 3730411668, 3308882513, 1834571716, - 2778729422, 3452657469, 763396788, 1597708317, 983953861, 1337754195, 2752347916, - 3755427117, 1646496505, 3748241449, 3942745717, 4222757079, 3805706235, 4220083901, - 1069923823, 1314928500, 3887639520, 1967477766, 218978249, 348955028, 3425797638, - 4168456281, 2331688720, 1562995454, 1837179689, 1814071277, 814903833, 1759522678, - 319754369, 2763991927, 1079270448, 3818528075, 1489552694, 2990539062, 3244609108, - 2944573315, 1004789460, 1636688014, 1251222487, 2351883247, 1356892540, 4143823654, - 3203790139, 2777648095, 3693390579, 983917956, 153421222, 750726325, 1014534145, - 2091792357, 296882400, 4018591985, 2038878506, 2247134268, 2840132188, 3508266160, - 1455028838, 399657268, 1682018695, 2640231287, 2480988791, 1853142849, 2077891037, - 1240264181, 1624064648, 1741738969, 2235369076, 4188888201, 2762062141, 3273470484, - 3569498651, 3627876603, 2621448256, 996186898, 3893981177, 4026533880, 2687846008, - 952803645, 2184871198, 3224094011, 1290870737, 861379413, 738368399, 4194384320, - 2837011297, 3833099205, 3753321702, 3772812520, 1948901330, 60430044, 4131290878, - 581506474, 1655622513, 3179087993, 1936928378, 2379952582, 2710043900, 3754224996, - 3499496781, 1499441233, 593780490, 2704745463, 1167581269, 3411391800, 4253618056, - 621889762, 4215403465, 1867781671, 8035741, 4038904206, 2045289976, 3252742933, - 3189323143, 956553276, 3096241999, 2188329018, 1865159158, 996778900, 2483934330, - 3673546814, 2717648418, 3123557619, 3129088144, 2145883445, 3653728520, 1528077261, - 21840044, 4230756652, 3216304963, 2373268568, 803158556, 1934462999, 1600525238, - 2889136188, 3716879184, 161827512, 2822794708, 59908073, 4051221847, 925440190, - 3599942370, 2938291976, 2931214731, 3300504219, 127787091, 4082579845, 1914923136, - 62963469, 3792639955, 1450796299, 2035446714, 257322213, 667987389, 811612039, - 1750779366, 2344204796, 3083649350, 2737193098, 3730411668, 3308882513, 2765191583, - 2778729422, 3452657469, 763396788, 1597708317, 1854746879, 1337754195, 2752347916, - 3755427117, 1646496505, 4020292301, 3942745717, 4222757079, 3805706235, 4220083901, - 1408262601, 1314928500, 3887639520, 1967477766, 218978249, 2173193841, 3425797638, - 4168456281, 2331688720, 1562995454, 2835294077, 1814071277, 814903833, 1759522678, - 319754369, 4048528178, 1079270448, 3818528075, 1489552694, 2990539062, 787253600, - 2944573315, 1004789460, 1636688014, 1251222487, 3584515216, 1356892540, 4143823654, - 3203790139, 2777648095, 1681621541, 983917956, 153421222, 750726325, 1014534145, - 3951869055, 296882400, 4018591985, 2038878506, 2247134268, 1990726826, 3508266160, - 1455028838, 399657268, 1682018695, 3360119279, 3151120565, 3011208718, 3694535943, - 104562665, 2827623271, 249712003, 3413221355, 2347164236, 3227498378, 1805068659, - 2118219686, 1568133029, 902801951, 175637375, 3812819970, 2162769758, 3845613089, - 1795179477, 171494391, 3765826349, 1725798906, 345463508, 2481043227, 226569380, - 3250095421, 1085199388, 3107594542, 4011388155, 1092611190, 3239339214, 4211849464, - 4109911546, 81212018, 3691937144, 2477407396, 3320520455, 3070067913, 3808621884, - 252917069, 3394860294, 1092442235, 2876536384, 1684120191, 431096075, 1701716708, - 639881684, 3066183997, 3660504927, 2047274, 3424756424, 760932520, 2457976057, - 1705265011, 2691137533, 3684307557, 3532744498, 2319162513, 1015534908, 1907173398, - 2820698743, 1264455116, 2323788906, 3062240844, 1878550513, 1717353426, 1805673248, - 62425157, 3662381032, 1964107209, 2559831960, 2117844804, 1228721677, 4240498866, - 3212920337, 2338600301, 931588693, 2379606585, 3643222352, 4154645082, 1115847065, - 2079427925, 2256943798, 2795103368, 2688136486, 1458062143, 1767222217, 635424385, - 284062050, 1547163554, 3380046528, 1145758046, 3935976713, 4017430175, 3863367362, - 3041367424, 303263160, 1465965696, 3757919837, 3083072836, 4024514094, 1381331179, - 2393446325, 3256476469, 4066482738, 3437941107, 1051266504, 921764078, 2933305619, - 1358097211, 4100978724, 2709958834, 574590507, 961767386, 21100886, 753746372, - 4072632446, 733729367, 3060214669, 289165105, 426065754, 2036100240, 2172365757, - 502856627, 84490194, 2630806596, 1206161269, 1009438449, 569581317, 1836947000, - 3125379675, 1756936428, 3772694822, 3670337911, 3020603818, 2376224883, 2539951453, - 2053395002, 3525193914, 1991480838, 3786481083, 873873707, 1693894743, 2450223985, - 754878026, 1943356492, 401524329, 759931885, 611231307, 147950334, 599693701, - 3358729722, 3649058074, 906423787, 1333804225, 875187278, 1115838692, 2476325972, - 3307226674, 3539078918, - }, -}; diff --git a/src/include/miopen/precalc_xorwow_skipahead_sequence_matrices.hpp b/src/include/miopen/precalc_xorwow_skipahead_sequence_matrices.hpp deleted file mode 100644 index 44e83842c8..0000000000 --- a/src/include/miopen/precalc_xorwow_skipahead_sequence_matrices.hpp +++ /dev/null @@ -1,3755 +0,0 @@ -#define XORWOW_DIM 5 -#define XORWOW_BITS 32 -#define XORWOW_PRECALC_MATRICES_SZ (XORWOW_BITS * XORWOW_DIM * XORWOW_DIM) -#define XORWOW_PRECALC_MATRICES_NUM 32 -#define XORWOW_JUMP_LOG2 2 -#define XORWOW_JUMP_LOG2_MASK ((1 << XORWOW_JUMP_LOG2) - 1) -#define XORWOW_SEQUENCE_JUMP_LOG2 67 - -static const unsigned int precalc_xorwow_skipahead_sequence_matrices - [XORWOW_PRECALC_MATRICES_NUM][XORWOW_PRECALC_MATRICES_SZ] = { - { - 850664906, 2293210629, 1517805917, 1215500405, 1612415445, 645388200, 824349799, - 3517232886, 4075591755, 3089899292, 4249786064, 3811424903, 1100783479, 53649761, - 2817264826, 3159462529, 1654848550, 950025444, 3095510002, 4080567211, 4111078399, - 3241719305, 2788212779, 4256963770, 2426893717, 4190211142, 1420776905, 3780537969, - 1102912875, 1657948873, 3354905256, 2519610308, 515777663, 3396785394, 1832603711, - 1154211550, 1915690212, 1933919046, 789578337, 337961173, 1359089498, 2249086205, - 3417955173, 862571348, 528120760, 1265685672, 1970052076, 3585976752, 3645339918, - 312171257, 1360991400, 1994321680, 2327168468, 2540437053, 1180483641, 2217962701, - 182726833, 590204372, 1904496495, 2545607041, 3697978033, 1084030545, 3397906968, - 2192325323, 2704204176, 1069092002, 2364406907, 1578647245, 3561974633, 3437665426, - 1464127305, 1616628807, 2243114101, 3639967880, 1702613633, 2437350057, 39991274, - 2024323584, 3795072940, 3604530798, 443099203, 643536212, 1919517328, 3931285769, - 427935569, 276421624, 2492081750, 262729512, 3088549877, 2922650665, 1816283755, - 4246096489, 842575914, 1460435650, 3050522190, 2640849794, 3697925816, 3465779075, - 3856929655, 1365559780, 2897029415, 2747033756, 3611830629, 1891542518, 1897590206, - 437451803, 677924906, 123809117, 3940574372, 687640291, 3488484529, 470218446, - 1092571016, 1537938503, 1073323937, 611300083, 3809285994, 3975678726, 925845389, - 2514775760, 2859302390, 2761919483, 993285307, 164095287, 3736193671, 2078946336, - 1418537059, 1202525920, 4234029440, 1313593624, 2484428922, 1833969372, 661495122, - 2217907395, 2795045321, 2950835531, 1402379354, 351314168, 1902476749, 1914974334, - 2873973176, 1321203603, 3316118265, 3282193947, 1342191737, 793441242, 3281524559, - 296088733, 487851702, 712098215, 1388727135, 1705533557, 3557800292, 399729516, - 1355829467, 291276309, 421164833, 1318404599, 2064519128, 1161612642, 2076623594, - 850664906, 2293210629, 1517805917, 1215500405, 3847487204, 645388200, 824349799, - 3517232886, 4075591755, 2755872609, 4249786064, 3811424903, 1100783479, 53649761, - 1417544262, 3159462529, 1654848550, 950025444, 3095510002, 1908900347, 4111078399, - 3241719305, 2788212779, 4256963770, 3750258343, 4190211142, 1420776905, 3780537969, - 1102912875, 1690550, 3354905256, 2519610308, 515777663, 3396785394, 2658162202, - 1154211550, 1915690212, 1933919046, 789578337, 189880016, 1359089498, 2249086205, - 3417955173, 862571348, 998719835, 1265685672, 1970052076, 3585976752, 3645339918, - 2973042959, 1360991400, 1994321680, 2327168468, 2540437053, 2283905032, 2217962701, - 182726833, 590204372, 1904496495, 110719262, 3697978033, 1084030545, 3397906968, - 2192325323, 4133333579, 1069092002, 2364406907, 1578647245, 3561974633, 3629845331, - 1464127305, 1616628807, 2243114101, 3639967880, 3256744141, 2437350057, 39991274, - 2024323584, 3795072940, 1024703328, 443099203, 643536212, 1919517328, 3931285769, - 2755167056, 276421624, 2492081750, 262729512, 3088549877, 2817867653, 1816283755, - 4246096489, 842575914, 1460435650, 2276077438, 2640849794, 3697925816, 3465779075, - 3856929655, 130551477, 2897029415, 2747033756, 3611830629, 1891542518, 804565809, - 437451803, 677924906, 123809117, 3940574372, 2446610749, 3488484529, 470218446, - 1092571016, 1537938503, 1502147484, 611300083, 3809285994, 3975678726, 925845389, - 872826112, 2859302390, 2761919483, 993285307, 164095287, 3901654538, 2078946336, - 1418537059, 1202525920, 4234029440, 704759480, 2484428922, 1833969372, 661495122, - 2217907395, 3287413716, 2950835531, 1402379354, 351314168, 1902476749, 2033316109, - 2873973176, 1321203603, 3316118265, 3282193947, 1316780684, 793441242, 3281524559, - 296088733, 487851702, 314311643, 1388727135, 1705533557, 3557800292, 399729516, - 1660074989, 291276309, 421164833, 1318404599, 2064519128, 3156334112, 2076623594, - 850664906, 2293210629, 1517805917, 335452425, 3847487204, 645388200, 824349799, - 3517232886, 954487767, 2755872609, 4249786064, 3811424903, 1100783479, 3408594583, - 1417544262, 3159462529, 1654848550, 950025444, 324339737, 1908900347, 4111078399, - 3241719305, 2788212779, 1890540205, 3750258343, 4190211142, 1420776905, 3780537969, - 3716648585, 1690550, 3354905256, 2519610308, 515777663, 3758156132, 2658162202, - 1154211550, 1915690212, 1933919046, 844149171, 189880016, 1359089498, 2249086205, - 3417955173, 1031812215, 998719835, 1265685672, 1970052076, 3585976752, 3174204115, - 2973042959, 1360991400, 1994321680, 2327168468, 714016907, 2283905032, 2217962701, - 182726833, 590204372, 2151450260, 110719262, 3697978033, 1084030545, 3397906968, - 767772303, 4133333579, 1069092002, 2364406907, 1578647245, 42955292, 3629845331, - 1464127305, 1616628807, 2243114101, 3222189776, 3256744141, 2437350057, 39991274, - 2024323584, 3142424684, 1024703328, 443099203, 643536212, 1919517328, 918511196, - 2755167056, 276421624, 2492081750, 262729512, 4246877536, 2817867653, 1816283755, - 4246096489, 842575914, 1425765936, 2276077438, 2640849794, 3697925816, 3465779075, - 1491702526, 130551477, 2897029415, 2747033756, 3611830629, 1844578694, 804565809, - 437451803, 677924906, 123809117, 3419189841, 2446610749, 3488484529, 470218446, - 1092571016, 3272535988, 1502147484, 611300083, 3809285994, 3975678726, 2853681168, - 872826112, 2859302390, 2761919483, 993285307, 1434560128, 3901654538, 2078946336, - 1418537059, 1202525920, 2530097881, 704759480, 2484428922, 1833969372, 661495122, - 503878844, 3287413716, 2950835531, 1402379354, 351314168, 4131886119, 2033316109, - 2873973176, 1321203603, 3316118265, 237900321, 1316780684, 793441242, 3281524559, - 296088733, 1730738847, 314311643, 1388727135, 1705533557, 3557800292, 1553835665, - 1660074989, 291276309, 421164833, 1318404599, 964731488, 3156334112, 2076623594, - 850664906, 2293210629, 1105350579, 335452425, 3847487204, 645388200, 824349799, - 2789953706, 954487767, 2755872609, 4249786064, 3811424903, 3937839949, 3408594583, - 1417544262, 3159462529, 1654848550, 624060530, 324339737, 1908900347, 4111078399, - 3241719305, 2294919498, 1890540205, 3750258343, 4190211142, 1420776905, 2279133729, - 3716648585, 1690550, 3354905256, 2519610308, 3563975602, 3758156132, 2658162202, - 1154211550, 1915690212, 3505586122, 844149171, 189880016, 1359089498, 2249086205, - 2389487504, 1031812215, 998719835, 1265685672, 1970052076, 2798611919, 3174204115, - 2973042959, 1360991400, 1994321680, 1684134678, 714016907, 2283905032, 2217962701, - 182726833, 1734988742, 2151450260, 110719262, 3697978033, 1084030545, 159906818, - 767772303, 4133333579, 1069092002, 2364406907, 1290801202, 42955292, 3629845331, - 1464127305, 1616628807, 987794861, 3222189776, 3256744141, 2437350057, 39991274, - 3644076751, 3142424684, 1024703328, 443099203, 643536212, 1487589384, 918511196, - 2755167056, 276421624, 2492081750, 137688638, 4246877536, 2817867653, 1816283755, - 4246096489, 1518475380, 1425765936, 2276077438, 2640849794, 3697925816, 4226506771, - 1491702526, 130551477, 2897029415, 2747033756, 2033599579, 1844578694, 804565809, - 437451803, 677924906, 2749065512, 3419189841, 2446610749, 3488484529, 470218446, - 290444026, 3272535988, 1502147484, 611300083, 3809285994, 2546040767, 2853681168, - 872826112, 2859302390, 2761919483, 4097961150, 1434560128, 3901654538, 2078946336, - 1418537059, 2725734455, 2530097881, 704759480, 2484428922, 1833969372, 3999408333, - 503878844, 3287413716, 2950835531, 1402379354, 3861442503, 4131886119, 2033316109, - 2873973176, 1321203603, 1267331405, 237900321, 1316780684, 793441242, 3281524559, - 1273427916, 1730738847, 314311643, 1388727135, 1705533557, 1474310231, 1553835665, - 1660074989, 291276309, 421164833, 3884815658, 3088049345, 3307042227, 3228948601, - 1717605083, 1864502063, 3799516572, 2372822470, 2691586476, 1172840854, 1577099080, - 870101866, 2139291021, 406996656, 255568268, 897760202, 674745664, 885214361, - 3753233375, 3015215223, 1711461259, 3241363282, 2125360928, 2493601640, 2350228245, - 3434627328, 2095642963, 3360932494, 3287396242, 4070512427, 3415702664, 1958354224, - 3280206940, 3929504236, 3390499817, 4144225735, 3621750606, 3205006592, 3495743785, - 269239326, 2181299371, 2898796651, 2613623219, 3988711298, 2162437858, 949553433, - 3289670000, 3559525307, 3366925567, 2112148665, 955626393, 1790865381, 699223558, - 3889584301, 1020750250, 4105283899, 2295851818, 4045668915, 2224770025, 766386910, - 4265157386, 89139307, 2099710177, 1012450874, 1875492446, 1927399417, 767450812, - 654474783, 4265293038, 4041215389, 4102336947, 4263617328, 2135826340, 2317231535, - 3773895729, 403151111, 1400693138, 4255050194, 755369466, 2325764302, 2617301159, - 4165707294, 1206304709, 2415645397, 4276004841, 1457022279, 662660652, 795140282, - 828519889, 805830562, 1179976369, 2212548232, 755708248, 1034682071, 899950902, - 1906046264, 1861009040, 310711525, 920739741, 2322414272, 3179236470, 81822135, - 4111390320, 1800166783, 112253014, 688771939, 1050990794, 3124647483, 287052171, - 1363630156, 3447798279, 1405733552, 3075862538, 1682808202, 1595154222, 1173705692, - 680713285, 2748212230, 568610527, 3434965538, 1114942930, 2835858745, 2575992250, - 3243355150, 2127580225, 1855934450, 3915941751, 2228679809, 1514780124, 1506688039, - 1033083295, 793807083, 1120681149, 4105670165, 3999570340, 2083020131, 1213356023, - 3684882757, 3375797774, 3577986103, 2092046164, 2593847443, 1826450612, 367828409, - 3198272513, 1941316667, 943707510, 907134807, 2020457947, 1462193665, 2964617539, - 4216491663, 2625270800, 2395371467, 3691003028, 3659016793, 2381847054, 3513105567, - 3013019506, 2731245927, - }, - { - 1680024716, 2112340059, 3387475367, 2080916186, 1431532386, 3907378472, 2636491350, - 2176128529, 2236616671, 3736851460, 2604001339, 3893075234, 3495918635, 4116370522, - 1384310379, 3660102574, 2030233939, 2759207091, 49347923, 97526506, 2566932710, - 1566181275, 3127827248, 578401670, 1499229308, 2581732444, 279715551, 809690877, - 1438444015, 878935323, 1495277039, 3417305339, 2858903785, 3074075088, 603749086, - 2370669734, 391683868, 3933465331, 2884128106, 1478317876, 1864988335, 2925823809, - 4133578805, 218104493, 368652174, 1998600344, 1109346044, 1716435313, 415435111, - 91393686, 2536620737, 1440068573, 481874870, 142128108, 988825519, 2077118779, - 2858045339, 4068162251, 115593872, 1364244587, 3550167006, 3728768059, 1772423685, - 2504624145, 248732306, 1412607307, 4081166331, 154438218, 1652901877, 3932533490, - 3142799969, 3154073676, 3112018078, 2757873595, 2364830126, 2855791484, 793851407, - 507785167, 263713916, 4060700051, 3291978358, 1584226715, 2546417990, 450747961, - 2951067700, 2706009093, 1788578194, 4030171132, 2610979903, 573420740, 4269115622, - 2180305819, 2646894726, 716649335, 3875715683, 853428184, 2436760738, 4190071217, - 2754423535, 540698101, 4082489821, 741976046, 267559495, 1591532642, 2500610323, - 3203248679, 147312102, 2772368222, 1412987047, 2295185573, 1932341300, 898396308, - 1837129999, 3113914292, 2613354524, 3141601915, 276087167, 1887389351, 757801450, - 3752353732, 2745818074, 1442953464, 3802648347, 223728071, 2169947402, 1338125300, - 3642174036, 2794462634, 2326349851, 862746036, 3577092599, 627103363, 552173564, - 4142604459, 2310329406, 583522272, 189323282, 1217612313, 73550248, 2434692829, - 2757269706, 2392210091, 3032922600, 3573904125, 2897178037, 2632631469, 3085332665, - 3775619904, 2563291734, 1351375865, 4043427793, 1803743084, 3112116579, 522940594, - 2690374983, 2613871529, 3810037031, 1765642390, 534554747, 1930852049, 2264349344, - 1680024716, 2112340059, 3387475367, 2080916186, 75966494, 3907378472, 2636491350, - 2176128529, 2236616671, 2372987046, 2604001339, 3893075234, 3495918635, 4116370522, - 534929913, 3660102574, 2030233939, 2759207091, 49347923, 987575186, 2566932710, - 1566181275, 3127827248, 578401670, 3731513754, 2581732444, 279715551, 809690877, - 1438444015, 2185866850, 1495277039, 3417305339, 2858903785, 3074075088, 4198538376, - 2370669734, 391683868, 3933465331, 2884128106, 1400216510, 1864988335, 2925823809, - 4133578805, 218104493, 2798390374, 1998600344, 1109346044, 1716435313, 415435111, - 1892535124, 2536620737, 1440068573, 481874870, 142128108, 329082740, 2077118779, - 2858045339, 4068162251, 115593872, 2644000449, 3550167006, 3728768059, 1772423685, - 2504624145, 2140118619, 1412607307, 4081166331, 154438218, 1652901877, 3804911318, - 3142799969, 3154073676, 3112018078, 2757873595, 50297646, 2855791484, 793851407, - 507785167, 263713916, 3324588195, 3291978358, 1584226715, 2546417990, 450747961, - 3455625012, 2706009093, 1788578194, 4030171132, 2610979903, 3835380965, 4269115622, - 2180305819, 2646894726, 716649335, 2607142354, 853428184, 2436760738, 4190071217, - 2754423535, 456808691, 4082489821, 741976046, 267559495, 1591532642, 2722205042, - 3203248679, 147312102, 2772368222, 1412987047, 1950543946, 1932341300, 898396308, - 1837129999, 3113914292, 428616392, 3141601915, 276087167, 1887389351, 757801450, - 963534966, 2745818074, 1442953464, 3802648347, 223728071, 229039300, 1338125300, - 3642174036, 2794462634, 2326349851, 206115203, 3577092599, 627103363, 552173564, - 4142604459, 1492461846, 583522272, 189323282, 1217612313, 73550248, 3552211807, - 2757269706, 2392210091, 3032922600, 3573904125, 810640644, 2632631469, 3085332665, - 3775619904, 2563291734, 922608790, 4043427793, 1803743084, 3112116579, 522940594, - 1785093944, 2613871529, 3810037031, 1765642390, 534554747, 3528050076, 2264349344, - 1680024716, 2112340059, 3387475367, 3295682653, 75966494, 3907378472, 2636491350, - 2176128529, 3574915532, 2372987046, 2604001339, 3893075234, 3495918635, 1280296085, - 534929913, 3660102574, 2030233939, 2759207091, 299776535, 987575186, 2566932710, - 1566181275, 3127827248, 3874691533, 3731513754, 2581732444, 279715551, 809690877, - 3100791084, 2185866850, 1495277039, 3417305339, 2858903785, 1310351481, 4198538376, - 2370669734, 391683868, 3933465331, 2749085130, 1400216510, 1864988335, 2925823809, - 4133578805, 3352814594, 2798390374, 1998600344, 1109346044, 1716435313, 1571752941, - 1892535124, 2536620737, 1440068573, 481874870, 2485033697, 329082740, 2077118779, - 2858045339, 4068162251, 3837440666, 2644000449, 3550167006, 3728768059, 1772423685, - 1176559812, 2140118619, 1412607307, 4081166331, 154438218, 2902622972, 3804911318, - 3142799969, 3154073676, 3112018078, 2403391233, 50297646, 2855791484, 793851407, - 507785167, 2351826747, 3324588195, 3291978358, 1584226715, 2546417990, 746876926, - 3455625012, 2706009093, 1788578194, 4030171132, 3779307353, 3835380965, 4269115622, - 2180305819, 2646894726, 2602235234, 2607142354, 853428184, 2436760738, 4190071217, - 2066757692, 456808691, 4082489821, 741976046, 267559495, 3001080633, 2722205042, - 3203248679, 147312102, 2772368222, 89950260, 1950543946, 1932341300, 898396308, - 1837129999, 947911286, 428616392, 3141601915, 276087167, 1887389351, 2583987247, - 963534966, 2745818074, 1442953464, 3802648347, 4229124441, 229039300, 1338125300, - 3642174036, 2794462634, 2472155633, 206115203, 3577092599, 627103363, 552173564, - 2586882739, 1492461846, 583522272, 189323282, 1217612313, 3501549884, 3552211807, - 2757269706, 2392210091, 3032922600, 740675778, 810640644, 2632631469, 3085332665, - 3775619904, 3643289881, 922608790, 4043427793, 1803743084, 3112116579, 2213337398, - 1785093944, 2613871529, 3810037031, 1765642390, 762472016, 3528050076, 2264349344, - 1680024716, 2112340059, 1372272974, 3295682653, 75966494, 3907378472, 2636491350, - 3117471955, 3574915532, 2372987046, 2604001339, 3893075234, 915576383, 1280296085, - 534929913, 3660102574, 2030233939, 346368350, 299776535, 987575186, 2566932710, - 1566181275, 3535223896, 3874691533, 3731513754, 2581732444, 279715551, 2456894951, - 3100791084, 2185866850, 1495277039, 3417305339, 1618871086, 1310351481, 4198538376, - 2370669734, 391683868, 2009676005, 2749085130, 1400216510, 1864988335, 2925823809, - 58955107, 3352814594, 2798390374, 1998600344, 1109346044, 3273979614, 1571752941, - 1892535124, 2536620737, 1440068573, 1174168447, 2485033697, 329082740, 2077118779, - 2858045339, 4062921629, 3837440666, 2644000449, 3550167006, 3728768059, 2642133401, - 1176559812, 2140118619, 1412607307, 4081166331, 3124905304, 2902622972, 3804911318, - 3142799969, 3154073676, 1449454613, 2403391233, 50297646, 2855791484, 793851407, - 3514201526, 2351826747, 3324588195, 3291978358, 1584226715, 3636681672, 746876926, - 3455625012, 2706009093, 1788578194, 3451519459, 3779307353, 3835380965, 4269115622, - 2180305819, 3987989524, 2602235234, 2607142354, 853428184, 2436760738, 2151617107, - 2066757692, 456808691, 4082489821, 741976046, 3590081269, 3001080633, 2722205042, - 3203248679, 147312102, 3432947806, 89950260, 1950543946, 1932341300, 898396308, - 3828432864, 947911286, 428616392, 3141601915, 276087167, 2517666433, 2583987247, - 963534966, 2745818074, 1442953464, 2223986807, 4229124441, 229039300, 1338125300, - 3642174036, 1053796945, 2472155633, 206115203, 3577092599, 627103363, 1113276084, - 2586882739, 1492461846, 583522272, 189323282, 1490604990, 3501549884, 3552211807, - 2757269706, 2392210091, 3545407532, 740675778, 810640644, 2632631469, 3085332665, - 755862267, 3643289881, 922608790, 4043427793, 1803743084, 1954166630, 2213337398, - 1785093944, 2613871529, 3810037031, 3042935707, 3162182177, 2791346436, 1901925289, - 863100941, 3367519168, 1972623238, 3664303070, 604922059, 3026817982, 1436412310, - 4096180631, 1597561857, 4206212303, 4127914332, 3228677359, 3985733659, 3597290113, - 4251197894, 3451370603, 609679338, 3360835257, 1372239885, 638572328, 3806422284, - 3974147336, 1804280837, 4209089291, 2021797469, 3557188838, 409727186, 2114649178, - 687702120, 2542445992, 1235991799, 460479179, 2008348175, 887884478, 3942327811, - 2999928223, 4171339789, 2286339235, 1293442231, 1575942850, 76122475, 1440527701, - 2006558403, 1544148172, 895899367, 681826913, 4094701935, 3995413790, 1027509154, - 2264990896, 1938238113, 213430250, 222469320, 609726517, 3581538106, 492802663, - 120480843, 1720004062, 1132674507, 911082758, 2909148131, 566658805, 3964114445, - 3483602509, 1793438750, 165562604, 3641830063, 2394205521, 3404874822, 1672998096, - 916151953, 1141264477, 3171661340, 3803396219, 3018337382, 1863902683, 2474641928, - 3250365071, 3897886220, 1219701051, 51332576, 1358614881, 1707407492, 3670647816, - 923357625, 343687395, 3991339686, 3913575403, 1267727936, 4001357856, 3820224848, - 2942896724, 3505936742, 1403285299, 1992762049, 567748449, 2202721585, 2781324216, - 1724850068, 2408314541, 3073975813, 3992810029, 2475242354, 540562053, 2185198943, - 3759352041, 3373885614, 1132999410, 1097554565, 4089342358, 3239542922, 2451748646, - 407290679, 3188103200, 1708016248, 26848241, 2796711130, 3090711568, 4068389322, - 3420916085, 3137567033, 2877819818, 22133454, 4629160, 3703695249, 1920151708, - 1175452162, 130015299, 3331834713, 1099225384, 689254331, 1851083761, 2654970209, - 3259297936, 3742819314, 3524284766, 2291819083, 3494031861, 16242889, 3545082774, - 1997878108, 777447699, 4244916543, 3508640253, 3782278393, 2107258964, 2139074576, - 1383217899, 2337934322, 3181899620, 1285955765, 2989610020, 3326862146, 1168587380, - 801203532, 3020809957, - }, - { - 3810471203, 1017064446, 1595207573, 441087832, 3326746890, 3294064431, 167972517, - 3625210015, 1011845006, 2980240819, 1778354660, 3041730987, 1598611350, 2015169745, - 2321724978, 3390812967, 2432904511, 113261909, 3957193232, 3806115908, 2965828929, - 2035392295, 3500116619, 2881232416, 1672212265, 1607201428, 425148945, 1262591961, - 2221781268, 4215047456, 2148245850, 2787488981, 1077262192, 2085467561, 3053954888, - 3584435116, 3013084787, 287099941, 1290407232, 4078552287, 2658945475, 4251530898, - 2403086478, 2884923598, 3545110453, 4105390090, 343200643, 3189888821, 4086304363, - 3466483195, 259435633, 2846377387, 497258846, 272775541, 985737911, 2957688879, - 2180784344, 3434619542, 3643384838, 2228652440, 3107480718, 2208729807, 596436263, - 3255120711, 3248886970, 519242965, 602979109, 1619614, 1391563565, 56262588, - 1584463910, 1849038201, 728022295, 848624947, 1813827408, 428214945, 1246345586, - 4213351865, 168985863, 456608054, 4277869380, 3886828599, 2264054549, 3110967170, - 3138175314, 2649164828, 3369378320, 3648350039, 3524848759, 1468470706, 3558859222, - 2669673235, 831851874, 4285651092, 4224147373, 1088456706, 231954609, 3118005852, - 225508069, 883105389, 856371341, 2001356578, 639336670, 2363501707, 3622399552, - 4024065226, 1093546838, 4263608561, 1852072422, 425195042, 2441102396, 296426333, - 384641750, 3559334435, 1757327033, 1016016207, 3595686646, 24777793, 623926105, - 2169195923, 1779396793, 646997837, 1459728476, 2644865980, 1994581089, 3956278544, - 919592580, 2153558858, 2029633394, 3837501009, 4016560170, 484838096, 3652199054, - 1971790561, 605295089, 637470291, 278970544, 3574824693, 295866521, 1755035156, - 2542341803, 1588716357, 1502596918, 4124554133, 3547049843, 1768033045, 1531734630, - 101448323, 3233017580, 1793222944, 3187853500, 186000900, 803444571, 2820254958, - 2009384608, 2384668855, 2222812920, 633608665, 2028480056, 1258028235, 545095949, - 3810471203, 1017064446, 1595207573, 441087832, 899068662, 3294064431, 167972517, - 3625210015, 1011845006, 3951305793, 1778354660, 3041730987, 1598611350, 2015169745, - 1885149424, 3390812967, 2432904511, 113261909, 3957193232, 3953443155, 2965828929, - 2035392295, 3500116619, 2881232416, 329153573, 1607201428, 425148945, 1262591961, - 2221781268, 78028761, 2148245850, 2787488981, 1077262192, 2085467561, 647235899, - 3584435116, 3013084787, 287099941, 1290407232, 1467385694, 2658945475, 4251530898, - 2403086478, 2884923598, 3489351040, 4105390090, 343200643, 3189888821, 4086304363, - 3521512280, 259435633, 2846377387, 497258846, 272775541, 1367093111, 2957688879, - 2180784344, 3434619542, 3643384838, 411877686, 3107480718, 2208729807, 596436263, - 3255120711, 584605030, 519242965, 602979109, 1619614, 1391563565, 3902518209, - 1584463910, 1849038201, 728022295, 848624947, 1932969318, 428214945, 1246345586, - 4213351865, 168985863, 2770345237, 4277869380, 3886828599, 2264054549, 3110967170, - 2953581033, 2649164828, 3369378320, 3648350039, 3524848759, 2380353977, 3558859222, - 2669673235, 831851874, 4285651092, 1214052447, 1088456706, 231954609, 3118005852, - 225508069, 1766983646, 856371341, 2001356578, 639336670, 2363501707, 1782816591, - 4024065226, 1093546838, 4263608561, 1852072422, 1149716600, 2441102396, 296426333, - 384641750, 3559334435, 2391309970, 1016016207, 3595686646, 24777793, 623926105, - 362098678, 1779396793, 646997837, 1459728476, 2644865980, 3238673748, 3956278544, - 919592580, 2153558858, 2029633394, 115778559, 4016560170, 484838096, 3652199054, - 1971790561, 737357475, 637470291, 278970544, 3574824693, 295866521, 3989745853, - 2542341803, 1588716357, 1502596918, 4124554133, 3016849744, 1768033045, 1531734630, - 101448323, 3233017580, 4157527581, 3187853500, 186000900, 803444571, 2820254958, - 1980528062, 2384668855, 2222812920, 633608665, 2028480056, 3166710281, 545095949, - 3810471203, 1017064446, 1595207573, 693962828, 899068662, 3294064431, 167972517, - 3625210015, 1486040398, 3951305793, 1778354660, 3041730987, 1598611350, 2859363132, - 1885149424, 3390812967, 2432904511, 113261909, 664880478, 3953443155, 2965828929, - 2035392295, 3500116619, 558081801, 329153573, 1607201428, 425148945, 1262591961, - 3716247699, 78028761, 2148245850, 2787488981, 1077262192, 4206362947, 647235899, - 3584435116, 3013084787, 287099941, 2536781098, 1467385694, 2658945475, 4251530898, - 2403086478, 3075072413, 3489351040, 4105390090, 343200643, 3189888821, 2540485172, - 3521512280, 259435633, 2846377387, 497258846, 2442427327, 1367093111, 2957688879, - 2180784344, 3434619542, 1593967423, 411877686, 3107480718, 2208729807, 596436263, - 1048686529, 584605030, 519242965, 602979109, 1619614, 2072745381, 3902518209, - 1584463910, 1849038201, 728022295, 846033949, 1932969318, 428214945, 1246345586, - 4213351865, 1066373275, 2770345237, 4277869380, 3886828599, 2264054549, 1877859690, - 2953581033, 2649164828, 3369378320, 3648350039, 2537763389, 2380353977, 3558859222, - 2669673235, 831851874, 522748140, 1214052447, 1088456706, 231954609, 3118005852, - 1381269315, 1766983646, 856371341, 2001356578, 639336670, 667275675, 1782816591, - 4024065226, 1093546838, 4263608561, 2057337961, 1149716600, 2441102396, 296426333, - 384641750, 340523210, 2391309970, 1016016207, 3595686646, 24777793, 3094832341, - 362098678, 1779396793, 646997837, 1459728476, 1169681568, 3238673748, 3956278544, - 919592580, 2153558858, 388335108, 115778559, 4016560170, 484838096, 3652199054, - 1764858181, 737357475, 637470291, 278970544, 3574824693, 3671458900, 3989745853, - 2542341803, 1588716357, 1502596918, 2102871406, 3016849744, 1768033045, 1531734630, - 101448323, 3964942332, 4157527581, 3187853500, 186000900, 803444571, 3425652083, - 1980528062, 2384668855, 2222812920, 633608665, 3035373876, 3166710281, 545095949, - 3810471203, 1017064446, 669282349, 693962828, 899068662, 3294064431, 167972517, - 2007256988, 1486040398, 3951305793, 1778354660, 3041730987, 2827768941, 2859363132, - 1885149424, 3390812967, 2432904511, 3700915653, 664880478, 3953443155, 2965828929, - 2035392295, 1461208330, 558081801, 329153573, 1607201428, 425148945, 1700881129, - 3716247699, 78028761, 2148245850, 2787488981, 2706775080, 4206362947, 647235899, - 3584435116, 3013084787, 2958545221, 2536781098, 1467385694, 2658945475, 4251530898, - 2241012567, 3075072413, 3489351040, 4105390090, 343200643, 490164649, 2540485172, - 3521512280, 259435633, 2846377387, 4073611831, 2442427327, 1367093111, 2957688879, - 2180784344, 1835510773, 1593967423, 411877686, 3107480718, 2208729807, 3306732468, - 1048686529, 584605030, 519242965, 602979109, 2978864605, 2072745381, 3902518209, - 1584463910, 1849038201, 3284115169, 846033949, 1932969318, 428214945, 1246345586, - 194166002, 1066373275, 2770345237, 4277869380, 3886828599, 1874087886, 1877859690, - 2953581033, 2649164828, 3369378320, 4145454028, 2537763389, 2380353977, 3558859222, - 2669673235, 739345884, 522748140, 1214052447, 1088456706, 231954609, 3605603781, - 1381269315, 1766983646, 856371341, 2001356578, 2049940324, 667275675, 1782816591, - 4024065226, 1093546838, 152524382, 2057337961, 1149716600, 2441102396, 296426333, - 3195130788, 340523210, 2391309970, 1016016207, 3595686646, 180492441, 3094832341, - 362098678, 1779396793, 646997837, 2458167607, 1169681568, 3238673748, 3956278544, - 919592580, 3421005218, 388335108, 115778559, 4016560170, 484838096, 2649676374, - 1764858181, 737357475, 637470291, 278970544, 2236401278, 3671458900, 3989745853, - 2542341803, 1588716357, 1241570134, 2102871406, 3016849744, 1768033045, 1531734630, - 1765654724, 3964942332, 4157527581, 3187853500, 186000900, 2189716659, 3425652083, - 1980528062, 2384668855, 2222812920, 3955466207, 2426547616, 3846752458, 3015538636, - 2342593365, 3613176865, 3484860981, 4278370194, 1979143878, 1159739458, 3714038404, - 396530346, 3276617756, 3293940597, 4050183149, 1418571985, 402563753, 2702853013, - 2289900621, 2267058511, 3482161995, 3375026019, 1988640267, 3674438074, 4124612310, - 1057883705, 434730475, 3210959778, 4102029739, 2140938750, 3176753074, 2356971512, - 3969685288, 1556275580, 2648433428, 3959375381, 478841344, 1496991528, 3309714981, - 569990368, 3660587501, 2550379574, 1177519842, 2652707373, 543943404, 1912551128, - 2278132032, 1484596780, 3570913985, 2982401320, 1413776035, 3177275459, 3036211597, - 1091740466, 3448424311, 1445187645, 3205024875, 3135795254, 823738729, 3742134467, - 4066657438, 1226311678, 2403605393, 537573634, 3457409768, 1940233423, 1761431281, - 1129427309, 2443661283, 3200814257, 4094866249, 2666869754, 604785127, 2213464116, - 3002782918, 468024929, 2490681314, 3666681384, 1583346053, 3049668798, 3592153237, - 2573082448, 3082970021, 1461796708, 832526980, 3728763274, 355291229, 4029588456, - 832358279, 2125298737, 3681181038, 3245535160, 1333342738, 1868897492, 446790068, - 1278093154, 2090118615, 4158925515, 4062165914, 822726809, 1154960183, 286518382, - 1170424276, 2554691236, 3674133415, 2765714969, 2330865375, 1908307334, 3537287082, - 410252600, 3977128218, 424210327, 2919071615, 2715518134, 64568844, 480972649, - 2488797168, 1302817038, 2213995265, 4229997295, 2200797852, 109368057, 3033807022, - 1907400078, 645977948, 1410909090, 3700787906, 3375062371, 629087832, 1344281719, - 4249981139, 3457543297, 1218556849, 864222854, 1458445945, 914545469, 3451164212, - 1088025757, 1129933985, 953788883, 2406172924, 170364546, 3505490646, 1027553899, - 2864067776, 436854871, 1342782209, 761167471, 2660173631, 4159507498, 4172028400, - 2442254644, 2110123720, 2315991253, 873066601, 1725470559, 3831299052, 678672031, - 1585431329, 3495750550, - }, - { - 1998393432, 2665389278, 3989307699, 3267631636, 3861682977, 3243522970, 1243992413, - 2200497260, 3821883021, 4187123083, 3451270040, 3044132745, 2101287249, 2340839784, - 227040990, 1724350416, 3228881240, 3123386528, 4279362126, 3098224464, 2635534069, - 3622906431, 206207480, 1894245533, 2152374527, 1011223653, 7271757, 2972858087, - 207942127, 3355362797, 2593296740, 174093751, 3713822176, 4212355586, 3335605224, - 1171716408, 2867257989, 1522213957, 2016192462, 4229688395, 2174928148, 1468226225, - 3938290338, 493240317, 3229423344, 2585475729, 3112454413, 1881171707, 2555908056, - 1997546352, 380428329, 3341885423, 3307510279, 3519476676, 3613100811, 2555826262, - 109341943, 2382715395, 3883409616, 1593551879, 2163678014, 3379783137, 2810374300, - 1516064864, 561144874, 316017838, 1899237567, 70857401, 3435185465, 4234661323, - 2580352177, 32879620, 4171670150, 1986234067, 3589478191, 2073132526, 2603712175, - 377997975, 2474419397, 3110698341, 812664089, 1778922726, 1686111212, 972784138, - 3936486236, 2711468739, 423435866, 1661961159, 802312780, 1868728136, 1760295704, - 3357409828, 215039860, 683184627, 4019111064, 3609261689, 2167554309, 1831085281, - 3389357802, 4193421575, 628277197, 2900207619, 993609502, 3429627083, 2636466084, - 3652352199, 1780133580, 1670387713, 4086070210, 4004540729, 783029246, 2165667566, - 1739001057, 377639972, 1102689625, 1945278055, 3941185940, 3685368326, 1881761572, - 2201338934, 801752, 2729497735, 492844690, 2998826141, 3844964457, 3679088359, - 2196391660, 4222269404, 357321611, 3727170055, 1819614072, 2348798457, 4294366646, - 1952884323, 3574345216, 2040734807, 232392443, 4183498179, 2614866055, 112120292, - 3624018350, 3340709877, 3097507723, 1268833488, 3570501956, 3338260086, 293812421, - 3683058169, 1147960351, 283731890, 2171233479, 1830154455, 4036602681, 1996981699, - 132803834, 40256165, 2158110401, 3575159090, 3196553513, 3559872992, 3402884675, - 1998393432, 2665389278, 3989307699, 3267631636, 3617519767, 3243522970, 1243992413, - 2200497260, 3821883021, 3715729085, 3451270040, 3044132745, 2101287249, 2340839784, - 3173635549, 1724350416, 3228881240, 3123386528, 4279362126, 2287520039, 2635534069, - 3622906431, 206207480, 1894245533, 96723416, 1011223653, 7271757, 2972858087, - 207942127, 1668335352, 2593296740, 174093751, 3713822176, 4212355586, 49226793, - 1171716408, 2867257989, 1522213957, 2016192462, 118712412, 2174928148, 1468226225, - 3938290338, 493240317, 3788174304, 2585475729, 3112454413, 1881171707, 2555908056, - 3351139844, 380428329, 3341885423, 3307510279, 3519476676, 1368994724, 2555826262, - 109341943, 2382715395, 3883409616, 1561509458, 2163678014, 3379783137, 2810374300, - 1516064864, 2313252274, 316017838, 1899237567, 70857401, 3435185465, 2585770746, - 2580352177, 32879620, 4171670150, 1986234067, 3317983509, 2073132526, 2603712175, - 377997975, 2474419397, 908728599, 812664089, 1778922726, 1686111212, 972784138, - 1992540005, 2711468739, 423435866, 1661961159, 802312780, 907108769, 1760295704, - 3357409828, 215039860, 683184627, 2806826652, 3609261689, 2167554309, 1831085281, - 3389357802, 2755692689, 628277197, 2900207619, 993609502, 3429627083, 3605915742, - 3652352199, 1780133580, 1670387713, 4086070210, 3717326627, 783029246, 2165667566, - 1739001057, 377639972, 2355216626, 1945278055, 3941185940, 3685368326, 1881761572, - 4024097818, 801752, 2729497735, 492844690, 2998826141, 2719601647, 3679088359, - 2196391660, 4222269404, 357321611, 1319821972, 1819614072, 2348798457, 4294366646, - 1952884323, 3573866689, 2040734807, 232392443, 4183498179, 2614866055, 440744432, - 3624018350, 3340709877, 3097507723, 1268833488, 224895395, 3338260086, 293812421, - 3683058169, 1147960351, 3433425235, 2171233479, 1830154455, 4036602681, 1996981699, - 2875889721, 40256165, 2158110401, 3575159090, 3196553513, 1094082574, 3402884675, - 1998393432, 2665389278, 3989307699, 4068940467, 3617519767, 3243522970, 1243992413, - 2200497260, 441678457, 3715729085, 3451270040, 3044132745, 2101287249, 2181502237, - 3173635549, 1724350416, 3228881240, 3123386528, 1968352124, 2287520039, 2635534069, - 3622906431, 206207480, 2065093599, 96723416, 1011223653, 7271757, 2972858087, - 1094044749, 1668335352, 2593296740, 174093751, 3713822176, 2887397643, 49226793, - 1171716408, 2867257989, 1522213957, 984348433, 118712412, 2174928148, 1468226225, - 3938290338, 2279430036, 3788174304, 2585475729, 3112454413, 1881171707, 4247636500, - 3351139844, 380428329, 3341885423, 3307510279, 2887754196, 1368994724, 2555826262, - 109341943, 2382715395, 2836761616, 1561509458, 2163678014, 3379783137, 2810374300, - 1635278016, 2313252274, 316017838, 1899237567, 70857401, 3481535811, 2585770746, - 2580352177, 32879620, 4171670150, 2248003250, 3317983509, 2073132526, 2603712175, - 377997975, 3286162818, 908728599, 812664089, 1778922726, 1686111212, 4024815755, - 1992540005, 2711468739, 423435866, 1661961159, 2257259057, 907108769, 1760295704, - 3357409828, 215039860, 3917391198, 2806826652, 3609261689, 2167554309, 1831085281, - 4238043113, 2755692689, 628277197, 2900207619, 993609502, 2036092353, 3605915742, - 3652352199, 1780133580, 1670387713, 118446953, 3717326627, 783029246, 2165667566, - 1739001057, 203160626, 2355216626, 1945278055, 3941185940, 3685368326, 546361979, - 4024097818, 801752, 2729497735, 492844690, 1023017124, 2719601647, 3679088359, - 2196391660, 4222269404, 621859651, 1319821972, 1819614072, 2348798457, 4294366646, - 1114888560, 3573866689, 2040734807, 232392443, 4183498179, 3959504609, 440744432, - 3624018350, 3340709877, 3097507723, 3613295037, 224895395, 3338260086, 293812421, - 3683058169, 1655305863, 3433425235, 2171233479, 1830154455, 4036602681, 3731384097, - 2875889721, 40256165, 2158110401, 3575159090, 1847744924, 1094082574, 3402884675, - 1998393432, 2665389278, 3781866777, 4068940467, 3617519767, 3243522970, 1243992413, - 2723708256, 441678457, 3715729085, 3451270040, 3044132745, 4013832842, 2181502237, - 3173635549, 1724350416, 3228881240, 2092292494, 1968352124, 2287520039, 2635534069, - 3622906431, 3186333458, 2065093599, 96723416, 1011223653, 7271757, 649658033, - 1094044749, 1668335352, 2593296740, 174093751, 4159420309, 2887397643, 49226793, - 1171716408, 2867257989, 2590077953, 984348433, 118712412, 2174928148, 1468226225, - 1065322711, 2279430036, 3788174304, 2585475729, 3112454413, 3932517386, 4247636500, - 3351139844, 380428329, 3341885423, 1285273904, 2887754196, 1368994724, 2555826262, - 109341943, 2318470582, 2836761616, 1561509458, 2163678014, 3379783137, 674658583, - 1635278016, 2313252274, 316017838, 1899237567, 2192372173, 3481535811, 2585770746, - 2580352177, 32879620, 300323274, 2248003250, 3317983509, 2073132526, 2603712175, - 3086543917, 3286162818, 908728599, 812664089, 1778922726, 2263290659, 4024815755, - 1992540005, 2711468739, 423435866, 819027349, 2257259057, 907108769, 1760295704, - 3357409828, 1142221093, 3917391198, 2806826652, 3609261689, 2167554309, 4108155875, - 4238043113, 2755692689, 628277197, 2900207619, 3041719497, 2036092353, 3605915742, - 3652352199, 1780133580, 2397410862, 118446953, 3717326627, 783029246, 2165667566, - 2721690354, 203160626, 2355216626, 1945278055, 3941185940, 2768842108, 546361979, - 4024097818, 801752, 2729497735, 4045063232, 1023017124, 2719601647, 3679088359, - 2196391660, 2666107451, 621859651, 1319821972, 1819614072, 2348798457, 3555102623, - 1114888560, 3573866689, 2040734807, 232392443, 3359040541, 3959504609, 440744432, - 3624018350, 3340709877, 1477919696, 3613295037, 224895395, 3338260086, 293812421, - 4210187101, 1655305863, 3433425235, 2171233479, 1830154455, 4150241150, 3731384097, - 2875889721, 40256165, 2158110401, 3350246687, 455561037, 2250400255, 3192153445, - 3258870230, 1500391873, 4142878334, 1155955691, 1483275844, 4189436981, 323745948, - 1976017426, 2804626790, 2717553615, 2315409034, 954508235, 3845175920, 3999878682, - 1247696432, 1743319509, 2998248398, 3694350012, 4072006361, 191306987, 2816321878, - 1324077734, 1083060006, 3406855480, 1619622379, 2160350, 3302238190, 3368021261, - 3685228564, 3863934685, 771728612, 854205233, 2304696695, 421449207, 1265752117, - 3852292419, 305345788, 1540622105, 1904883477, 833469256, 134406680, 3012455058, - 4035477953, 2925192459, 1559200592, 3851612860, 718484562, 1377960276, 1586892849, - 1361298269, 3417917896, 1281324499, 1012538763, 1350578667, 3946475598, 2982283954, - 3548792804, 284542749, 1194648577, 3087899716, 3966595444, 2088330116, 3641652062, - 327128507, 593906557, 1092448919, 2459189516, 4053392241, 3356198248, 2352376508, - 470648997, 1017041256, 3234172340, 3928191489, 3266226858, 4219289150, 1229098319, - 4275351308, 2720777751, 3566728718, 638322822, 2369792461, 2869492261, 3120083828, - 1890399556, 3309991008, 3785452464, 4128660314, 3726791982, 167177896, 461294981, - 3988638998, 2937794823, 3981029822, 1111681402, 2015965721, 7261806, 2669786265, - 1083582734, 3270228881, 3892235938, 2695872715, 4246051290, 3214293333, 343604199, - 3215604888, 661024127, 2931754053, 3787840039, 2053363765, 363432336, 112334132, - 2871797223, 138911320, 3981126938, 2027332192, 1804730644, 590150270, 641538574, - 6802174, 3551446076, 3908480472, 1004531022, 2097228524, 1919074232, 154482247, - 121437972, 1215661323, 1178068273, 1097220699, 2823681422, 262636065, 2943371149, - 1768780720, 3866040605, 1855991583, 3988248086, 629223947, 3380612330, 3552916762, - 197596340, 573801686, 2049230598, 2910471867, 2686314264, 1726228846, 3516983332, - 726840185, 1241204222, 2237574317, 70568042, 1932610099, 2221862221, 1510378092, - 4050391637, 4077539568, - }, - { - 3872117793, 803220151, 70843412, 1661103032, 1976811457, 2186373604, 564259972, - 1475436923, 2260980893, 4245534505, 1075107552, 3692990573, 370098873, 4045905424, - 2420395420, 2332395402, 207483321, 622317750, 3004242500, 833623111, 3151161301, - 1629139881, 352228793, 2439953368, 3183333619, 2703537080, 3218957129, 3164695888, - 1741641842, 963394141, 4241612717, 1034476784, 2035880432, 3977821313, 1543311495, - 3010014356, 1638490901, 2364265378, 3420329129, 333361555, 1133565821, 1450937015, - 616059115, 3216393887, 3041978455, 3990855695, 1238628750, 512746184, 3256670217, - 1616316512, 2791405051, 93474487, 2865892488, 1901471398, 2930857966, 2178431077, - 2325598341, 3189256113, 1302432091, 808592927, 2945846737, 3487931071, 2018175258, - 752981057, 1097082589, 1307115286, 175147508, 3611190164, 850238914, 3318706185, - 199743319, 328621708, 3183670050, 3609998315, 4075306371, 3554549067, 2119566187, - 1498503842, 1261870696, 2216745780, 950288337, 1117344941, 2150569143, 2899286760, - 1594966374, 888858617, 35840654, 2829539211, 2511395669, 3607190544, 3278412778, - 2249895907, 1320858068, 3576889788, 266766189, 1522426851, 1903494122, 1928370573, - 2628132591, 3322025904, 220280169, 433606853, 1428961479, 986074592, 2128892987, - 467697583, 1616913929, 325674890, 444442578, 649166208, 1689709565, 1493452467, - 2222122038, 121114616, 2134348225, 3512035688, 1283058921, 4230441398, 3701238559, - 337534132, 1418548715, 1190006478, 500654385, 1766924757, 1944680746, 940574010, - 922744002, 186142284, 3131162902, 1693891092, 3031823448, 2143051534, 1429025284, - 1487843160, 3606456133, 2079235652, 2447285474, 2669283767, 3232117829, 2490054343, - 3225501736, 2911340385, 382319031, 1516937595, 622543191, 1388990570, 1749179860, - 1924483707, 2593474505, 472539197, 122872799, 2586347240, 880588515, 4046335279, - 1712182607, 4270737941, 1336703451, 3390078162, 382216945, 3733326081, 460422073, - 3872117793, 803220151, 70843412, 1661103032, 250339760, 2186373604, 564259972, - 1475436923, 2260980893, 657986735, 1075107552, 3692990573, 370098873, 4045905424, - 3201950123, 2332395402, 207483321, 622317750, 3004242500, 3732213278, 3151161301, - 1629139881, 352228793, 2439953368, 3572618926, 2703537080, 3218957129, 3164695888, - 1741641842, 685933373, 4241612717, 1034476784, 2035880432, 3977821313, 3855995181, - 3010014356, 1638490901, 2364265378, 3420329129, 2355603679, 1133565821, 1450937015, - 616059115, 3216393887, 1733804102, 3990855695, 1238628750, 512746184, 3256670217, - 2651059231, 2791405051, 93474487, 2865892488, 1901471398, 2113461797, 2178431077, - 2325598341, 3189256113, 1302432091, 2986990416, 2945846737, 3487931071, 2018175258, - 752981057, 2428033310, 1307115286, 175147508, 3611190164, 850238914, 1033628405, - 199743319, 328621708, 3183670050, 3609998315, 4024297327, 3554549067, 2119566187, - 1498503842, 1261870696, 290361143, 950288337, 1117344941, 2150569143, 2899286760, - 168826051, 888858617, 35840654, 2829539211, 2511395669, 2890882060, 3278412778, - 2249895907, 1320858068, 3576889788, 1794920145, 1522426851, 1903494122, 1928370573, - 2628132591, 1251697758, 220280169, 433606853, 1428961479, 986074592, 2707115661, - 467697583, 1616913929, 325674890, 444442578, 122781510, 1689709565, 1493452467, - 2222122038, 121114616, 3425723636, 3512035688, 1283058921, 4230441398, 3701238559, - 1646155473, 1418548715, 1190006478, 500654385, 1766924757, 3920475367, 940574010, - 922744002, 186142284, 3131162902, 54639113, 3031823448, 2143051534, 1429025284, - 1487843160, 4152687885, 2079235652, 2447285474, 2669283767, 3232117829, 1601035152, - 3225501736, 2911340385, 382319031, 1516937595, 3508441679, 1388990570, 1749179860, - 1924483707, 2593474505, 2835403456, 122872799, 2586347240, 880588515, 4046335279, - 2958058367, 4270737941, 1336703451, 3390078162, 382216945, 450517882, 460422073, - 3872117793, 803220151, 70843412, 2066343874, 250339760, 2186373604, 564259972, - 1475436923, 1683787449, 657986735, 1075107552, 3692990573, 370098873, 2615082840, - 3201950123, 2332395402, 207483321, 622317750, 2655424371, 3732213278, 3151161301, - 1629139881, 352228793, 3236724760, 3572618926, 2703537080, 3218957129, 3164695888, - 9775065, 685933373, 4241612717, 1034476784, 2035880432, 1621920075, 3855995181, - 3010014356, 1638490901, 2364265378, 1509475888, 2355603679, 1133565821, 1450937015, - 616059115, 3666188236, 1733804102, 3990855695, 1238628750, 512746184, 3900473826, - 2651059231, 2791405051, 93474487, 2865892488, 222759186, 2113461797, 2178431077, - 2325598341, 3189256113, 2505499508, 2986990416, 2945846737, 3487931071, 2018175258, - 2766733928, 2428033310, 1307115286, 175147508, 3611190164, 1909211603, 1033628405, - 199743319, 328621708, 3183670050, 1680331218, 4024297327, 3554549067, 2119566187, - 1498503842, 3516256046, 290361143, 950288337, 1117344941, 2150569143, 3182619063, - 168826051, 888858617, 35840654, 2829539211, 645798943, 2890882060, 3278412778, - 2249895907, 1320858068, 1436708568, 1794920145, 1522426851, 1903494122, 1928370573, - 3693049252, 1251697758, 220280169, 433606853, 1428961479, 3724415861, 2707115661, - 467697583, 1616913929, 325674890, 1448052253, 122781510, 1689709565, 1493452467, - 2222122038, 2177448198, 3425723636, 3512035688, 1283058921, 4230441398, 3050940272, - 1646155473, 1418548715, 1190006478, 500654385, 1106232, 3920475367, 940574010, - 922744002, 186142284, 4144806511, 54639113, 3031823448, 2143051534, 1429025284, - 2067453848, 4152687885, 2079235652, 2447285474, 2669283767, 428527087, 1601035152, - 3225501736, 2911340385, 382319031, 2565464472, 3508441679, 1388990570, 1749179860, - 1924483707, 1737735237, 2835403456, 122872799, 2586347240, 880588515, 597822462, - 2958058367, 4270737941, 1336703451, 3390078162, 2532634475, 450517882, 460422073, - 3872117793, 803220151, 801648827, 2066343874, 250339760, 2186373604, 564259972, - 3417948976, 1683787449, 657986735, 1075107552, 3692990573, 2235306692, 2615082840, - 3201950123, 2332395402, 207483321, 699310933, 2655424371, 3732213278, 3151161301, - 1629139881, 1152704006, 3236724760, 3572618926, 2703537080, 3218957129, 2726926336, - 9775065, 685933373, 4241612717, 1034476784, 2398119652, 1621920075, 3855995181, - 3010014356, 1638490901, 252854480, 1509475888, 2355603679, 1133565821, 1450937015, - 2655911639, 3666188236, 1733804102, 3990855695, 1238628750, 1115900497, 3900473826, - 2651059231, 2791405051, 93474487, 1862985957, 222759186, 2113461797, 2178431077, - 2325598341, 4179075132, 2505499508, 2986990416, 2945846737, 3487931071, 564667776, - 2766733928, 2428033310, 1307115286, 175147508, 1759077815, 1909211603, 1033628405, - 199743319, 328621708, 2552816198, 1680331218, 4024297327, 3554549067, 2119566187, - 2267805778, 3516256046, 290361143, 950288337, 1117344941, 2897506172, 3182619063, - 168826051, 888858617, 35840654, 2035476068, 645798943, 2890882060, 3278412778, - 2249895907, 3278449102, 1436708568, 1794920145, 1522426851, 1903494122, 1500763736, - 3693049252, 1251697758, 220280169, 433606853, 3914497854, 3724415861, 2707115661, - 467697583, 1616913929, 918435305, 1448052253, 122781510, 1689709565, 1493452467, - 609575172, 2177448198, 3425723636, 3512035688, 1283058921, 3661181550, 3050940272, - 1646155473, 1418548715, 1190006478, 1047301661, 1106232, 3920475367, 940574010, - 922744002, 2510633517, 4144806511, 54639113, 3031823448, 2143051534, 3242814908, - 2067453848, 4152687885, 2079235652, 2447285474, 736638210, 428527087, 1601035152, - 3225501736, 2911340385, 1849570436, 2565464472, 3508441679, 1388990570, 1749179860, - 84517579, 1737735237, 2835403456, 122872799, 2586347240, 4002124614, 597822462, - 2958058367, 4270737941, 1336703451, 3078170472, 1186434751, 700631413, 1497890797, - 1195347450, 2560167391, 1116697259, 1254138573, 747913260, 240954704, 3107512667, - 360584144, 3422778960, 3516528389, 3301260366, 1254513537, 122269053, 1579582456, - 873334104, 3918835024, 1731872444, 1974410416, 1811172641, 4172523062, 4092675777, - 4124987343, 1936078756, 1757348689, 2694415512, 128641660, 1744777659, 3173116729, - 983733754, 1430789547, 701906842, 3367232568, 3266433501, 3572590347, 1453272962, - 2106553114, 993786201, 2149441250, 1295181065, 2962229026, 3709052556, 3255608941, - 3677730029, 483873127, 102227292, 2626265293, 2018984578, 2266388762, 1191709548, - 2152725916, 583672623, 2230473473, 1995194269, 1740347812, 2558095372, 3070195183, - 3023333227, 2497183195, 1908755188, 773027539, 3646876518, 2272586839, 493318726, - 2107067517, 2000805278, 2530829636, 3183628745, 677565332, 1497629423, 82094920, - 2214054433, 2635367545, 470855467, 2184853389, 2942188934, 188335670, 3656661644, - 1883526235, 3990873975, 1490784356, 4047548172, 3149642641, 3289988179, 2590918909, - 2893039564, 2350687346, 4252624874, 15372456, 1614496594, 2364847678, 2604511825, - 422365460, 4195174772, 3266964836, 2008671995, 54038434, 781948549, 1276017666, - 2756376612, 2436825273, 1711863836, 3541493950, 3821378841, 1007557618, 345375815, - 2081905201, 2227278118, 1185927141, 1082173792, 3567361925, 1940465859, 541632942, - 1830210248, 3757851982, 775883450, 1666577465, 1004944607, 878440834, 2146344131, - 4195798476, 370164841, 3649112729, 37066142, 2311278904, 1935745497, 2304799402, - 4107299626, 1348526232, 2473609635, 3284032699, 2374292786, 1762329186, 857978496, - 1039346432, 2621413355, 29961014, 3582263091, 4268542513, 3890612190, 3096173646, - 2026544230, 3856142618, 2347115934, 319800326, 3255916105, 2430273059, 823505311, - 874255188, 1401925393, 4203707857, 4259159566, 2606881118, 1978288664, 1447576038, - 3860341401, 412510348, - }, - { - 4052471963, 683640040, 3043876021, 3466644483, 4222418025, 3035140128, 1466027937, - 18198088, 3410320851, 3040963721, 488404231, 3157371815, 769336092, 3240417718, - 808582581, 2075839263, 835026995, 3123726486, 3284240985, 1898453053, 3606056482, - 512836002, 2715428547, 4182302879, 1644882480, 3160187826, 390292489, 980889545, - 2776206633, 2482799995, 617042280, 3501667414, 689451808, 497018701, 238525753, - 3890163301, 896679896, 1544533015, 3412477225, 3116575138, 4250402651, 3990990746, - 819056741, 1459334146, 158377590, 3444755752, 8230450, 1378706455, 684191332, - 3217423797, 2842520097, 1631477948, 2591254230, 959644473, 1020694107, 1748401915, - 3452514983, 3892766171, 1227786994, 2086180800, 2394613217, 2091953150, 870094953, - 2306851481, 571550601, 488878212, 873197214, 2630100528, 2067476907, 2162307009, - 2026119728, 115875280, 2905867426, 248774881, 3110900450, 2236032812, 1888510348, - 708001855, 996960491, 3514196956, 1407967546, 1826568876, 3659618284, 2614104317, - 2230066308, 1055135881, 2537437343, 1858044413, 2608594891, 2750681169, 3241939420, - 3966440877, 2375002886, 2417753441, 1405878685, 1081133199, 1496940727, 382467042, - 2745477587, 1209424459, 811187075, 1385604734, 2623887355, 3443875720, 394141555, - 4142998949, 4195414618, 1489846841, 2253433808, 1171450286, 84131191, 4387588, - 2641405140, 3525405389, 3273000909, 423660319, 2366546732, 3698878607, 2161119729, - 4263629085, 3029102089, 2692507376, 3266869596, 1658012061, 1960169440, 1002311379, - 3724446882, 2004188516, 999513506, 2200093802, 4141037460, 351865836, 412875013, - 1535823315, 3880657632, 3109944987, 3207577548, 3462087941, 584875517, 2635241084, - 3834145971, 1693380373, 3524443732, 934775214, 1960588847, 2226778032, 1044609478, - 12199016, 1120582000, 226430296, 665553142, 2570993348, 1685535237, 3325420136, - 3925248326, 2855346376, 1205558328, 808835317, 3295908896, 4170076136, 2438272365, - 4052471963, 683640040, 3043876021, 3466644483, 1385549869, 3035140128, 1466027937, - 18198088, 3410320851, 2171386836, 488404231, 3157371815, 769336092, 3240417718, - 2921774554, 2075839263, 835026995, 3123726486, 3284240985, 72352110, 3606056482, - 512836002, 2715428547, 4182302879, 3869483469, 3160187826, 390292489, 980889545, - 2776206633, 1385691983, 617042280, 3501667414, 689451808, 497018701, 2600411809, - 3890163301, 896679896, 1544533015, 3412477225, 356556378, 4250402651, 3990990746, - 819056741, 1459334146, 199003993, 3444755752, 8230450, 1378706455, 684191332, - 1750733272, 2842520097, 1631477948, 2591254230, 959644473, 2113375576, 1748401915, - 3452514983, 3892766171, 1227786994, 275473920, 2394613217, 2091953150, 870094953, - 2306851481, 897057645, 488878212, 873197214, 2630100528, 2067476907, 944114068, - 2026119728, 115875280, 2905867426, 248774881, 989201307, 2236032812, 1888510348, - 708001855, 996960491, 2121706374, 1407967546, 1826568876, 3659618284, 2614104317, - 2931815032, 1055135881, 2537437343, 1858044413, 2608594891, 1423973935, 3241939420, - 3966440877, 2375002886, 2417753441, 2514473440, 1081133199, 1496940727, 382467042, - 2745477587, 81977310, 811187075, 1385604734, 2623887355, 3443875720, 2100629879, - 4142998949, 4195414618, 1489846841, 2253433808, 337182869, 84131191, 4387588, - 2641405140, 3525405389, 661876463, 423660319, 2366546732, 3698878607, 2161119729, - 309510684, 3029102089, 2692507376, 3266869596, 1658012061, 11119541, 1002311379, - 3724446882, 2004188516, 999513506, 3486722046, 4141037460, 351865836, 412875013, - 1535823315, 2818130700, 3109944987, 3207577548, 3462087941, 584875517, 322875622, - 3834145971, 1693380373, 3524443732, 934775214, 3879414752, 2226778032, 1044609478, - 12199016, 1120582000, 4207259464, 665553142, 2570993348, 1685535237, 3325420136, - 553869152, 2855346376, 1205558328, 808835317, 3295908896, 470585896, 2438272365, - 4052471963, 683640040, 3043876021, 1588419572, 1385549869, 3035140128, 1466027937, - 18198088, 363815288, 2171386836, 488404231, 3157371815, 769336092, 2464768302, - 2921774554, 2075839263, 835026995, 3123726486, 4229246330, 72352110, 3606056482, - 512836002, 2715428547, 319830805, 3869483469, 3160187826, 390292489, 980889545, - 2966401462, 1385691983, 617042280, 3501667414, 689451808, 4047377762, 2600411809, - 3890163301, 896679896, 1544533015, 764316452, 356556378, 4250402651, 3990990746, - 819056741, 965331966, 199003993, 3444755752, 8230450, 1378706455, 51902971, - 1750733272, 2842520097, 1631477948, 2591254230, 426039404, 2113375576, 1748401915, - 3452514983, 3892766171, 2833368447, 275473920, 2394613217, 2091953150, 870094953, - 3524323828, 897057645, 488878212, 873197214, 2630100528, 3939852929, 944114068, - 2026119728, 115875280, 2905867426, 3192643919, 989201307, 2236032812, 1888510348, - 708001855, 2166012172, 2121706374, 1407967546, 1826568876, 3659618284, 135277096, - 2931815032, 1055135881, 2537437343, 1858044413, 2588429924, 1423973935, 3241939420, - 3966440877, 2375002886, 2477142003, 2514473440, 1081133199, 1496940727, 382467042, - 1760129281, 81977310, 811187075, 1385604734, 2623887355, 4070531513, 2100629879, - 4142998949, 4195414618, 1489846841, 2688068550, 337182869, 84131191, 4387588, - 2641405140, 1837403234, 661876463, 423660319, 2366546732, 3698878607, 2916121190, - 309510684, 3029102089, 2692507376, 3266869596, 303422295, 11119541, 1002311379, - 3724446882, 2004188516, 2652711421, 3486722046, 4141037460, 351865836, 412875013, - 113149471, 2818130700, 3109944987, 3207577548, 3462087941, 1443140792, 322875622, - 3834145971, 1693380373, 3524443732, 901891935, 3879414752, 2226778032, 1044609478, - 12199016, 2213168758, 4207259464, 665553142, 2570993348, 1685535237, 1114492412, - 553869152, 2855346376, 1205558328, 808835317, 3266626294, 470585896, 2438272365, - 4052471963, 683640040, 3581539398, 1588419572, 1385549869, 3035140128, 1466027937, - 4075470388, 363815288, 2171386836, 488404231, 3157371815, 2759472233, 2464768302, - 2921774554, 2075839263, 835026995, 1030654310, 4229246330, 72352110, 3606056482, - 512836002, 961858496, 319830805, 3869483469, 3160187826, 390292489, 2366221117, - 2966401462, 1385691983, 617042280, 3501667414, 295865937, 4047377762, 2600411809, - 3890163301, 896679896, 21714884, 764316452, 356556378, 4250402651, 3990990746, - 1012967081, 965331966, 199003993, 3444755752, 8230450, 1255302023, 51902971, - 1750733272, 2842520097, 1631477948, 2321320272, 426039404, 2113375576, 1748401915, - 3452514983, 2847013518, 2833368447, 275473920, 2394613217, 2091953150, 1250695522, - 3524323828, 897057645, 488878212, 873197214, 1452317325, 3939852929, 944114068, - 2026119728, 115875280, 4061820350, 3192643919, 989201307, 2236032812, 1888510348, - 3986446165, 2166012172, 2121706374, 1407967546, 1826568876, 2910745432, 135277096, - 2931815032, 1055135881, 2537437343, 2976455307, 2588429924, 1423973935, 3241939420, - 3966440877, 2418897705, 2477142003, 2514473440, 1081133199, 1496940727, 1321648771, - 1760129281, 81977310, 811187075, 1385604734, 17644628, 4070531513, 2100629879, - 4142998949, 4195414618, 2697310527, 2688068550, 337182869, 84131191, 4387588, - 1724191700, 1837403234, 661876463, 423660319, 2366546732, 693430992, 2916121190, - 309510684, 3029102089, 2692507376, 3917396098, 303422295, 11119541, 1002311379, - 3724446882, 841468294, 2652711421, 3486722046, 4141037460, 351865836, 1733384185, - 113149471, 2818130700, 3109944987, 3207577548, 2326233100, 1443140792, 322875622, - 3834145971, 1693380373, 1580706359, 901891935, 3879414752, 2226778032, 1044609478, - 3805470822, 2213168758, 4207259464, 665553142, 2570993348, 3406548636, 1114492412, - 553869152, 2855346376, 1205558328, 4287831475, 1329654114, 2347235746, 2477803138, - 2962371859, 3610024283, 4197266903, 1162294689, 1746713323, 2815058477, 2152552186, - 4214791071, 2382522482, 3713914466, 3974765132, 348354997, 1670276150, 2173074887, - 381736894, 3866219357, 1919366695, 3635118824, 2298653261, 3534332682, 1627699897, - 4168636618, 3787938690, 2144231271, 2067679462, 217001062, 2308928337, 1620415125, - 3526559172, 749451561, 2456947371, 3543607786, 1893824735, 962598819, 2332807164, - 1691114891, 2543992233, 2914780639, 1610287145, 1700599697, 3185174208, 552323208, - 2367242224, 3797136972, 3415066418, 2468049249, 1677937401, 40445671, 2886682530, - 2585715434, 194932329, 2994003812, 3099556382, 680852222, 135838738, 1371063256, - 995454898, 3754526418, 803635682, 634588682, 3869250783, 2442285521, 1455637058, - 570621479, 2512681851, 1220136924, 750260121, 2909903038, 1582019728, 955115170, - 1608265445, 2157390890, 2303678604, 1568394164, 831914289, 1971271392, 1294799854, - 1489945167, 442427880, 1305083700, 1211218668, 2380073713, 2798736785, 2193524273, - 3227386915, 1636588977, 3612937642, 435113647, 1591761830, 536210039, 2475747073, - 4223795480, 1786737271, 1444661534, 3249410301, 3333695212, 4169107188, 3280638635, - 702659930, 1444127970, 225340755, 2255629368, 746584456, 3965677674, 2671132955, - 2080717656, 2145343886, 3712441197, 368422910, 1297685674, 4076123901, 26214470, - 2948764826, 40503299, 1198194334, 2100063637, 1966331612, 2189582064, 2064696934, - 1797550642, 3469793941, 2868963812, 851437659, 240918534, 365060070, 3530600064, - 39695324, 1753898837, 1286976449, 3131971360, 2406485219, 3365373704, 3224113403, - 1651742834, 587601940, 1574206085, 3739575036, 1413669616, 38172232, 293127854, - 4126190109, 1891744061, 787878666, 456643669, 4228710325, 2025132037, 1492133135, - 3122840937, 969442079, 3272420439, 3836126369, 1877655562, 2766212758, 3867984746, - 3348077578, 1841216706, - }, - { - 1676507466, 1017841240, 2992644565, 476936158, 2468072723, 3113105154, 1154120402, - 460889625, 1942263502, 1761593999, 3020908939, 3078194866, 310971889, 1644896012, - 3756044556, 3549937583, 3710822994, 3554313733, 2174654326, 4251063242, 2340485150, - 950951909, 4288936895, 3744348848, 706644559, 1085927825, 1595992020, 3288724966, - 1367247946, 2950094970, 3925419886, 2628739022, 2528254629, 3582224789, 3907345559, - 3373329273, 4255542251, 1185418446, 4018656113, 2854344020, 1381160022, 3642438773, - 4284399225, 935780030, 4142412144, 1263328494, 1154237693, 2684443667, 3067549398, - 4253090033, 1251034970, 1874233020, 3222830495, 3866931656, 286048055, 3146635362, - 1436483376, 2821876495, 3927829532, 2648886905, 2142862852, 1368937545, 2647327844, - 1072219385, 2621337706, 3543274652, 911792564, 1204178178, 4127214323, 2821691380, - 3101998294, 730811902, 1989156224, 2872353003, 278290276, 1390223786, 2657819643, - 552729795, 1736270535, 2759207116, 1897013739, 3657020278, 1387364861, 1966588302, - 1049203087, 486446521, 3675999281, 714737345, 686837530, 85509025, 3609089773, - 2117061768, 3935682560, 3859508784, 4105287041, 1808988481, 83680601, 1464326680, - 1657693523, 3318062731, 1391154023, 234460119, 3551348221, 2245244809, 3635923821, - 2814385745, 3497626257, 916790795, 245338628, 2514528380, 3711787525, 2239286063, - 1054058916, 3963706010, 3176203796, 2230543409, 2173597546, 3786733892, 1396036965, - 1038764273, 2032556038, 3216540537, 3298170974, 1008892557, 141155464, 1863766055, - 3931110690, 191299053, 2019139711, 2409528317, 739418419, 1377144055, 2876702705, - 3911939673, 1197696462, 2814009721, 600813233, 1535885024, 1486280357, 3084650548, - 2324695947, 2293284974, 2036339249, 3465600153, 1624446108, 327866771, 3356772175, - 1826625240, 1947102360, 3661848193, 1421374867, 3228945021, 1358646008, 1067180174, - 2190741258, 643362354, 109899594, 2064362635, 3249674888, 2165543887, 4180291913, - 1676507466, 1017841240, 2992644565, 476936158, 3608467942, 3113105154, 1154120402, - 460889625, 1942263502, 1862994005, 3020908939, 3078194866, 310971889, 1644896012, - 693774191, 3549937583, 3710822994, 3554313733, 2174654326, 37658897, 2340485150, - 950951909, 4288936895, 3744348848, 2258231402, 1085927825, 1595992020, 3288724966, - 1367247946, 3850509554, 3925419886, 2628739022, 2528254629, 3582224789, 3124287811, - 3373329273, 4255542251, 1185418446, 4018656113, 1989726178, 1381160022, 3642438773, - 4284399225, 935780030, 3622052196, 1263328494, 1154237693, 2684443667, 3067549398, - 2786224913, 1251034970, 1874233020, 3222830495, 3866931656, 1529490307, 3146635362, - 1436483376, 2821876495, 3927829532, 979247444, 2142862852, 1368937545, 2647327844, - 1072219385, 294065371, 3543274652, 911792564, 1204178178, 4127214323, 103582737, - 3101998294, 730811902, 1989156224, 2872353003, 1885087777, 1390223786, 2657819643, - 552729795, 1736270535, 3325206451, 1897013739, 3657020278, 1387364861, 1966588302, - 2117065739, 486446521, 3675999281, 714737345, 686837530, 3946214694, 3609089773, - 2117061768, 3935682560, 3859508784, 2916136885, 1808988481, 83680601, 1464326680, - 1657693523, 3438751781, 1391154023, 234460119, 3551348221, 2245244809, 3948410079, - 2814385745, 3497626257, 916790795, 245338628, 1767303496, 3711787525, 2239286063, - 1054058916, 3963706010, 4140631909, 2230543409, 2173597546, 3786733892, 1396036965, - 1116033475, 2032556038, 3216540537, 3298170974, 1008892557, 667272562, 1863766055, - 3931110690, 191299053, 2019139711, 272901326, 739418419, 1377144055, 2876702705, - 3911939673, 3839312742, 2814009721, 600813233, 1535885024, 1486280357, 4256065219, - 2324695947, 2293284974, 2036339249, 3465600153, 1215859603, 327866771, 3356772175, - 1826625240, 1947102360, 4240407984, 1421374867, 3228945021, 1358646008, 1067180174, - 4100357988, 643362354, 109899594, 2064362635, 3249674888, 2898852084, 4180291913, - 1676507466, 1017841240, 2992644565, 1569683812, 3608467942, 3113105154, 1154120402, - 460889625, 966040649, 1862994005, 3020908939, 3078194866, 310971889, 786634113, - 693774191, 3549937583, 3710822994, 3554313733, 1578429713, 37658897, 2340485150, - 950951909, 4288936895, 2528123823, 2258231402, 1085927825, 1595992020, 3288724966, - 3544041088, 3850509554, 3925419886, 2628739022, 2528254629, 2562145937, 3124287811, - 3373329273, 4255542251, 1185418446, 3693565710, 1989726178, 1381160022, 3642438773, - 4284399225, 3271478204, 3622052196, 1263328494, 1154237693, 2684443667, 3615401444, - 2786224913, 1251034970, 1874233020, 3222830495, 2572413057, 1529490307, 3146635362, - 1436483376, 2821876495, 3993894153, 979247444, 2142862852, 1368937545, 2647327844, - 1353904396, 294065371, 3543274652, 911792564, 1204178178, 3165709748, 103582737, - 3101998294, 730811902, 1989156224, 893293786, 1885087777, 1390223786, 2657819643, - 552729795, 3388458110, 3325206451, 1897013739, 3657020278, 1387364861, 3025318046, - 2117065739, 486446521, 3675999281, 714737345, 2085926890, 3946214694, 3609089773, - 2117061768, 3935682560, 868009118, 2916136885, 1808988481, 83680601, 1464326680, - 797410789, 3438751781, 1391154023, 234460119, 3551348221, 4068940987, 3948410079, - 2814385745, 3497626257, 916790795, 3722456098, 1767303496, 3711787525, 2239286063, - 1054058916, 2030352819, 4140631909, 2230543409, 2173597546, 3786733892, 3211336683, - 1116033475, 2032556038, 3216540537, 3298170974, 2589589144, 667272562, 1863766055, - 3931110690, 191299053, 1139480458, 272901326, 739418419, 1377144055, 2876702705, - 1954361769, 3839312742, 2814009721, 600813233, 1535885024, 3587775605, 4256065219, - 2324695947, 2293284974, 2036339249, 1534849280, 1215859603, 327866771, 3356772175, - 1826625240, 720372669, 4240407984, 1421374867, 3228945021, 1358646008, 3409069246, - 4100357988, 643362354, 109899594, 2064362635, 4243434294, 2898852084, 4180291913, - 1676507466, 1017841240, 3243922356, 1569683812, 3608467942, 3113105154, 1154120402, - 1479311403, 966040649, 1862994005, 3020908939, 3078194866, 1556392996, 786634113, - 693774191, 3549937583, 3710822994, 920664071, 1578429713, 37658897, 2340485150, - 950951909, 740197415, 2528123823, 2258231402, 1085927825, 1595992020, 2580760267, - 3544041088, 3850509554, 3925419886, 2628739022, 3867556156, 2562145937, 3124287811, - 3373329273, 4255542251, 3185271749, 3693565710, 1989726178, 1381160022, 3642438773, - 3042165367, 3271478204, 3622052196, 1263328494, 1154237693, 1016814036, 3615401444, - 2786224913, 1251034970, 1874233020, 2956086971, 2572413057, 1529490307, 3146635362, - 1436483376, 1513970396, 3993894153, 979247444, 2142862852, 1368937545, 3275665128, - 1353904396, 294065371, 3543274652, 911792564, 2209636872, 3165709748, 103582737, - 3101998294, 730811902, 965151434, 893293786, 1885087777, 1390223786, 2657819643, - 3278634059, 3388458110, 3325206451, 1897013739, 3657020278, 4293473749, 3025318046, - 2117065739, 486446521, 3675999281, 620561205, 2085926890, 3946214694, 3609089773, - 2117061768, 163384588, 868009118, 2916136885, 1808988481, 83680601, 10243015, - 797410789, 3438751781, 1391154023, 234460119, 1278218413, 4068940987, 3948410079, - 2814385745, 3497626257, 1233272798, 3722456098, 1767303496, 3711787525, 2239286063, - 3968895688, 2030352819, 4140631909, 2230543409, 2173597546, 2866251044, 3211336683, - 1116033475, 2032556038, 3216540537, 4233849723, 2589589144, 667272562, 1863766055, - 3931110690, 2468422423, 1139480458, 272901326, 739418419, 1377144055, 4240143411, - 1954361769, 3839312742, 2814009721, 600813233, 3976840004, 3587775605, 4256065219, - 2324695947, 2293284974, 437604123, 1534849280, 1215859603, 327866771, 3356772175, - 2757237699, 720372669, 4240407984, 1421374867, 3228945021, 3284801305, 3409069246, - 4100357988, 643362354, 109899594, 1301585321, 2528806870, 1838904064, 448772403, - 1097849740, 1899994097, 618309123, 1911948510, 2309256224, 1861398151, 905306403, - 1067595802, 36868624, 3780886191, 835126206, 3190251977, 2672497726, 2085944002, - 2912993968, 2493776706, 667136329, 1474890786, 2383346554, 943528949, 3376706013, - 2495573574, 144956345, 793159960, 1591274917, 477107637, 1383815442, 67384899, - 2355242218, 1687409818, 3801093871, 2108217811, 3455908733, 4172160797, 3935534685, - 631067839, 1187677548, 2280856137, 3020767646, 2063176246, 3736904984, 2952933848, - 2975164686, 4144473303, 34670977, 1250976509, 3484166554, 1532744745, 225700994, - 1878713627, 2122358980, 1456610194, 2917522161, 2818947075, 102678939, 53743858, - 2095250656, 4023979225, 3094092874, 4128760696, 3411610028, 3020200609, 2225866341, - 586320946, 63813522, 1238216159, 2825692263, 2169937231, 3298517640, 1542128261, - 2205544184, 1258655704, 2629012083, 4113650203, 3198617867, 2742310794, 3372657381, - 3115904410, 1948638822, 1123521744, 1080429281, 4086706732, 4142693211, 817377147, - 2570194641, 26001503, 2861456160, 4185725555, 2573003804, 1618628779, 2588489212, - 3996192609, 1555844274, 1003123505, 1326350123, 1130583849, 3017128756, 74119042, - 4041266437, 1938014170, 3528465794, 4203969698, 1913054398, 3617979809, 2218810167, - 2453899816, 1997423206, 477446533, 303090065, 757937082, 1523238256, 3140505311, - 1422588701, 3642014639, 1740624195, 1276017154, 3072526193, 3675105122, 1335122682, - 4080595263, 2308519420, 3299182769, 1461978532, 3098694217, 2982399822, 3088698511, - 586759229, 3548750902, 1449857891, 2866451663, 2525162286, 57294602, 4107991297, - 1214672265, 2940391280, 4285346034, 3338216759, 737207923, 4264163846, 59219141, - 2300024654, 1876616814, 1976543605, 783571061, 1724699622, 1967524469, 1650309916, - 3322257631, 3975521122, 273342162, 1156754241, 185315896, 3368133921, 66314655, - 4153777915, 3519901897, - }, - { - 3672467167, 68684525, 1738833632, 3081329135, 2583806115, 2291130512, 503032614, - 3658059597, 571493931, 685537959, 3498787788, 422428426, 3879256913, 1173158320, - 4000800121, 298972869, 1718342816, 2541691685, 2490502642, 2321452806, 4223212804, - 1812334632, 3717655725, 4238191852, 3001307165, 2621896355, 2572404999, 3590094954, - 760765206, 2293618001, 1392353032, 1733137169, 2674005018, 4067961151, 1505710487, - 451078217, 2591688848, 12635611, 507045428, 694822241, 1789383090, 1140183890, - 1720695967, 1994318191, 3340349873, 2793804971, 1054433135, 2345087879, 3179939285, - 1651968615, 1793223686, 1055357758, 914271617, 483007580, 2127727816, 2754998083, - 3179053982, 598442002, 1950227301, 213053613, 3566888111, 2832258993, 4260365359, - 443662829, 1706542890, 3852730296, 3643260763, 2163607277, 1812905006, 171529637, - 215187467, 2369406909, 1929000706, 2572441025, 2133955541, 810692262, 1337974799, - 4030350704, 2159178715, 3769451556, 1026825278, 593628480, 1817383139, 878832429, - 2253876350, 203612980, 2102950440, 3407143936, 1912362251, 1595387637, 2827580539, - 305467658, 3292706746, 44135525, 4001933553, 3697343089, 760470915, 587414402, - 1419378814, 2852774010, 3891626781, 2757016765, 1090707384, 3997074427, 1047182100, - 2855539022, 36229159, 1591415533, 3471572739, 1237952140, 2614469314, 213338525, - 886212578, 2620301943, 713590207, 2430496777, 1198164420, 2644841698, 3654164701, - 36283572, 1461695896, 1770331341, 1641501876, 3470919184, 3181021559, 3053795110, - 3533531372, 3134337355, 668308383, 388340999, 3221275220, 1589659138, 294382235, - 1447443579, 690177534, 1799726917, 2838977761, 4172949119, 2360858031, 159385920, - 2248389027, 1790015671, 3925738275, 1049918544, 4107349511, 1619955951, 4188275966, - 1672572975, 2672697497, 1863413666, 747724021, 4037561738, 1605940213, 445253292, - 3362434828, 610898209, 1473244091, 735444769, 1540599852, 2449351720, 1032410949, - 3672467167, 68684525, 1738833632, 3081329135, 519684794, 2291130512, 503032614, - 3658059597, 571493931, 2400186105, 3498787788, 422428426, 3879256913, 1173158320, - 4120704752, 298972869, 1718342816, 2541691685, 2490502642, 1686027891, 4223212804, - 1812334632, 3717655725, 4238191852, 642431972, 2621896355, 2572404999, 3590094954, - 760765206, 2949609717, 1392353032, 1733137169, 2674005018, 4067961151, 1526077846, - 451078217, 2591688848, 12635611, 507045428, 2417951415, 1789383090, 1140183890, - 1720695967, 1994318191, 3465605863, 2793804971, 1054433135, 2345087879, 3179939285, - 3079297626, 1793223686, 1055357758, 914271617, 483007580, 306802527, 2754998083, - 3179053982, 598442002, 1950227301, 2473418737, 3566888111, 2832258993, 4260365359, - 443662829, 2097776414, 3852730296, 3643260763, 2163607277, 1812905006, 3957721904, - 215187467, 2369406909, 1929000706, 2572441025, 3779486126, 810692262, 1337974799, - 4030350704, 2159178715, 1127012865, 1026825278, 593628480, 1817383139, 878832429, - 361018423, 203612980, 2102950440, 3407143936, 1912362251, 1475218277, 2827580539, - 305467658, 3292706746, 44135525, 1900092336, 3697343089, 760470915, 587414402, - 1419378814, 343303227, 3891626781, 2757016765, 1090707384, 3997074427, 745490961, - 2855539022, 36229159, 1591415533, 3471572739, 3920625546, 2614469314, 213338525, - 886212578, 2620301943, 827771411, 2430496777, 1198164420, 2644841698, 3654164701, - 2747674190, 1461695896, 1770331341, 1641501876, 3470919184, 919857376, 3053795110, - 3533531372, 3134337355, 668308383, 201138876, 3221275220, 1589659138, 294382235, - 1447443579, 4211579707, 1799726917, 2838977761, 4172949119, 2360858031, 416103844, - 2248389027, 1790015671, 3925738275, 1049918544, 3481887924, 1619955951, 4188275966, - 1672572975, 2672697497, 564854400, 747724021, 4037561738, 1605940213, 445253292, - 604900912, 610898209, 1473244091, 735444769, 1540599852, 3036173307, 1032410949, - 3672467167, 68684525, 1738833632, 973022696, 519684794, 2291130512, 503032614, - 3658059597, 1500301452, 2400186105, 3498787788, 422428426, 3879256913, 3923611748, - 4120704752, 298972869, 1718342816, 2541691685, 2323881484, 1686027891, 4223212804, - 1812334632, 3717655725, 2109094458, 642431972, 2621896355, 2572404999, 3590094954, - 1837882537, 2949609717, 1392353032, 1733137169, 2674005018, 3252348987, 1526077846, - 451078217, 2591688848, 12635611, 3971261781, 2417951415, 1789383090, 1140183890, - 1720695967, 2906966040, 3465605863, 2793804971, 1054433135, 2345087879, 915518921, - 3079297626, 1793223686, 1055357758, 914271617, 791633499, 306802527, 2754998083, - 3179053982, 598442002, 324402573, 2473418737, 3566888111, 2832258993, 4260365359, - 2168046398, 2097776414, 3852730296, 3643260763, 2163607277, 2595175979, 3957721904, - 215187467, 2369406909, 1929000706, 657446369, 3779486126, 810692262, 1337974799, - 4030350704, 1865557469, 1127012865, 1026825278, 593628480, 1817383139, 3414354529, - 361018423, 203612980, 2102950440, 3407143936, 1739372987, 1475218277, 2827580539, - 305467658, 3292706746, 825045562, 1900092336, 3697343089, 760470915, 587414402, - 2000637694, 343303227, 3891626781, 2757016765, 1090707384, 4015377800, 745490961, - 2855539022, 36229159, 1591415533, 2208656873, 3920625546, 2614469314, 213338525, - 886212578, 2729976209, 827771411, 2430496777, 1198164420, 2644841698, 1922667440, - 2747674190, 1461695896, 1770331341, 1641501876, 357535311, 919857376, 3053795110, - 3533531372, 3134337355, 1004072597, 201138876, 3221275220, 1589659138, 294382235, - 1148950143, 4211579707, 1799726917, 2838977761, 4172949119, 892664404, 416103844, - 2248389027, 1790015671, 3925738275, 2612357890, 3481887924, 1619955951, 4188275966, - 1672572975, 2005534713, 564854400, 747724021, 4037561738, 1605940213, 2620990454, - 604900912, 610898209, 1473244091, 735444769, 3571225334, 3036173307, 1032410949, - 3672467167, 68684525, 3327351604, 973022696, 519684794, 2291130512, 503032614, - 3814902238, 1500301452, 2400186105, 3498787788, 422428426, 1756753750, 3923611748, - 4120704752, 298972869, 1718342816, 652903081, 2323881484, 1686027891, 4223212804, - 1812334632, 1599640566, 2109094458, 642431972, 2621896355, 2572404999, 1668409355, - 1837882537, 2949609717, 1392353032, 1733137169, 3691709793, 3252348987, 1526077846, - 451078217, 2591688848, 3353622601, 3971261781, 2417951415, 1789383090, 1140183890, - 4113853791, 2906966040, 3465605863, 2793804971, 1054433135, 2195882948, 915518921, - 3079297626, 1793223686, 1055357758, 898713552, 791633499, 306802527, 2754998083, - 3179053982, 2469350088, 324402573, 2473418737, 3566888111, 2832258993, 1377718274, - 2168046398, 2097776414, 3852730296, 3643260763, 3492388484, 2595175979, 3957721904, - 215187467, 2369406909, 4243449339, 657446369, 3779486126, 810692262, 1337974799, - 3960230785, 1865557469, 1127012865, 1026825278, 593628480, 732793312, 3414354529, - 361018423, 203612980, 2102950440, 2401792405, 1739372987, 1475218277, 2827580539, - 305467658, 2454275289, 825045562, 1900092336, 3697343089, 760470915, 2146882409, - 2000637694, 343303227, 3891626781, 2757016765, 3997473261, 4015377800, 745490961, - 2855539022, 36229159, 2375394427, 2208656873, 3920625546, 2614469314, 213338525, - 2055366274, 2729976209, 827771411, 2430496777, 1198164420, 1789631187, 1922667440, - 2747674190, 1461695896, 1770331341, 4284442852, 357535311, 919857376, 3053795110, - 3533531372, 2124270060, 1004072597, 201138876, 3221275220, 1589659138, 1418386120, - 1148950143, 4211579707, 1799726917, 2838977761, 3540708069, 892664404, 416103844, - 2248389027, 1790015671, 3936883, 2612357890, 3481887924, 1619955951, 4188275966, - 2963623483, 2005534713, 564854400, 747724021, 4037561738, 3431155922, 2620990454, - 604900912, 610898209, 1473244091, 3880001339, 2879060316, 3300897679, 3960972039, - 3201086624, 3814462934, 3426650044, 1930881632, 1981178788, 2956279691, 4272406256, - 372705521, 1359389771, 1590302979, 3940206208, 3817999127, 2527835456, 2739078164, - 716997849, 3235607043, 2550297745, 3688700200, 354502605, 2285793656, 2339138034, - 3912354142, 2262255668, 469322622, 1319943359, 1916101235, 200441823, 509436982, - 2160284593, 1687919695, 4153615582, 495735041, 3694469424, 2086893117, 4223008799, - 105344742, 1698033424, 1149223145, 4183918790, 4176151950, 415739351, 817762972, - 3768072560, 1931430949, 2698979439, 3481477932, 1994322914, 4078299950, 1268233995, - 3254069145, 91029129, 498234704, 1636613942, 3710087092, 3876816560, 3510446387, - 3870169008, 1370156410, 2442498047, 2324396523, 1258730334, 621954739, 1053015373, - 491820717, 3386515432, 2203703266, 120167176, 2383669740, 1038666440, 2927342870, - 3583197824, 1236241846, 2474675929, 679052891, 2451259584, 2177706146, 606842882, - 3546980104, 2289281509, 353873434, 2041926837, 1238346748, 2729109726, 2843938395, - 2938124210, 2554443866, 1494477920, 693378319, 2020963566, 2000385949, 3744098787, - 650307220, 2631327075, 1529128757, 595871428, 3206666562, 458062987, 875238192, - 3729317374, 1368843921, 3478430230, 3234384578, 3232435428, 321359326, 994274524, - 361184397, 4285497594, 915263578, 1486882838, 9988613, 829077170, 677216046, - 4141828204, 165804609, 1086678519, 2933434608, 1351662802, 2640085040, 2611502932, - 2033698714, 2008873254, 3995557835, 1020873906, 67873555, 2230337823, 1263800417, - 1148712155, 3985159589, 2979503513, 2854714997, 1539343345, 2751484352, 1569100732, - 2020758949, 2126757134, 3426641899, 2808587825, 1953320148, 1096398464, 1502907172, - 3751230087, 765557661, 765290990, 3056075500, 2040620632, 422573751, 3613558930, - 1741145769, 273531216, 837238736, 494297893, 2903251124, 1636782182, 4256592784, - 3652746656, 4258393217, - }, - { - 2657510202, 270297201, 2970166904, 3151626326, 973127447, 1523852613, 598650578, - 10289043, 1138773500, 1379558769, 2202575480, 1622690708, 181345079, 228706650, - 2807760507, 3061024281, 2310359315, 3094465578, 4062753882, 2744510393, 3844622451, - 1759718963, 2393602744, 977540509, 870449791, 1484134272, 2838962253, 3079492430, - 2617141201, 3744868057, 994295425, 1302594555, 277777192, 1793039043, 1620482692, - 2518563014, 1163760339, 2709515777, 4220588138, 531143270, 2528377633, 931694828, - 1472659070, 900489303, 3538137811, 3849822545, 1304182427, 2423451948, 587259647, - 296795227, 3843393378, 100570026, 1824916038, 3155192628, 1205830295, 2205840913, - 2598785234, 2138099222, 1585588098, 1304106911, 2443465671, 3007665864, 3350433156, - 3623458138, 629407548, 3209244941, 2102270358, 952701496, 2715374730, 2142960491, - 2566649458, 2386659994, 4201648072, 367516884, 211986877, 3970312395, 4153651951, - 3794120671, 614826776, 769672874, 2218713182, 236114529, 1614697510, 2420862368, - 3471485219, 3080341429, 2394724619, 3585194114, 1394678495, 2137969611, 3955498999, - 2765569351, 3084915757, 765232390, 1406483345, 2796499268, 2491128017, 1052428931, - 1713430644, 3921576513, 3753414774, 973530327, 2545412294, 1841110931, 1174406073, - 1104865218, 1586606252, 2612244473, 1407875673, 1823397519, 2613642581, 3163449384, - 3129975397, 2059184961, 818092118, 3182607992, 1658516909, 2467681581, 1065789733, - 799857247, 2492902195, 168866110, 2251316716, 1607684829, 2347941418, 2382781983, - 3298500129, 3609200925, 3060374324, 2602420483, 2357812057, 3739699403, 3260652552, - 205015857, 1936033273, 3955997259, 821264237, 1882720491, 159294165, 3197657094, - 528058988, 2768830342, 805087358, 896645931, 1360375456, 3417488932, 3863200799, - 4033907887, 983658874, 1828706965, 875027318, 1310362653, 3711487613, 4148261033, - 3145162047, 485182003, 2633647498, 1369395018, 4163384029, 1827719274, 270658892, - 2657510202, 270297201, 2970166904, 3151626326, 499420828, 1523852613, 598650578, - 10289043, 1138773500, 640170086, 2202575480, 1622690708, 181345079, 228706650, - 3957853780, 3061024281, 2310359315, 3094465578, 4062753882, 2049506087, 3844622451, - 1759718963, 2393602744, 977540509, 2346891936, 1484134272, 2838962253, 3079492430, - 2617141201, 2112540708, 994295425, 1302594555, 277777192, 1793039043, 981072592, - 2518563014, 1163760339, 2709515777, 4220588138, 1992965594, 2528377633, 931694828, - 1472659070, 900489303, 32461040, 3849822545, 1304182427, 2423451948, 587259647, - 3728056788, 3843393378, 100570026, 1824916038, 3155192628, 1194916233, 2205840913, - 2598785234, 2138099222, 1585588098, 2944318376, 2443465671, 3007665864, 3350433156, - 3623458138, 1413669939, 3209244941, 2102270358, 952701496, 2715374730, 826676012, - 2566649458, 2386659994, 4201648072, 367516884, 4272143576, 3970312395, 4153651951, - 3794120671, 614826776, 4106382849, 2218713182, 236114529, 1614697510, 2420862368, - 138091237, 3080341429, 2394724619, 3585194114, 1394678495, 2113895281, 3955498999, - 2765569351, 3084915757, 765232390, 2247301699, 2796499268, 2491128017, 1052428931, - 1713430644, 1076867271, 3753414774, 973530327, 2545412294, 1841110931, 3427639042, - 1104865218, 1586606252, 2612244473, 1407875673, 2159805028, 2613642581, 3163449384, - 3129975397, 2059184961, 1251595655, 3182607992, 1658516909, 2467681581, 1065789733, - 524065102, 2492902195, 168866110, 2251316716, 1607684829, 877205873, 2382781983, - 3298500129, 3609200925, 3060374324, 1983477493, 2357812057, 3739699403, 3260652552, - 205015857, 3578808491, 3955997259, 821264237, 1882720491, 159294165, 3639531297, - 528058988, 2768830342, 805087358, 896645931, 2309781073, 3417488932, 3863200799, - 4033907887, 983658874, 3756437847, 875027318, 1310362653, 3711487613, 4148261033, - 3264363953, 485182003, 2633647498, 1369395018, 4163384029, 184614728, 270658892, - 2657510202, 270297201, 2970166904, 884907665, 499420828, 1523852613, 598650578, - 10289043, 2023902217, 640170086, 2202575480, 1622690708, 181345079, 1358722197, - 3957853780, 3061024281, 2310359315, 3094465578, 4156960892, 2049506087, 3844622451, - 1759718963, 2393602744, 1018272187, 2346891936, 1484134272, 2838962253, 3079492430, - 663361761, 2112540708, 994295425, 1302594555, 277777192, 4201292427, 981072592, - 2518563014, 1163760339, 2709515777, 3301905324, 1992965594, 2528377633, 931694828, - 1472659070, 3170286187, 32461040, 3849822545, 1304182427, 2423451948, 166213287, - 3728056788, 3843393378, 100570026, 1824916038, 1534589402, 1194916233, 2205840913, - 2598785234, 2138099222, 767439709, 2944318376, 2443465671, 3007665864, 3350433156, - 257274072, 1413669939, 3209244941, 2102270358, 952701496, 893224047, 826676012, - 2566649458, 2386659994, 4201648072, 1336000731, 4272143576, 3970312395, 4153651951, - 3794120671, 2381517352, 4106382849, 2218713182, 236114529, 1614697510, 2427291612, - 138091237, 3080341429, 2394724619, 3585194114, 1339840651, 2113895281, 3955498999, - 2765569351, 3084915757, 1920073265, 2247301699, 2796499268, 2491128017, 1052428931, - 1720704700, 1076867271, 3753414774, 973530327, 2545412294, 655938239, 3427639042, - 1104865218, 1586606252, 2612244473, 748629647, 2159805028, 2613642581, 3163449384, - 3129975397, 1868740512, 1251595655, 3182607992, 1658516909, 2467681581, 3092135795, - 524065102, 2492902195, 168866110, 2251316716, 229376275, 877205873, 2382781983, - 3298500129, 3609200925, 1270454086, 1983477493, 2357812057, 3739699403, 3260652552, - 3976376418, 3578808491, 3955997259, 821264237, 1882720491, 2211365699, 3639531297, - 528058988, 2768830342, 805087358, 1351870678, 2309781073, 3417488932, 3863200799, - 4033907887, 2317721807, 3756437847, 875027318, 1310362653, 3711487613, 1929459086, - 3264363953, 485182003, 2633647498, 1369395018, 2141675718, 184614728, 270658892, - 2657510202, 270297201, 3337954073, 884907665, 499420828, 1523852613, 598650578, - 3874207188, 2023902217, 640170086, 2202575480, 1622690708, 2020255059, 1358722197, - 3957853780, 3061024281, 2310359315, 753738868, 4156960892, 2049506087, 3844622451, - 1759718963, 1672276116, 1018272187, 2346891936, 1484134272, 2838962253, 1680679979, - 663361761, 2112540708, 994295425, 1302594555, 1941500850, 4201292427, 981072592, - 2518563014, 1163760339, 184357645, 3301905324, 1992965594, 2528377633, 931694828, - 3462653134, 3170286187, 32461040, 3849822545, 1304182427, 396808784, 166213287, - 3728056788, 3843393378, 100570026, 876691173, 1534589402, 1194916233, 2205840913, - 2598785234, 4286653520, 767439709, 2944318376, 2443465671, 3007665864, 2793587144, - 257274072, 1413669939, 3209244941, 2102270358, 2792966616, 893224047, 826676012, - 2566649458, 2386659994, 798757973, 1336000731, 4272143576, 3970312395, 4153651951, - 2930383268, 2381517352, 4106382849, 2218713182, 236114529, 1936008889, 2427291612, - 138091237, 3080341429, 2394724619, 4157586029, 1339840651, 2113895281, 3955498999, - 2765569351, 2243544114, 1920073265, 2247301699, 2796499268, 2491128017, 3372810009, - 1720704700, 1076867271, 3753414774, 973530327, 484392041, 655938239, 3427639042, - 1104865218, 1586606252, 1373046326, 748629647, 2159805028, 2613642581, 3163449384, - 1558595520, 1868740512, 1251595655, 3182607992, 1658516909, 3503432306, 3092135795, - 524065102, 2492902195, 168866110, 4106973392, 229376275, 877205873, 2382781983, - 3298500129, 2366096961, 1270454086, 1983477493, 2357812057, 3739699403, 4223323197, - 3976376418, 3578808491, 3955997259, 821264237, 1581729952, 2211365699, 3639531297, - 528058988, 2768830342, 3946263978, 1351870678, 2309781073, 3417488932, 3863200799, - 3948072426, 2317721807, 3756437847, 875027318, 1310362653, 3439391360, 1929459086, - 3264363953, 485182003, 2633647498, 3576868480, 2527748673, 3116247125, 4020801612, - 2594734840, 3308177137, 665011257, 40118275, 3584569179, 3399729283, 3867174947, - 658488234, 1099195903, 2274511402, 1872529118, 2518961094, 2633598693, 4160728307, - 449442630, 164837956, 1010805767, 605336924, 1178031445, 3949359502, 2585151633, - 611885521, 293204651, 3389557188, 1172294301, 2503819061, 659842653, 504992348, - 3762165683, 1799777932, 4161843209, 1924622448, 1006263939, 115233249, 2775142171, - 3228632586, 885407023, 2514866293, 3615088636, 2488824172, 2631364137, 1454226414, - 3888177876, 70646265, 2291458600, 2370783730, 1566625834, 3652033806, 4136806683, - 2819973124, 3207365429, 989185345, 3343822313, 2580472874, 4077285847, 4032963783, - 2883518039, 2253593637, 904631114, 2654790756, 2967911632, 2131672564, 1594073414, - 2370718497, 3769371275, 1547951748, 2473303924, 651625138, 2159175883, 4062995539, - 696224922, 3388626509, 100118553, 770731124, 2149458689, 3223175313, 3524052514, - 2651241522, 78236806, 3212708723, 1045780878, 2257575290, 3709360831, 966829465, - 61269250, 405063245, 331731998, 2472078870, 1138237364, 1135091387, 3245001409, - 3817992705, 1738939574, 1397617581, 2896546651, 4207083421, 3802162100, 391930524, - 1326819828, 85308067, 3235336831, 686989692, 1947564282, 842881662, 2887279866, - 3850666935, 2001895525, 2673649961, 2106555006, 1762053005, 2334552700, 26094213, - 1184502058, 2048598709, 4039640450, 1439363714, 1022688817, 1053169108, 170896272, - 444231850, 1500204748, 1077470703, 1630597179, 1382588806, 138805391, 1636536505, - 3118018426, 3461152216, 2486547351, 2045361316, 2976067436, 468876399, 1407419455, - 3226137264, 414206328, 1011039713, 3537947031, 2359787831, 258556532, 3615987029, - 3372097337, 3586352388, 1056198830, 1852291192, 3888893481, 746156045, 4203877603, - 297851145, 2615507398, 1141098641, 1881412583, 3014341741, 2125186797, 229307235, - 3476606674, 3553854689, - }, - { - 3768542219, 2777948797, 3328832678, 3488502819, 2708053041, 2217907094, 2133505056, - 2218961277, 2148551748, 1420045625, 1709182366, 1816409641, 3791695288, 4207813971, - 22588497, 2211317602, 616238454, 2394270012, 3212896041, 213408768, 2199328374, - 3188624050, 811443809, 2818548979, 3150758902, 2022548260, 2462701924, 3793704672, - 2358080321, 483288372, 450033142, 772942770, 2224873625, 241543410, 312552314, - 1268067149, 915918620, 3906238422, 132545832, 3486041298, 2414090506, 3798383292, - 2257004699, 130309284, 1158673651, 152325583, 3499865580, 4094273597, 1029041593, - 93538481, 3963199522, 4215066819, 2851084137, 950351173, 2758084052, 3408506640, - 2468905351, 3982226741, 3591899344, 2972879639, 3321078070, 252381865, 409397320, - 741653003, 1936712854, 1198684021, 922916691, 10413506, 3546896248, 1704703870, - 1479762464, 104399432, 4144557684, 68239720, 2666028745, 362625839, 2591539911, - 2837165752, 2180226515, 4076543943, 2956460273, 312410753, 2566731139, 2532653524, - 2399030172, 207904356, 354574195, 485696336, 3816686234, 3016971115, 4272692603, - 2352732136, 33493163, 780255811, 4092242980, 4121521600, 2119254314, 42767673, - 1081488778, 2757446871, 2267513620, 3472164720, 2750308207, 1707164045, 3125591821, - 3236687597, 299194858, 537384087, 1695155491, 2078250102, 1705861659, 2416322096, - 1692335914, 1178915980, 3405431297, 4059323309, 2014660182, 3847682866, 4037583683, - 2629253995, 867809161, 2167953720, 2290558548, 417635396, 53496289, 1890906570, - 2842247580, 807266805, 1226139132, 2067929784, 1697038549, 3312131466, 1234311530, - 3199840935, 4185078776, 1807030355, 215385887, 845421530, 1350380353, 4209181096, - 2576197887, 1275262872, 2806513944, 2718623701, 2779287384, 71403197, 219220133, - 2181111477, 2000396844, 3595837555, 1232425455, 2630647391, 3280867676, 2622740782, - 1578938469, 3624564545, 992324522, 3056113148, 3473635768, 3664935418, 1786902552, - 3768542219, 2777948797, 3328832678, 3488502819, 2530862473, 2217907094, 2133505056, - 2218961277, 2148551748, 4050672856, 1709182366, 1816409641, 3791695288, 4207813971, - 4175126713, 2211317602, 616238454, 2394270012, 3212896041, 732700649, 2199328374, - 3188624050, 811443809, 2818548979, 972036137, 2022548260, 2462701924, 3793704672, - 2358080321, 1200725173, 450033142, 772942770, 2224873625, 241543410, 1907109304, - 1268067149, 915918620, 3906238422, 132545832, 301668366, 2414090506, 3798383292, - 2257004699, 130309284, 1228520287, 152325583, 3499865580, 4094273597, 1029041593, - 3267460249, 3963199522, 4215066819, 2851084137, 950351173, 47361585, 3408506640, - 2468905351, 3982226741, 3591899344, 1878226915, 3321078070, 252381865, 409397320, - 741653003, 1716437506, 1198684021, 922916691, 10413506, 3546896248, 1591998796, - 1479762464, 104399432, 4144557684, 68239720, 3810955599, 362625839, 2591539911, - 2837165752, 2180226515, 3908378015, 2956460273, 312410753, 2566731139, 2532653524, - 687490649, 207904356, 354574195, 485696336, 3816686234, 378445403, 4272692603, - 2352732136, 33493163, 780255811, 1303281526, 4121521600, 2119254314, 42767673, - 1081488778, 1734311274, 2267513620, 3472164720, 2750308207, 1707164045, 4212588163, - 3236687597, 299194858, 537384087, 1695155491, 2250704950, 1705861659, 2416322096, - 1692335914, 1178915980, 677982197, 4059323309, 2014660182, 3847682866, 4037583683, - 1765435945, 867809161, 2167953720, 2290558548, 417635396, 2125103002, 1890906570, - 2842247580, 807266805, 1226139132, 2056644398, 1697038549, 3312131466, 1234311530, - 3199840935, 3063718636, 1807030355, 215385887, 845421530, 1350380353, 3610667273, - 2576197887, 1275262872, 2806513944, 2718623701, 2492912955, 71403197, 219220133, - 2181111477, 2000396844, 3465351710, 1232425455, 2630647391, 3280867676, 2622740782, - 1331873639, 3624564545, 992324522, 3056113148, 3473635768, 782257020, 1786902552, - 3768542219, 2777948797, 3328832678, 856888454, 2530862473, 2217907094, 2133505056, - 2218961277, 3752437534, 4050672856, 1709182366, 1816409641, 3791695288, 1581813910, - 4175126713, 2211317602, 616238454, 2394270012, 1796414157, 732700649, 2199328374, - 3188624050, 811443809, 4225173324, 972036137, 2022548260, 2462701924, 3793704672, - 1410793611, 1200725173, 450033142, 772942770, 2224873625, 3889840648, 1907109304, - 1268067149, 915918620, 3906238422, 1249098244, 301668366, 2414090506, 3798383292, - 2257004699, 1620796656, 1228520287, 152325583, 3499865580, 4094273597, 82853050, - 3267460249, 3963199522, 4215066819, 2851084137, 1212493334, 47361585, 3408506640, - 2468905351, 3982226741, 3195419905, 1878226915, 3321078070, 252381865, 409397320, - 1584154733, 1716437506, 1198684021, 922916691, 10413506, 1734068880, 1591998796, - 1479762464, 104399432, 4144557684, 1973878859, 3810955599, 362625839, 2591539911, - 2837165752, 1727282404, 3908378015, 2956460273, 312410753, 2566731139, 3656295687, - 687490649, 207904356, 354574195, 485696336, 355953909, 378445403, 4272692603, - 2352732136, 33493163, 3784169684, 1303281526, 4121521600, 2119254314, 42767673, - 2331527847, 1734311274, 2267513620, 3472164720, 2750308207, 820692528, 4212588163, - 3236687597, 299194858, 537384087, 781151234, 2250704950, 1705861659, 2416322096, - 1692335914, 4288008793, 677982197, 4059323309, 2014660182, 3847682866, 3328850880, - 1765435945, 867809161, 2167953720, 2290558548, 542850707, 2125103002, 1890906570, - 2842247580, 807266805, 3803006390, 2056644398, 1697038549, 3312131466, 1234311530, - 809106036, 3063718636, 1807030355, 215385887, 845421530, 654189622, 3610667273, - 2576197887, 1275262872, 2806513944, 1517875462, 2492912955, 71403197, 219220133, - 2181111477, 3826277490, 3465351710, 1232425455, 2630647391, 3280867676, 3343597872, - 1331873639, 3624564545, 992324522, 3056113148, 3725661598, 782257020, 1786902552, - 3768542219, 2777948797, 3392298403, 856888454, 2530862473, 2217907094, 2133505056, - 4160889036, 3752437534, 4050672856, 1709182366, 1816409641, 1282922706, 1581813910, - 4175126713, 2211317602, 616238454, 3806252779, 1796414157, 732700649, 2199328374, - 3188624050, 983474330, 4225173324, 972036137, 2022548260, 2462701924, 880446667, - 1410793611, 1200725173, 450033142, 772942770, 3179870546, 3889840648, 1907109304, - 1268067149, 915918620, 4261932110, 1249098244, 301668366, 2414090506, 3798383292, - 471794009, 1620796656, 1228520287, 152325583, 3499865580, 1275109063, 82853050, - 3267460249, 3963199522, 4215066819, 4209882674, 1212493334, 47361585, 3408506640, - 2468905351, 1324785625, 3195419905, 1878226915, 3321078070, 252381865, 4259927884, - 1584154733, 1716437506, 1198684021, 922916691, 1800164165, 1734068880, 1591998796, - 1479762464, 104399432, 2774114308, 1973878859, 3810955599, 362625839, 2591539911, - 2126614872, 1727282404, 3908378015, 2956460273, 312410753, 4098052715, 3656295687, - 687490649, 207904356, 354574195, 937379582, 355953909, 378445403, 4272692603, - 2352732136, 2694800574, 3784169684, 1303281526, 4121521600, 2119254314, 1741415022, - 2331527847, 1734311274, 2267513620, 3472164720, 480821513, 820692528, 4212588163, - 3236687597, 299194858, 1128762168, 781151234, 2250704950, 1705861659, 2416322096, - 160918735, 4288008793, 677982197, 4059323309, 2014660182, 3354205317, 3328850880, - 1765435945, 867809161, 2167953720, 3363861382, 542850707, 2125103002, 1890906570, - 2842247580, 2459935488, 3803006390, 2056644398, 1697038549, 3312131466, 2378675900, - 809106036, 3063718636, 1807030355, 215385887, 3528413525, 654189622, 3610667273, - 2576197887, 1275262872, 993221887, 1517875462, 2492912955, 71403197, 219220133, - 1805256638, 3826277490, 3465351710, 1232425455, 2630647391, 3718538519, 3343597872, - 1331873639, 3624564545, 992324522, 3490576382, 2532191937, 1108692984, 802110050, - 3984561242, 1973015939, 1351080551, 2382044123, 2393286227, 860228704, 179528099, - 3569709850, 233527199, 3657599850, 3269634908, 3278075383, 4037814788, 952837871, - 2050210570, 2376157484, 2566048929, 4200278597, 123440514, 573557299, 1585379806, - 4012659271, 4000306490, 2508478465, 970078629, 4064973573, 645149301, 109544347, - 647594029, 2097163688, 1515080116, 2142799649, 2519702653, 3122920796, 1952249156, - 3932382760, 2155292687, 2517875978, 249059416, 4282787227, 2595461065, 1004349415, - 2151451255, 2510715277, 3004500356, 3410567758, 344538405, 1946747709, 470298928, - 1033671146, 4207801290, 1411375630, 3419808553, 3218285984, 3584735265, 811222695, - 3898833227, 3535298390, 3764741581, 3927026520, 2850086968, 2818485449, 1963038474, - 1871366998, 1900570117, 997663534, 746627295, 1827737271, 3814054979, 728285698, - 1696496343, 1696888597, 1010837663, 1756050352, 785994134, 1436861536, 1949153732, - 2360018842, 1703393654, 2248338006, 3884572674, 789998735, 1155994673, 2022469457, - 223162974, 309571006, 725482797, 3909032036, 2531190541, 373676789, 1061107200, - 4231921550, 558635876, 2773807977, 1860218585, 1150041015, 2252812038, 2413330952, - 191909567, 3518171813, 3513416318, 2679253717, 3850755687, 1564154710, 324714884, - 1600953447, 4095583159, 1796641692, 2518000547, 3621187982, 501166402, 2112782420, - 1704276185, 2249859782, 3754293422, 1942321901, 1851019104, 240158224, 3181132144, - 2281632719, 808029657, 1721710011, 2287207169, 3044484177, 2363339534, 805273402, - 3696016147, 3549191229, 3353631259, 2946802391, 383414270, 300735554, 471515206, - 1907815837, 1576327662, 3825043525, 2817119733, 1973847200, 1398317206, 2221853087, - 501440864, 642467132, 494410179, 1191241925, 3549838846, 3621239619, 2640266286, - 4140123024, 315957218, 3696758268, 2502777875, 2150738616, 1570099119, 2598276767, - 3585886712, 230047417, - }, - { - 220882755, 630187688, 2600079656, 3103815531, 4259457395, 306940008, 760977254, - 558299017, 73879495, 2342545344, 572800511, 3922797738, 3754011306, 698257357, - 1274843132, 1455757442, 1014649591, 3205662508, 2997738251, 613949432, 2267018388, - 2925762681, 3702061213, 299380602, 1711070497, 4140032336, 4134705925, 2836703879, - 3776863395, 507121465, 3480792188, 1862887216, 247780795, 2528677869, 2881446422, - 271754977, 833498724, 1489102731, 3636156177, 1839744487, 2011839858, 2353400914, - 510437606, 561141583, 2979592314, 3844268262, 3011027242, 3113817193, 3491178377, - 1448376742, 2478683391, 2597550150, 699310968, 1979488062, 277591964, 1312002175, - 168047351, 1826859926, 2030631355, 3097860388, 1950614326, 4070838751, 4454933, - 1890661188, 3929835227, 1008498572, 3301557438, 3906313590, 1240635175, 280935563, - 113509402, 226900299, 1246395851, 1220916742, 2651515540, 2058590162, 1983114332, - 2040467861, 780818345, 544262576, 2826997265, 349354812, 2360120613, 1181324247, - 2380347783, 3938729706, 1610628643, 2008635822, 2937909233, 1583978206, 3589167073, - 1942470196, 402177406, 2636510744, 3709747478, 2428569572, 4071828137, 2880315633, - 1433558231, 1137076031, 3833202201, 2378168250, 1412413704, 3349323744, 1740721660, - 3155643175, 2580327273, 3020661883, 1658910832, 2065649368, 3277572880, 3795585437, - 1266185861, 2925935368, 4147230645, 203577834, 2230529041, 2864778434, 270386174, - 2867122465, 2676624544, 2035972330, 500973884, 2983028740, 117131866, 1456450936, - 429171245, 3921563262, 342800398, 255116920, 1219580025, 1549741331, 3832317567, - 3750096895, 4036554472, 4099775516, 1451717480, 149159438, 3593827664, 1406572509, - 27774796, 1138983585, 1577536190, 978350835, 2704344602, 95204061, 1507155668, - 304760810, 1981315657, 3139306913, 3908131532, 3767856445, 3851422551, 2018732047, - 2474676116, 2745551516, 1585868430, 1125303733, 3147584753, 2368921260, 1524991519, - 220882755, 630187688, 2600079656, 3103815531, 2671841243, 306940008, 760977254, - 558299017, 73879495, 1196617651, 572800511, 3922797738, 3754011306, 698257357, - 1982654891, 1455757442, 1014649591, 3205662508, 2997738251, 3769735713, 2267018388, - 2925762681, 3702061213, 299380602, 2224634157, 4140032336, 4134705925, 2836703879, - 3776863395, 1027030708, 3480792188, 1862887216, 247780795, 2528677869, 300214141, - 271754977, 833498724, 1489102731, 3636156177, 1683033001, 2011839858, 2353400914, - 510437606, 561141583, 2832813585, 3844268262, 3011027242, 3113817193, 3491178377, - 316500941, 2478683391, 2597550150, 699310968, 1979488062, 4092049617, 1312002175, - 168047351, 1826859926, 2030631355, 2797906491, 1950614326, 4070838751, 4454933, - 1890661188, 2602196847, 1008498572, 3301557438, 3906313590, 1240635175, 946440664, - 113509402, 226900299, 1246395851, 1220916742, 1314772486, 2058590162, 1983114332, - 2040467861, 780818345, 3064382079, 2826997265, 349354812, 2360120613, 1181324247, - 3434653713, 3938729706, 1610628643, 2008635822, 2937909233, 2815835447, 3589167073, - 1942470196, 402177406, 2636510744, 865459039, 2428569572, 4071828137, 2880315633, - 1433558231, 1582478959, 3833202201, 2378168250, 1412413704, 3349323744, 3686787615, - 3155643175, 2580327273, 3020661883, 1658910832, 3152644489, 3277572880, 3795585437, - 1266185861, 2925935368, 3101079227, 203577834, 2230529041, 2864778434, 270386174, - 3024925346, 2676624544, 2035972330, 500973884, 2983028740, 974511421, 1456450936, - 429171245, 3921563262, 342800398, 1540218139, 1219580025, 1549741331, 3832317567, - 3750096895, 2195381148, 4099775516, 1451717480, 149159438, 3593827664, 3715984838, - 27774796, 1138983585, 1577536190, 978350835, 2060213898, 95204061, 1507155668, - 304760810, 1981315657, 774471092, 3908131532, 3767856445, 3851422551, 2018732047, - 1649125731, 2745551516, 1585868430, 1125303733, 3147584753, 1661721342, 1524991519, - 220882755, 630187688, 2600079656, 3647143842, 2671841243, 306940008, 760977254, - 558299017, 3406011854, 1196617651, 572800511, 3922797738, 3754011306, 2314291278, - 1982654891, 1455757442, 1014649591, 3205662508, 3471741326, 3769735713, 2267018388, - 2925762681, 3702061213, 1593850639, 2224634157, 4140032336, 4134705925, 2836703879, - 3918266498, 1027030708, 3480792188, 1862887216, 247780795, 3383776045, 300214141, - 271754977, 833498724, 1489102731, 2477093804, 1683033001, 2011839858, 2353400914, - 510437606, 2361664959, 2832813585, 3844268262, 3011027242, 3113817193, 1468705704, - 316500941, 2478683391, 2597550150, 699310968, 1593029686, 4092049617, 1312002175, - 168047351, 1826859926, 3922295193, 2797906491, 1950614326, 4070838751, 4454933, - 485374579, 2602196847, 1008498572, 3301557438, 3906313590, 2102043683, 946440664, - 113509402, 226900299, 1246395851, 2635764090, 1314772486, 2058590162, 1983114332, - 2040467861, 354791, 3064382079, 2826997265, 349354812, 2360120613, 2126504772, - 3434653713, 3938729706, 1610628643, 2008635822, 2400061949, 2815835447, 3589167073, - 1942470196, 402177406, 806469309, 865459039, 2428569572, 4071828137, 2880315633, - 2512200928, 1582478959, 3833202201, 2378168250, 1412413704, 2767451252, 3686787615, - 3155643175, 2580327273, 3020661883, 1040874588, 3152644489, 3277572880, 3795585437, - 1266185861, 238446394, 3101079227, 203577834, 2230529041, 2864778434, 653319712, - 3024925346, 2676624544, 2035972330, 500973884, 3839534784, 974511421, 1456450936, - 429171245, 3921563262, 602320448, 1540218139, 1219580025, 1549741331, 3832317567, - 1068872823, 2195381148, 4099775516, 1451717480, 149159438, 1699607068, 3715984838, - 27774796, 1138983585, 1577536190, 837921790, 2060213898, 95204061, 1507155668, - 304760810, 1446133066, 774471092, 3908131532, 3767856445, 3851422551, 2672625648, - 1649125731, 2745551516, 1585868430, 1125303733, 2181520384, 1661721342, 1524991519, - 220882755, 630187688, 1599074811, 3647143842, 2671841243, 306940008, 760977254, - 3020017536, 3406011854, 1196617651, 572800511, 3922797738, 810584907, 2314291278, - 1982654891, 1455757442, 1014649591, 1775783567, 3471741326, 3769735713, 2267018388, - 2925762681, 319055602, 1593850639, 2224634157, 4140032336, 4134705925, 2794599326, - 3918266498, 1027030708, 3480792188, 1862887216, 659607854, 3383776045, 300214141, - 271754977, 833498724, 4054679386, 2477093804, 1683033001, 2011839858, 2353400914, - 283191425, 2361664959, 2832813585, 3844268262, 3011027242, 137688840, 1468705704, - 316500941, 2478683391, 2597550150, 1468220070, 1593029686, 4092049617, 1312002175, - 168047351, 1602414610, 3922295193, 2797906491, 1950614326, 4070838751, 3858763082, - 485374579, 2602196847, 1008498572, 3301557438, 2719858709, 2102043683, 946440664, - 113509402, 226900299, 798285817, 2635764090, 1314772486, 2058590162, 1983114332, - 2526463430, 354791, 3064382079, 2826997265, 349354812, 249430921, 2126504772, - 3434653713, 3938729706, 1610628643, 967431506, 2400061949, 2815835447, 3589167073, - 1942470196, 669129162, 806469309, 865459039, 2428569572, 4071828137, 3447449944, - 2512200928, 1582478959, 3833202201, 2378168250, 1945768856, 2767451252, 3686787615, - 3155643175, 2580327273, 2905624117, 1040874588, 3152644489, 3277572880, 3795585437, - 3869959934, 238446394, 3101079227, 203577834, 2230529041, 1087760616, 653319712, - 3024925346, 2676624544, 2035972330, 741014356, 3839534784, 974511421, 1456450936, - 429171245, 598362053, 602320448, 1540218139, 1219580025, 1549741331, 401740328, - 1068872823, 2195381148, 4099775516, 1451717480, 412110161, 1699607068, 3715984838, - 27774796, 1138983585, 1531670562, 837921790, 2060213898, 95204061, 1507155668, - 90279751, 1446133066, 774471092, 3908131532, 3767856445, 1136546910, 2672625648, - 1649125731, 2745551516, 1585868430, 857721974, 1470900829, 64550776, 3252081369, - 1649342279, 378546910, 3444980597, 3134750739, 1010105258, 1395608241, 1003208120, - 3960524028, 3200241620, 3079969898, 1508044775, 4153769914, 2838198142, 773928818, - 25836261, 3812652461, 3870603819, 931071963, 2565579710, 2930918109, 1091097445, - 2223628368, 2934719684, 430797922, 2102409587, 720592077, 1675280068, 2578226918, - 1316600845, 3427946098, 682896800, 1861125007, 251332674, 1502078012, 3020904394, - 1458399451, 3088315263, 1635399147, 3605560130, 1755669804, 3754169290, 962708070, - 3896576937, 3764679740, 2707457262, 3082551065, 1558451132, 1046028905, 3206335691, - 731828142, 1765772975, 1195727587, 2664218451, 4234957963, 4073757168, 3230123616, - 683981262, 607599877, 3659273671, 3230354324, 3393069074, 3250708814, 2229361194, - 3923623619, 4093221649, 2441971643, 1787414237, 3603907850, 2254399656, 3508336126, - 3578571587, 2383062806, 1012097006, 4250629546, 2086981615, 4190388250, 1399942361, - 400707931, 3159618664, 2129750192, 911779896, 2736829998, 1909644954, 2168355517, - 1583901478, 3468891177, 509297602, 769296769, 1865028750, 43276967, 3375387845, - 2647467777, 1544731454, 3479333955, 4279441447, 485490313, 3523606596, 2880752852, - 1946443431, 3760803481, 3115278477, 3693898557, 2387822435, 334412170, 2054111717, - 269013084, 170401553, 3456013554, 3395842846, 508189059, 3398715186, 3862791669, - 2741070272, 65318715, 1933435210, 4086198650, 3033193928, 1242412691, 3397363281, - 3187419149, 1019508117, 562380742, 961415837, 2990412400, 3597720222, 2754100415, - 1793508822, 966564784, 1875616532, 394646945, 1386107842, 2750810827, 2931007937, - 3356489930, 2358364634, 3703772713, 3188884403, 2793590498, 3285138686, 515002680, - 521510516, 887213583, 1873460781, 1583027667, 613895001, 557578628, 1892686243, - 1974572772, 595831726, 229299738, 3847982294, 448248098, 1493822844, 3326405260, - 2752463855, 128616819, - }, - { - 2014765631, 3938779303, 1811986049, 2101875601, 887194972, 3966228860, 3450775746, - 4026039255, 4082308025, 432404123, 3181099213, 1228097256, 2368258457, 2953933351, - 2582232464, 1657799516, 3525421629, 3927364159, 978138132, 3603597064, 342571522, - 2100072168, 676229632, 2062864895, 3713317279, 4255773013, 1179492389, 3501226604, - 1641801113, 2066614519, 3303232023, 1717768923, 2333501450, 4246883128, 1655087824, - 1547897374, 754215285, 2832638094, 1365153701, 3309513970, 765738141, 1177808869, - 324127419, 1171195868, 3494966448, 1714410667, 3645762436, 603810292, 489970006, - 3126166124, 1616642501, 2646836239, 734727001, 118064420, 1164195304, 3692353914, - 2267623847, 405457397, 2510437259, 3245015882, 2110859730, 967046702, 265790493, - 3007163818, 3178475505, 1784447992, 3662964942, 1509963062, 1867864652, 1377871790, - 4185567242, 657897796, 1762205546, 3895944199, 693988565, 2359023506, 1667660316, - 478341078, 1898651054, 2352226314, 202416031, 855532642, 2290137962, 1573485803, - 3675269487, 2346740592, 3111005795, 1741227661, 1222572879, 2176146608, 1595608675, - 197451178, 1729118168, 876472937, 3201705210, 2315408645, 699968623, 2185639066, - 3960900430, 539499973, 4226318752, 266371152, 2714506838, 945022093, 1378798863, - 1925020181, 574417318, 1341455098, 3664225722, 3020780778, 3256613994, 2081255019, - 3082744844, 3572803922, 1726854692, 1167777002, 557257486, 3577067012, 3806913480, - 1578577194, 2631896730, 3937479909, 826138924, 670145071, 832017019, 715875283, - 2008704579, 804955545, 4184114494, 867004874, 3586861289, 1190193155, 3288754776, - 2271906590, 2007547109, 2206084232, 1621944575, 973990634, 3981493346, 1972746975, - 1040348653, 2895228417, 691038334, 775610724, 3837025597, 714850057, 2912426839, - 774555258, 3553866253, 2096154755, 1645117543, 3401470072, 2056183169, 2029528044, - 3145728013, 1090530001, 49523828, 2228313334, 4013648604, 4289025873, 2749397923, - 2014765631, 3938779303, 1811986049, 2101875601, 1575247143, 3966228860, 3450775746, - 4026039255, 4082308025, 532118065, 3181099213, 1228097256, 2368258457, 2953933351, - 896129082, 1657799516, 3525421629, 3927364159, 978138132, 3403930517, 342571522, - 2100072168, 676229632, 2062864895, 38934050, 4255773013, 1179492389, 3501226604, - 1641801113, 4195983797, 3303232023, 1717768923, 2333501450, 4246883128, 2854551758, - 1547897374, 754215285, 2832638094, 1365153701, 1406220165, 765738141, 1177808869, - 324127419, 1171195868, 625985592, 1714410667, 3645762436, 603810292, 489970006, - 344948229, 1616642501, 2646836239, 734727001, 118064420, 3360380275, 3692353914, - 2267623847, 405457397, 2510437259, 3697919521, 2110859730, 967046702, 265790493, - 3007163818, 1395299303, 1784447992, 3662964942, 1509963062, 1867864652, 849544728, - 4185567242, 657897796, 1762205546, 3895944199, 1219998053, 2359023506, 1667660316, - 478341078, 1898651054, 943166064, 202416031, 855532642, 2290137962, 1573485803, - 4238971908, 2346740592, 3111005795, 1741227661, 1222572879, 2670048596, 1595608675, - 197451178, 1729118168, 876472937, 94688481, 2315408645, 699968623, 2185639066, - 3960900430, 1224638706, 4226318752, 266371152, 2714506838, 945022093, 2683523818, - 1925020181, 574417318, 1341455098, 3664225722, 1168593559, 3256613994, 2081255019, - 3082744844, 3572803922, 2816021735, 1167777002, 557257486, 3577067012, 3806913480, - 740433434, 2631896730, 3937479909, 826138924, 670145071, 4127240538, 715875283, - 2008704579, 804955545, 4184114494, 1692532062, 3586861289, 1190193155, 3288754776, - 2271906590, 922306057, 2206084232, 1621944575, 973990634, 3981493346, 3555073644, - 1040348653, 2895228417, 691038334, 775610724, 1798939042, 714850057, 2912426839, - 774555258, 3553866253, 4209859609, 1645117543, 3401470072, 2056183169, 2029528044, - 2169159734, 1090530001, 49523828, 2228313334, 4013648604, 1499948031, 2749397923, - 2014765631, 3938779303, 1811986049, 2169095159, 1575247143, 3966228860, 3450775746, - 4026039255, 1220311069, 532118065, 3181099213, 1228097256, 2368258457, 649921441, - 896129082, 1657799516, 3525421629, 3927364159, 2546335470, 3403930517, 342571522, - 2100072168, 676229632, 3090148374, 38934050, 4255773013, 1179492389, 3501226604, - 2613176152, 4195983797, 3303232023, 1717768923, 2333501450, 3923041739, 2854551758, - 1547897374, 754215285, 2832638094, 731392091, 1406220165, 765738141, 1177808869, - 324127419, 12876722, 625985592, 1714410667, 3645762436, 603810292, 2789313462, - 344948229, 1616642501, 2646836239, 734727001, 3369486664, 3360380275, 3692353914, - 2267623847, 405457397, 4284067044, 3697919521, 2110859730, 967046702, 265790493, - 2779045063, 1395299303, 1784447992, 3662964942, 1509963062, 3233239196, 849544728, - 4185567242, 657897796, 1762205546, 2086953994, 1219998053, 2359023506, 1667660316, - 478341078, 4137166515, 943166064, 202416031, 855532642, 2290137962, 1053737970, - 4238971908, 2346740592, 3111005795, 1741227661, 3570501235, 2670048596, 1595608675, - 197451178, 1729118168, 4162077327, 94688481, 2315408645, 699968623, 2185639066, - 2842562270, 1224638706, 4226318752, 266371152, 2714506838, 755620309, 2683523818, - 1925020181, 574417318, 1341455098, 3323621213, 1168593559, 3256613994, 2081255019, - 3082744844, 3845230416, 2816021735, 1167777002, 557257486, 3577067012, 66225918, - 740433434, 2631896730, 3937479909, 826138924, 522548622, 4127240538, 715875283, - 2008704579, 804955545, 22190845, 1692532062, 3586861289, 1190193155, 3288754776, - 610751818, 922306057, 2206084232, 1621944575, 973990634, 1771882567, 3555073644, - 1040348653, 2895228417, 691038334, 149995790, 1798939042, 714850057, 2912426839, - 774555258, 2020442761, 4209859609, 1645117543, 3401470072, 2056183169, 460813741, - 2169159734, 1090530001, 49523828, 2228313334, 504317288, 1499948031, 2749397923, - 2014765631, 3938779303, 1175469033, 2169095159, 1575247143, 3966228860, 3450775746, - 1032079910, 1220311069, 532118065, 3181099213, 1228097256, 3353583885, 649921441, - 896129082, 1657799516, 3525421629, 2830418357, 2546335470, 3403930517, 342571522, - 2100072168, 1099053459, 3090148374, 38934050, 4255773013, 1179492389, 1634035942, - 2613176152, 4195983797, 3303232023, 1717768923, 4175785502, 3923041739, 2854551758, - 1547897374, 754215285, 3674851127, 731392091, 1406220165, 765738141, 1177808869, - 776475327, 12876722, 625985592, 1714410667, 3645762436, 759189800, 2789313462, - 344948229, 1616642501, 2646836239, 1228911282, 3369486664, 3360380275, 3692353914, - 2267623847, 3058856811, 4284067044, 3697919521, 2110859730, 967046702, 3601311392, - 2779045063, 1395299303, 1784447992, 3662964942, 284169442, 3233239196, 849544728, - 4185567242, 657897796, 905886381, 2086953994, 1219998053, 2359023506, 1667660316, - 2784311626, 4137166515, 943166064, 202416031, 855532642, 895862877, 1053737970, - 4238971908, 2346740592, 3111005795, 1509264114, 3570501235, 2670048596, 1595608675, - 197451178, 3653040124, 4162077327, 94688481, 2315408645, 699968623, 1071988392, - 2842562270, 1224638706, 4226318752, 266371152, 4214356293, 755620309, 2683523818, - 1925020181, 574417318, 119162126, 3323621213, 1168593559, 3256613994, 2081255019, - 1317924999, 3845230416, 2816021735, 1167777002, 557257486, 638058809, 66225918, - 740433434, 2631896730, 3937479909, 411228024, 522548622, 4127240538, 715875283, - 2008704579, 2246049355, 22190845, 1692532062, 3586861289, 1190193155, 4078828073, - 610751818, 922306057, 2206084232, 1621944575, 907181435, 1771882567, 3555073644, - 1040348653, 2895228417, 940846326, 149995790, 1798939042, 714850057, 2912426839, - 3345560812, 2020442761, 4209859609, 1645117543, 3401470072, 2036328600, 460813741, - 2169159734, 1090530001, 49523828, 510136795, 2233313725, 1046048857, 700202384, - 926275751, 4104982908, 3910999868, 1125213128, 492681420, 3891914731, 956545470, - 115696875, 2957144177, 77090391, 467732901, 2599813105, 3888976883, 2098926023, - 2844817051, 2069408123, 2239429902, 3793992219, 3020240490, 1721698056, 3012473888, - 1537226153, 725991171, 61376035, 381912667, 3904514327, 40015731, 4263210119, - 2876064791, 2732785471, 2934318283, 3134934287, 3161129365, 3789733734, 2954419388, - 2742205850, 3488450208, 3252908703, 410643191, 3246033194, 2846558783, 828879673, - 2428687670, 389617242, 63987225, 439842832, 2635895570, 3468152776, 4086700701, - 3370617315, 2400127386, 4266992260, 3026019128, 4225721108, 1328114488, 2808680961, - 3574018824, 4060262451, 2329039960, 1165344648, 3635963149, 2414703792, 2269100254, - 832995451, 2143875746, 1031309981, 2129333746, 2606784227, 805236091, 666141116, - 2749351381, 53998350, 2270447972, 2092784991, 877961283, 3019419608, 2438459472, - 2273921167, 332279281, 3759993687, 2465113760, 3732237006, 322823266, 491053374, - 686619591, 4192648122, 4118497267, 1948902148, 988375775, 2098328675, 3107501958, - 2979856583, 2274193457, 6179961, 188209161, 1491245003, 3005972885, 1658799053, - 3420802262, 2128792168, 3272743598, 4163716849, 817350318, 3372322557, 1525295885, - 490587460, 3634834949, 2584809384, 3638373352, 2603765126, 3223396315, 4021061386, - 2143780551, 248332433, 3654752967, 27201989, 3994156272, 5505477, 4260955724, - 1175998822, 2665646240, 866875674, 3569324422, 202962714, 896177244, 3146417201, - 1168925859, 2228636445, 105395449, 2567482935, 1301265751, 3739617610, 2486939910, - 1891847857, 2647840744, 1141826566, 3360553996, 4267671927, 2546157872, 1143297884, - 2049385137, 4288036836, 3347190546, 3480408604, 2756408254, 2396048567, 1151717877, - 2211243289, 4221659024, 21773193, 1665317870, 3116384869, 3231689469, 3689471824, - 1520574310, 463615415, - }, - { - 2647200400, 1592194261, 1289872755, 2079982087, 3431487085, 1101851783, 3373292799, - 2148994262, 2785319928, 3206527339, 2731839331, 1280366172, 1146205735, 2930495205, - 2876978398, 3885758458, 2062812458, 2448488486, 192141900, 3861696664, 2677929258, - 3606729729, 2920965773, 1156521508, 3168665640, 298794036, 957896625, 2606719899, - 3699219026, 3120096333, 2531109351, 1920936462, 2848868175, 1406404729, 2956672675, - 1376226240, 3667482110, 2551426756, 3433640449, 2603906744, 4217864690, 47671552, - 2993859190, 1269153270, 3463588775, 1655126430, 3453916724, 2157890969, 252769449, - 1583335064, 2560819344, 52639671, 39618615, 313192112, 2625914283, 2964928555, - 4226359627, 4141969666, 183405146, 1455378225, 2994063945, 3146629795, 2992956289, - 368634554, 4110058153, 1156556441, 3690317172, 906928962, 3773042217, 948650679, - 4134172918, 2922802573, 1417921660, 291400676, 3120733115, 3225369425, 3200455006, - 2207799613, 1766261260, 914727455, 1927023103, 572959294, 3447057855, 4042691162, - 840021910, 4187195325, 3627831667, 1671506539, 242673485, 3330397756, 776552069, - 684550924, 261597601, 1857936262, 1022869402, 4209077103, 14248159, 2366156245, - 1910356465, 2020463550, 873419743, 4290775093, 3670978210, 1726974037, 784115717, - 3574834402, 357805142, 3820795621, 1854247318, 1161642656, 3977404318, 2054677775, - 1737374322, 2852015019, 4277252452, 1392810771, 3742661504, 1900815804, 1965911170, - 3540183220, 2106191537, 3606954134, 2108636204, 2981827052, 2506861567, 184003599, - 3319252632, 1370913077, 940955681, 2244100002, 3683661822, 3215832318, 3463899341, - 134577035, 3404365179, 2262494389, 88039196, 114405083, 1071449574, 4008494055, - 765981248, 758357266, 2564125377, 901977407, 955156196, 3900980822, 134031448, - 2566915950, 3445833537, 3138903399, 2113076217, 713587277, 3396078039, 3987657193, - 3004104774, 800324742, 652529813, 3999083342, 486108562, 2103591900, 104743736, - 2647200400, 1592194261, 1289872755, 2079982087, 552781204, 1101851783, 3373292799, - 2148994262, 2785319928, 1222851809, 2731839331, 1280366172, 1146205735, 2930495205, - 942360591, 3885758458, 2062812458, 2448488486, 192141900, 1909229999, 2677929258, - 3606729729, 2920965773, 1156521508, 2995805883, 298794036, 957896625, 2606719899, - 3699219026, 2447513005, 2531109351, 1920936462, 2848868175, 1406404729, 2751142611, - 1376226240, 3667482110, 2551426756, 3433640449, 3724189478, 4217864690, 47671552, - 2993859190, 1269153270, 2144136371, 1655126430, 3453916724, 2157890969, 252769449, - 2959496542, 2560819344, 52639671, 39618615, 313192112, 2367743540, 2964928555, - 4226359627, 4141969666, 183405146, 2006751422, 2994063945, 3146629795, 2992956289, - 368634554, 1529794973, 1156556441, 3690317172, 906928962, 3773042217, 2005599428, - 4134172918, 2922802573, 1417921660, 291400676, 2664982078, 3225369425, 3200455006, - 2207799613, 1766261260, 2623711877, 1927023103, 572959294, 3447057855, 4042691162, - 3510199782, 4187195325, 3627831667, 1671506539, 242673485, 1978730938, 776552069, - 684550924, 261597601, 1857936262, 3273582958, 4209077103, 14248159, 2366156245, - 1910356465, 457933823, 873419743, 4290775093, 3670978210, 1726974037, 1414288023, - 3574834402, 357805142, 3820795621, 1854247318, 2679386897, 3977404318, 2054677775, - 1737374322, 2852015019, 2411291453, 1392810771, 3742661504, 1900815804, 1965911170, - 3719529323, 2106191537, 3606954134, 2108636204, 2981827052, 3702357099, 184003599, - 3319252632, 1370913077, 940955681, 4262675711, 3683661822, 3215832318, 3463899341, - 134577035, 3494669542, 2262494389, 88039196, 114405083, 1071449574, 1060831201, - 765981248, 758357266, 2564125377, 901977407, 3003279383, 3900980822, 134031448, - 2566915950, 3445833537, 2846863035, 2113076217, 713587277, 3396078039, 3987657193, - 2067196130, 800324742, 652529813, 3999083342, 486108562, 2321935002, 104743736, - 2647200400, 1592194261, 1289872755, 466892855, 552781204, 1101851783, 3373292799, - 2148994262, 3078568050, 1222851809, 2731839331, 1280366172, 1146205735, 1710937426, - 942360591, 3885758458, 2062812458, 2448488486, 3418446265, 1909229999, 2677929258, - 3606729729, 2920965773, 1103324742, 2995805883, 298794036, 957896625, 2606719899, - 675602173, 2447513005, 2531109351, 1920936462, 2848868175, 1509959171, 2751142611, - 1376226240, 3667482110, 2551426756, 2447143807, 3724189478, 4217864690, 47671552, - 2993859190, 2821422976, 2144136371, 1655126430, 3453916724, 2157890969, 3665277070, - 2959496542, 2560819344, 52639671, 39618615, 2817859210, 2367743540, 2964928555, - 4226359627, 4141969666, 2856219617, 2006751422, 2994063945, 3146629795, 2992956289, - 3176479073, 1529794973, 1156556441, 3690317172, 906928962, 97899274, 2005599428, - 4134172918, 2922802573, 1417921660, 1492426675, 2664982078, 3225369425, 3200455006, - 2207799613, 2275640124, 2623711877, 1927023103, 572959294, 3447057855, 1036984002, - 3510199782, 4187195325, 3627831667, 1671506539, 1827895694, 1978730938, 776552069, - 684550924, 261597601, 3493571621, 3273582958, 4209077103, 14248159, 2366156245, - 211745521, 457933823, 873419743, 4290775093, 3670978210, 1909994881, 1414288023, - 3574834402, 357805142, 3820795621, 1911700755, 2679386897, 3977404318, 2054677775, - 1737374322, 213019511, 2411291453, 1392810771, 3742661504, 1900815804, 237536256, - 3719529323, 2106191537, 3606954134, 2108636204, 665423205, 3702357099, 184003599, - 3319252632, 1370913077, 3583034472, 4262675711, 3683661822, 3215832318, 3463899341, - 4027471772, 3494669542, 2262494389, 88039196, 114405083, 3580272354, 1060831201, - 765981248, 758357266, 2564125377, 2592170747, 3003279383, 3900980822, 134031448, - 2566915950, 1722483656, 2846863035, 2113076217, 713587277, 3396078039, 244197359, - 2067196130, 800324742, 652529813, 3999083342, 2310369213, 2321935002, 104743736, - 2647200400, 1592194261, 1610483859, 466892855, 552781204, 1101851783, 3373292799, - 2617595614, 3078568050, 1222851809, 2731839331, 1280366172, 808483717, 1710937426, - 942360591, 3885758458, 2062812458, 3260452154, 3418446265, 1909229999, 2677929258, - 3606729729, 341113837, 1103324742, 2995805883, 298794036, 957896625, 2309730124, - 675602173, 2447513005, 2531109351, 1920936462, 2268824429, 1509959171, 2751142611, - 1376226240, 3667482110, 2745634237, 2447143807, 3724189478, 4217864690, 47671552, - 2787057737, 2821422976, 2144136371, 1655126430, 3453916724, 339095616, 3665277070, - 2959496542, 2560819344, 52639671, 3200765881, 2817859210, 2367743540, 2964928555, - 4226359627, 3206913703, 2856219617, 2006751422, 2994063945, 3146629795, 1042016834, - 3176479073, 1529794973, 1156556441, 3690317172, 171871257, 97899274, 2005599428, - 4134172918, 2922802573, 1501051393, 1492426675, 2664982078, 3225369425, 3200455006, - 1356823782, 2275640124, 2623711877, 1927023103, 572959294, 319456586, 1036984002, - 3510199782, 4187195325, 3627831667, 3026392291, 1827895694, 1978730938, 776552069, - 684550924, 2862336749, 3493571621, 3273582958, 4209077103, 14248159, 1597498830, - 211745521, 457933823, 873419743, 4290775093, 2990300609, 1909994881, 1414288023, - 3574834402, 357805142, 1957211849, 1911700755, 2679386897, 3977404318, 2054677775, - 1006075205, 213019511, 2411291453, 1392810771, 3742661504, 1443139437, 237536256, - 3719529323, 2106191537, 3606954134, 2671394121, 665423205, 3702357099, 184003599, - 3319252632, 1632983188, 3583034472, 4262675711, 3683661822, 3215832318, 4080585934, - 4027471772, 3494669542, 2262494389, 88039196, 677218369, 3580272354, 1060831201, - 765981248, 758357266, 1277026792, 2592170747, 3003279383, 3900980822, 134031448, - 4189207981, 1722483656, 2846863035, 2113076217, 713587277, 2098603934, 244197359, - 2067196130, 800324742, 652529813, 1307843279, 3205610699, 1606722715, 2749781905, - 3500078806, 320007706, 4092615096, 608085660, 1869480444, 459160631, 3657609957, - 1944540526, 2184854884, 3497113751, 2817682182, 3367276652, 2069300794, 1466691974, - 3078806052, 3998756116, 2068892089, 1789981386, 4196184114, 4004623319, 3029515569, - 3206332209, 3424306963, 1805804276, 899469644, 1149853995, 903917909, 1185042552, - 447265042, 3579272434, 116409560, 2211704275, 1237721838, 3636618157, 3191931082, - 2430339315, 3551966793, 1533877057, 1700891210, 3317627852, 828148584, 1733460943, - 3866870689, 3970886915, 1624935507, 3236099078, 4209593953, 1951283095, 1579020365, - 1165668813, 1423097998, 1294879824, 3406063424, 3313007028, 2090501113, 842064167, - 729587893, 2949702260, 2099637920, 260436310, 1056109549, 657874983, 2734575906, - 4088958435, 3265216971, 1081848592, 2593212854, 4028921684, 2868974814, 2299228627, - 49944924, 955114665, 2844328062, 885136505, 4262681333, 977883895, 998890598, - 2026602293, 87852872, 2197997810, 910583259, 3151223623, 3960726944, 1778982325, - 3322631234, 2940402694, 1619768059, 1592832128, 1434542537, 2107314297, 1170789408, - 3357335254, 3317662644, 1409884338, 73741139, 1093507243, 329113525, 4199840577, - 442295615, 3348669654, 435948047, 1154137005, 3151357655, 2101029905, 2430218233, - 2474305449, 2316834456, 1736616135, 1575712778, 370866908, 4256943043, 2805503887, - 1099763491, 2473785999, 3215573143, 472701386, 3070116154, 3969279119, 3331310102, - 3932945670, 1502564397, 1294139579, 3073834823, 3115143551, 3602082994, 3707103500, - 2570195094, 1268510174, 3561337287, 112422529, 1483304061, 3712148969, 3729628891, - 2741131557, 4035019342, 2395091348, 208448216, 607199897, 4049058939, 3463267226, - 3821711834, 1697628853, 691151709, 3014869414, 11610545, 3895793639, 3019679196, - 1246664817, 753245113, 2236232962, 4172861179, 4030183420, 2367787106, 2209331085, - 4170801007, 3609895913, - }, - { - 930278208, 223382535, 720499309, 2613473585, 4173439516, 2132019243, 468054579, - 1141433627, 1328639101, 3222455434, 4023859457, 892124224, 2940688706, 2894552260, - 1595432126, 2558709596, 2057191226, 1116728192, 3767370344, 1457278707, 3171850455, - 3733161247, 149922078, 3860652874, 743952057, 1024625539, 3982786483, 2077838781, - 3713742913, 2790452624, 3014482913, 2928857967, 476371337, 611803300, 3000092437, - 57069608, 1554852195, 1406780525, 2288998898, 460131340, 3945168588, 18495216, - 547882902, 1624966119, 2229423551, 1492565146, 706052879, 2733955743, 1450476708, - 2565285196, 2491601298, 850297175, 331472128, 3275065709, 3154247398, 1364512871, - 1193063601, 579449294, 4097747196, 3572372000, 2712499116, 1172861307, 3964137156, - 1300564854, 1057993198, 2785733262, 3548277076, 2572944411, 3299232577, 2031854568, - 2468534978, 602097973, 2068619195, 2639336890, 1694467033, 1581263823, 809076686, - 2892861850, 4042078087, 3178152001, 706023882, 3236709493, 3603158102, 2575690800, - 2831218686, 2492604085, 207296828, 1561595438, 2961967115, 3304283504, 835276604, - 3005485731, 58946395, 3979071161, 1560535337, 2679058432, 1061627241, 1142692919, - 1476802977, 1306619165, 1297953898, 4282928317, 3630719944, 2305895643, 2656730970, - 916308118, 4160016206, 3541795573, 4222235077, 1289754532, 1963633184, 3595798857, - 2273299603, 1687478595, 2249344966, 2267127964, 2201115693, 917609614, 3731921025, - 1634893875, 3039440017, 1122674005, 2906728840, 921166852, 3525309936, 633872502, - 821930665, 3861074060, 3309559821, 304858441, 1530517912, 1140212033, 3168869568, - 3223449972, 1343718360, 2831361172, 1723616626, 3675867172, 2586694335, 2374941766, - 387033391, 1528180036, 1561421035, 2735360720, 3952587140, 13543969, 3987997725, - 4273177532, 2200558169, 461920718, 459441276, 4225054447, 2248193798, 1103878063, - 3027778665, 1844457031, 1364117386, 1575430424, 2276483962, 2665252582, 1572038262, - 930278208, 223382535, 720499309, 2613473585, 4025056228, 2132019243, 468054579, - 1141433627, 1328639101, 304940359, 4023859457, 892124224, 2940688706, 2894552260, - 2006939659, 2558709596, 2057191226, 1116728192, 3767370344, 3026555841, 3171850455, - 3733161247, 149922078, 3860652874, 2068299929, 1024625539, 3982786483, 2077838781, - 3713742913, 2512419366, 3014482913, 2928857967, 476371337, 611803300, 259065762, - 57069608, 1554852195, 1406780525, 2288998898, 2261401631, 3945168588, 18495216, - 547882902, 1624966119, 3049748661, 1492565146, 706052879, 2733955743, 1450476708, - 910808481, 2491601298, 850297175, 331472128, 3275065709, 3877736250, 1364512871, - 1193063601, 579449294, 4097747196, 3029512053, 2712499116, 1172861307, 3964137156, - 1300564854, 2398462790, 2785733262, 3548277076, 2572944411, 3299232577, 3497485227, - 2468534978, 602097973, 2068619195, 2639336890, 4271191874, 1581263823, 809076686, - 2892861850, 4042078087, 3046259144, 706023882, 3236709493, 3603158102, 2575690800, - 591682100, 2492604085, 207296828, 1561595438, 2961967115, 3885379584, 835276604, - 3005485731, 58946395, 3979071161, 2784795951, 2679058432, 1061627241, 1142692919, - 1476802977, 2864266022, 1297953898, 4282928317, 3630719944, 2305895643, 3374260620, - 916308118, 4160016206, 3541795573, 4222235077, 3025355241, 1963633184, 3595798857, - 2273299603, 1687478595, 4115795122, 2267127964, 2201115693, 917609614, 3731921025, - 2905712346, 3039440017, 1122674005, 2906728840, 921166852, 2881663141, 633872502, - 821930665, 3861074060, 3309559821, 2816533968, 1530517912, 1140212033, 3168869568, - 3223449972, 1894667948, 2831361172, 1723616626, 3675867172, 2586694335, 3974041178, - 387033391, 1528180036, 1561421035, 2735360720, 2014321457, 13543969, 3987997725, - 4273177532, 2200558169, 2259553303, 459441276, 4225054447, 2248193798, 1103878063, - 3889361594, 1844457031, 1364117386, 1575430424, 2276483962, 3302182736, 1572038262, - 930278208, 223382535, 720499309, 4173186621, 4025056228, 2132019243, 468054579, - 1141433627, 2396654717, 304940359, 4023859457, 892124224, 2940688706, 2903529759, - 2006939659, 2558709596, 2057191226, 1116728192, 715931354, 3026555841, 3171850455, - 3733161247, 149922078, 3342675578, 2068299929, 1024625539, 3982786483, 2077838781, - 1157097180, 2512419366, 3014482913, 2928857967, 476371337, 2192872017, 259065762, - 57069608, 1554852195, 1406780525, 4165039782, 2261401631, 3945168588, 18495216, - 547882902, 2453072030, 3049748661, 1492565146, 706052879, 2733955743, 2233423433, - 910808481, 2491601298, 850297175, 331472128, 1154483111, 3877736250, 1364512871, - 1193063601, 579449294, 690173400, 3029512053, 2712499116, 1172861307, 3964137156, - 2683130322, 2398462790, 2785733262, 3548277076, 2572944411, 4075824857, 3497485227, - 2468534978, 602097973, 2068619195, 2711665545, 4271191874, 1581263823, 809076686, - 2892861850, 3558962856, 3046259144, 706023882, 3236709493, 3603158102, 274706518, - 591682100, 2492604085, 207296828, 1561595438, 1532885415, 3885379584, 835276604, - 3005485731, 58946395, 4143205928, 2784795951, 2679058432, 1061627241, 1142692919, - 2539503297, 2864266022, 1297953898, 4282928317, 3630719944, 3333189589, 3374260620, - 916308118, 4160016206, 3541795573, 1771535488, 3025355241, 1963633184, 3595798857, - 2273299603, 1735171204, 4115795122, 2267127964, 2201115693, 917609614, 4220503034, - 2905712346, 3039440017, 1122674005, 2906728840, 868453017, 2881663141, 633872502, - 821930665, 3861074060, 1928586970, 2816533968, 1530517912, 1140212033, 3168869568, - 1082127627, 1894667948, 2831361172, 1723616626, 3675867172, 496773835, 3974041178, - 387033391, 1528180036, 1561421035, 2763161987, 2014321457, 13543969, 3987997725, - 4273177532, 2110570579, 2259553303, 459441276, 4225054447, 2248193798, 53021618, - 3889361594, 1844457031, 1364117386, 1575430424, 1105247032, 3302182736, 1572038262, - 930278208, 223382535, 2159964170, 4173186621, 4025056228, 2132019243, 468054579, - 1397544344, 2396654717, 304940359, 4023859457, 892124224, 2800429255, 2903529759, - 2006939659, 2558709596, 2057191226, 296054924, 715931354, 3026555841, 3171850455, - 3733161247, 863280930, 3342675578, 2068299929, 1024625539, 3982786483, 949122664, - 1157097180, 2512419366, 3014482913, 2928857967, 2585465463, 2192872017, 259065762, - 57069608, 1554852195, 3650462338, 4165039782, 2261401631, 3945168588, 18495216, - 524715648, 2453072030, 3049748661, 1492565146, 706052879, 123143857, 2233423433, - 910808481, 2491601298, 850297175, 3272095697, 1154483111, 3877736250, 1364512871, - 1193063601, 2394240337, 690173400, 3029512053, 2712499116, 1172861307, 598335483, - 2683130322, 2398462790, 2785733262, 3548277076, 678275336, 4075824857, 3497485227, - 2468534978, 602097973, 1861912463, 2711665545, 4271191874, 1581263823, 809076686, - 3324887617, 3558962856, 3046259144, 706023882, 3236709493, 1776103512, 274706518, - 591682100, 2492604085, 207296828, 1739697610, 1532885415, 3885379584, 835276604, - 3005485731, 2931144546, 4143205928, 2784795951, 2679058432, 1061627241, 1487949699, - 2539503297, 2864266022, 1297953898, 4282928317, 4101955339, 3333189589, 3374260620, - 916308118, 4160016206, 376029432, 1771535488, 3025355241, 1963633184, 3595798857, - 2826786027, 1735171204, 4115795122, 2267127964, 2201115693, 2572535497, 4220503034, - 2905712346, 3039440017, 1122674005, 2482828099, 868453017, 2881663141, 633872502, - 821930665, 2579406681, 1928586970, 2816533968, 1530517912, 1140212033, 2547368381, - 1082127627, 1894667948, 2831361172, 1723616626, 1903980411, 496773835, 3974041178, - 387033391, 1528180036, 2681142643, 2763161987, 2014321457, 13543969, 3987997725, - 2583502227, 2110570579, 2259553303, 459441276, 4225054447, 177868402, 53021618, - 3889361594, 1844457031, 1364117386, 2369166739, 240269378, 689700242, 297384154, - 1052178701, 2154172820, 614713903, 3000863907, 3916962502, 94341217, 2609111975, - 1621831476, 4260159710, 694869580, 1708268072, 2751452128, 3843916827, 3400387883, - 2394104046, 2348934617, 3263438569, 3818556032, 1695621950, 410888855, 347231182, - 612084657, 1858306225, 3024940417, 2482215564, 2728249904, 2825132299, 329106327, - 3333110741, 2742025573, 2947035922, 3758718780, 2191527983, 864130510, 2586839659, - 662702978, 817620197, 2888275812, 3372817000, 2982240654, 2211025418, 2043458594, - 498221898, 1559803796, 209509183, 3004637012, 2204871924, 2445352606, 4026842262, - 3211433366, 3533095828, 4172447076, 865408944, 2797639687, 3201749441, 1286664278, - 1158271235, 2641361834, 4261559289, 3643706696, 2017210420, 2067296744, 3548126272, - 3846378526, 3885857009, 3013926193, 368948443, 3839554625, 2032663713, 4185819024, - 4279332940, 137321733, 3515190288, 4281845500, 2738024944, 3350239126, 1456676856, - 1246688651, 2478709188, 12570320, 989306366, 2347610707, 2849134988, 2351681449, - 4063448910, 1193872626, 3645565330, 1863134777, 1345198234, 2504863006, 3815974850, - 3075495578, 2400383333, 2727346254, 985812393, 1432182882, 3668977714, 231840487, - 647229148, 274547428, 2856186783, 1273158535, 900081267, 1566366419, 562584841, - 2247144789, 3522587233, 1686032132, 586483076, 1207387616, 3040778905, 2532774045, - 3681992451, 1034866888, 4029685195, 3307070989, 2412903766, 3156200186, 2625083166, - 4148004113, 1756566287, 2319065855, 2924909429, 3050022486, 2464491722, 1137782196, - 2749457821, 790410752, 3511746957, 2277733721, 2871407058, 3858561909, 2176119631, - 952943025, 2987154266, 120799539, 2862346597, 3689389598, 3329995989, 715438735, - 1035277216, 3079684809, 677298106, 2364292665, 4110165256, 4096954153, 356732100, - 2361282903, 4050817284, 2010946835, 1824397679, 4087204231, 4178036725, 4197370951, - 503070461, 1879838906, - }, - { - 4117851084, 2941903397, 1156439261, 1922510465, 2925632294, 2272105738, 641404242, - 3414739418, 2602896978, 672876430, 1998875331, 1325970749, 1633717408, 3567722815, - 2088144733, 95705225, 580635702, 3543633503, 1469889369, 239816045, 2254984383, - 1632695776, 2033839470, 4117902893, 509938588, 1291002316, 3600816069, 2962644092, - 4269959520, 3161890066, 1908855486, 1177948589, 473118979, 3205649854, 2027137481, - 227656706, 1485922673, 3380103860, 2080286336, 2588604114, 1727893393, 3602757903, - 3126385963, 2101893784, 3058515017, 833779022, 719369683, 3768029740, 1123855192, - 2580550821, 3694463505, 1137588651, 1724433728, 3847324234, 2368689516, 1226895255, - 1126753016, 2557024841, 3187601018, 3790080711, 2423256074, 2463913828, 1753321774, - 1621519784, 3456900204, 3550875802, 3783120790, 2740104077, 2010858632, 1569277627, - 1492853575, 2182681191, 3866043645, 2566155095, 770150438, 2482504045, 3916834400, - 222960658, 342285296, 3354506276, 1371039946, 3717269950, 3632913684, 2557531969, - 3934379214, 1732115898, 1598596195, 1180866173, 3526785234, 2740387380, 3540138766, - 338607286, 3262593182, 2413619772, 2248013920, 3557851982, 2470276596, 1549877186, - 447909575, 4010548064, 282941857, 3418249797, 3300699992, 1957423733, 2615274674, - 370155667, 1109991145, 933065597, 3947818943, 3221736239, 402503017, 4016454981, - 3640556350, 243947268, 1175418215, 2752078014, 371928981, 3354635500, 3471578165, - 2735623932, 445067764, 2732367763, 3225606514, 1214718589, 2197756425, 2134958042, - 680726116, 3098695319, 2103463364, 4058022972, 2428195541, 2433504485, 4042288512, - 2383438250, 3821638336, 2375226348, 806148488, 197247918, 768984129, 412771011, - 4020619856, 3030619444, 3242554868, 282156707, 3718880754, 2938924979, 4189583150, - 1604586306, 1245779881, 4006389745, 2437150739, 1749517801, 2903749036, 1247308303, - 2580023735, 2457849017, 342934950, 216040419, 3176519601, 4151509434, 2404801649, - 4117851084, 2941903397, 1156439261, 1922510465, 14864453, 2272105738, 641404242, - 3414739418, 2602896978, 2179417586, 1998875331, 1325970749, 1633717408, 3567722815, - 428880410, 95705225, 580635702, 3543633503, 1469889369, 3132946201, 2254984383, - 1632695776, 2033839470, 4117902893, 3029657560, 1291002316, 3600816069, 2962644092, - 4269959520, 397442545, 1908855486, 1177948589, 473118979, 3205649854, 990384909, - 227656706, 1485922673, 3380103860, 2080286336, 3295033100, 1727893393, 3602757903, - 3126385963, 2101893784, 1132286601, 833779022, 719369683, 3768029740, 1123855192, - 283414013, 3694463505, 1137588651, 1724433728, 3847324234, 1735742473, 1226895255, - 1126753016, 2557024841, 3187601018, 2090644528, 2423256074, 2463913828, 1753321774, - 1621519784, 1037552449, 3550875802, 3783120790, 2740104077, 2010858632, 3730461081, - 1492853575, 2182681191, 3866043645, 2566155095, 2782805925, 2482504045, 3916834400, - 222960658, 342285296, 2406892654, 1371039946, 3717269950, 3632913684, 2557531969, - 4071148456, 1732115898, 1598596195, 1180866173, 3526785234, 1110366522, 3540138766, - 338607286, 3262593182, 2413619772, 995824548, 3557851982, 2470276596, 1549877186, - 447909575, 2962194596, 282941857, 3418249797, 3300699992, 1957423733, 1859612288, - 370155667, 1109991145, 933065597, 3947818943, 3005912276, 402503017, 4016454981, - 3640556350, 243947268, 2884057401, 2752078014, 371928981, 3354635500, 3471578165, - 908942821, 445067764, 2732367763, 3225606514, 1214718589, 4104754911, 2134958042, - 680726116, 3098695319, 2103463364, 2946640978, 2428195541, 2433504485, 4042288512, - 2383438250, 1252490765, 2375226348, 806148488, 197247918, 768984129, 2578888115, - 4020619856, 3030619444, 3242554868, 282156707, 3433259466, 2938924979, 4189583150, - 1604586306, 1245779881, 616758943, 2437150739, 1749517801, 2903749036, 1247308303, - 2722580830, 2457849017, 342934950, 216040419, 3176519601, 545097903, 2404801649, - 4117851084, 2941903397, 1156439261, 1253296096, 14864453, 2272105738, 641404242, - 3414739418, 2989955985, 2179417586, 1998875331, 1325970749, 1633717408, 1896726594, - 428880410, 95705225, 580635702, 3543633503, 3294258049, 3132946201, 2254984383, - 1632695776, 2033839470, 829668922, 3029657560, 1291002316, 3600816069, 2962644092, - 715635401, 397442545, 1908855486, 1177948589, 473118979, 443010703, 990384909, - 227656706, 1485922673, 3380103860, 727464961, 3295033100, 1727893393, 3602757903, - 3126385963, 3020775130, 1132286601, 833779022, 719369683, 3768029740, 2215591597, - 283414013, 3694463505, 1137588651, 1724433728, 2124955521, 1735742473, 1226895255, - 1126753016, 2557024841, 1719121879, 2090644528, 2423256074, 2463913828, 1753321774, - 1283364713, 1037552449, 3550875802, 3783120790, 2740104077, 3326764615, 3730461081, - 1492853575, 2182681191, 3866043645, 1353658829, 2782805925, 2482504045, 3916834400, - 222960658, 2681616579, 2406892654, 1371039946, 3717269950, 3632913684, 2373372484, - 4071148456, 1732115898, 1598596195, 1180866173, 3787873944, 1110366522, 3540138766, - 338607286, 3262593182, 1714619779, 995824548, 3557851982, 2470276596, 1549877186, - 2342751414, 2962194596, 282941857, 3418249797, 3300699992, 2080590834, 1859612288, - 370155667, 1109991145, 933065597, 4126279826, 3005912276, 402503017, 4016454981, - 3640556350, 618040940, 2884057401, 2752078014, 371928981, 3354635500, 2952377979, - 908942821, 445067764, 2732367763, 3225606514, 935181950, 4104754911, 2134958042, - 680726116, 3098695319, 652514936, 2946640978, 2428195541, 2433504485, 4042288512, - 1834165243, 1252490765, 2375226348, 806148488, 197247918, 1459523569, 2578888115, - 4020619856, 3030619444, 3242554868, 2222750155, 3433259466, 2938924979, 4189583150, - 1604586306, 400149547, 616758943, 2437150739, 1749517801, 2903749036, 571531698, - 2722580830, 2457849017, 342934950, 216040419, 3302138698, 545097903, 2404801649, - 4117851084, 2941903397, 2926001994, 1253296096, 14864453, 2272105738, 641404242, - 2446601571, 2989955985, 2179417586, 1998875331, 1325970749, 2470418771, 1896726594, - 428880410, 95705225, 580635702, 95529058, 3294258049, 3132946201, 2254984383, - 1632695776, 3381575123, 829668922, 3029657560, 1291002316, 3600816069, 332821128, - 715635401, 397442545, 1908855486, 1177948589, 3324147260, 443010703, 990384909, - 227656706, 1485922673, 3468390490, 727464961, 3295033100, 1727893393, 3602757903, - 3849734062, 3020775130, 1132286601, 833779022, 719369683, 3336941985, 2215591597, - 283414013, 3694463505, 1137588651, 1245145305, 2124955521, 1735742473, 1226895255, - 1126753016, 1536376839, 1719121879, 2090644528, 2423256074, 2463913828, 4035794571, - 1283364713, 1037552449, 3550875802, 3783120790, 4233012781, 3326764615, 3730461081, - 1492853575, 2182681191, 654850701, 1353658829, 2782805925, 2482504045, 3916834400, - 1556782509, 2681616579, 2406892654, 1371039946, 3717269950, 1227526114, 2373372484, - 4071148456, 1732115898, 1598596195, 1777009717, 3787873944, 1110366522, 3540138766, - 338607286, 1161080599, 1714619779, 995824548, 3557851982, 2470276596, 3162659171, - 2342751414, 2962194596, 282941857, 3418249797, 1032034511, 2080590834, 1859612288, - 370155667, 1109991145, 2568097099, 4126279826, 3005912276, 402503017, 4016454981, - 3196575353, 618040940, 2884057401, 2752078014, 371928981, 4223799564, 2952377979, - 908942821, 445067764, 2732367763, 174723563, 935181950, 4104754911, 2134958042, - 680726116, 649687363, 652514936, 2946640978, 2428195541, 2433504485, 3735735592, - 1834165243, 1252490765, 2375226348, 806148488, 3720638976, 1459523569, 2578888115, - 4020619856, 3030619444, 283333114, 2222750155, 3433259466, 2938924979, 4189583150, - 870522428, 400149547, 616758943, 2437150739, 1749517801, 999295363, 571531698, - 2722580830, 2457849017, 342934950, 3151292467, 2839665217, 2452261456, 208520727, - 2269948412, 344787478, 3987474076, 3770524881, 2718719281, 2537804795, 850790212, - 639946566, 2073602691, 2316769983, 3577216077, 3538374748, 61447995, 3718817085, - 1476398788, 3239144530, 3595014456, 454482110, 286330934, 2119173159, 1693518756, - 1464218560, 1201825197, 1112746405, 2988579776, 1626663767, 2236015969, 4018896468, - 1885926862, 671386673, 1583086162, 1114723892, 2936863300, 2620955107, 2628074015, - 426673611, 3370181092, 3462245129, 3590185736, 2630441788, 171626554, 3647663038, - 880996914, 1155913149, 2653278555, 508978149, 2031872445, 3041145171, 1339819022, - 127509725, 1336955078, 727702092, 693349672, 999665905, 2287631318, 961427722, - 3355851447, 821851136, 2370998072, 4027450519, 2054803464, 144596514, 3295312213, - 2579322479, 2982266864, 4275468400, 179988815, 2123828208, 1486957870, 2484928010, - 288096701, 1211834301, 1819157080, 3569000238, 4164201803, 3042117433, 2741571248, - 3688451311, 29376415, 437788821, 994675658, 1014591996, 296335443, 363551454, - 2628890394, 332401256, 2288239762, 3766239385, 317162173, 3721125104, 2296650899, - 56428392, 3900411067, 2303724992, 3735005983, 1377320198, 612032498, 1171213235, - 2494454628, 1894368149, 4124860986, 1694123597, 2306091209, 2075278956, 3898366152, - 937522278, 32800830, 324902076, 2365753207, 2251160429, 1692543836, 2920424644, - 119047416, 1821685115, 1486296407, 3055580738, 3711421730, 1522703457, 1422399573, - 2515073038, 3788816887, 3490575947, 2395299159, 4248373284, 3383561277, 3128107243, - 2344292809, 1806504793, 3087395022, 4113720664, 2749262038, 395148869, 1331347439, - 2682558741, 1253966763, 4204248490, 2083995727, 2717069903, 4144872894, 1857751053, - 2166276701, 1419950839, 1145170701, 3225260742, 211743500, 2746391743, 3333387219, - 4115426799, 3801457092, 3327614811, 1460971336, 2256342146, 3186427137, 2684216499, - 1035644397, 2948948308, - }, - { - 216975964, 4145824263, 2147471723, 4154469597, 161744273, 1299764439, 3468614543, - 4190517158, 4124232403, 754999274, 208153182, 1442063188, 2940825403, 729331312, - 2124186505, 1136411459, 1083787490, 442264548, 442338468, 464589685, 3509461223, - 4241734851, 370778328, 4003105058, 3163637982, 3747133182, 1433548371, 1876378240, - 536564977, 1171222160, 3268902719, 2725776746, 1547771137, 2818791461, 4129042013, - 2677094853, 1594765197, 1556725424, 569252594, 2640731848, 2947042710, 2633188904, - 1640957370, 1806863786, 2803403654, 2632220187, 2740076538, 383549855, 3211856699, - 3933793958, 1988232112, 404006876, 1369488120, 1963339964, 609604643, 2488118016, - 3936354252, 1980115609, 189069630, 860670414, 85775513, 2447581620, 886385122, - 3047212472, 2470718978, 4044569663, 430717074, 1598133481, 1905362808, 2981511487, - 1842297666, 2992320857, 1682119455, 1753461544, 700013801, 3025873251, 3413358770, - 1673092091, 113651375, 2618875026, 1479752146, 81598739, 1530418117, 962911586, - 778994423, 3944331100, 4116504755, 480712357, 1078821437, 1091665476, 3696871260, - 2074607518, 3226459752, 3767432525, 768289441, 3142741843, 2969151790, 1814889320, - 2122849610, 451935137, 2784993892, 1836517944, 1565951586, 1663606442, 1578543925, - 33407321, 1445768530, 2156093253, 3164165477, 3093293932, 298945371, 2558835427, - 1386275152, 2649603495, 893605644, 1147537351, 1889670166, 3203610476, 2598985714, - 966335150, 3384227644, 2584671737, 552770393, 2430097209, 3085150053, 3633667948, - 1319147485, 1999467843, 3676133150, 2314789604, 1443760911, 1552954684, 2411684219, - 3708965016, 2607719926, 484007519, 491681421, 2498192461, 6342020, 4226570819, - 2329860147, 1097040622, 1270325434, 2572535504, 918592905, 193599782, 4223250613, - 1640082589, 1817957216, 2937344769, 3768793871, 2982566292, 1607453458, 4096207317, - 696907828, 2431936270, 627206620, 3267100287, 1161821973, 2322099303, 1700245615, - 216975964, 4145824263, 2147471723, 4154469597, 2836499116, 1299764439, 3468614543, - 4190517158, 4124232403, 2176257299, 208153182, 1442063188, 2940825403, 729331312, - 2954254860, 1136411459, 1083787490, 442264548, 442338468, 3098695824, 3509461223, - 4241734851, 370778328, 4003105058, 2963948505, 3747133182, 1433548371, 1876378240, - 536564977, 1565224991, 3268902719, 2725776746, 1547771137, 2818791461, 2118790546, - 2677094853, 1594765197, 1556725424, 569252594, 610771792, 2947042710, 2633188904, - 1640957370, 1806863786, 2121022793, 2632220187, 2740076538, 383549855, 3211856699, - 14274926, 1988232112, 404006876, 1369488120, 1963339964, 1661081792, 2488118016, - 3936354252, 1980115609, 189069630, 595192504, 85775513, 2447581620, 886385122, - 3047212472, 1596069326, 4044569663, 430717074, 1598133481, 1905362808, 2670961612, - 1842297666, 2992320857, 1682119455, 1753461544, 1121764918, 3025873251, 3413358770, - 1673092091, 113651375, 1721474883, 1479752146, 81598739, 1530418117, 962911586, - 3478535046, 3944331100, 4116504755, 480712357, 1078821437, 1456786415, 3696871260, - 2074607518, 3226459752, 3767432525, 2947648865, 3142741843, 2969151790, 1814889320, - 2122849610, 3367879697, 2784993892, 1836517944, 1565951586, 1663606442, 2621366329, - 33407321, 1445768530, 2156093253, 3164165477, 619266142, 298945371, 2558835427, - 1386275152, 2649603495, 97967685, 1147537351, 1889670166, 3203610476, 2598985714, - 504495866, 3384227644, 2584671737, 552770393, 2430097209, 2168477293, 3633667948, - 1319147485, 1999467843, 3676133150, 2755203144, 1443760911, 1552954684, 2411684219, - 3708965016, 2301846628, 484007519, 491681421, 2498192461, 6342020, 318325395, - 2329860147, 1097040622, 1270325434, 2572535504, 3458698828, 193599782, 4223250613, - 1640082589, 1817957216, 1861636211, 3768793871, 2982566292, 1607453458, 4096207317, - 1871072589, 2431936270, 627206620, 3267100287, 1161821973, 3904037207, 1700245615, - 216975964, 4145824263, 2147471723, 2789343849, 2836499116, 1299764439, 3468614543, - 4190517158, 639361502, 2176257299, 208153182, 1442063188, 2940825403, 2962998954, - 2954254860, 1136411459, 1083787490, 442264548, 1812626669, 3098695824, 3509461223, - 4241734851, 370778328, 1673951193, 2963948505, 3747133182, 1433548371, 1876378240, - 3651623536, 1565224991, 3268902719, 2725776746, 1547771137, 1938402968, 2118790546, - 2677094853, 1594765197, 1556725424, 3267956202, 610771792, 2947042710, 2633188904, - 1640957370, 1448040688, 2121022793, 2632220187, 2740076538, 383549855, 300148175, - 14274926, 1988232112, 404006876, 1369488120, 3313508750, 1661081792, 2488118016, - 3936354252, 1980115609, 2631341293, 595192504, 85775513, 2447581620, 886385122, - 2465820467, 1596069326, 4044569663, 430717074, 1598133481, 4191772516, 2670961612, - 1842297666, 2992320857, 1682119455, 997741285, 1121764918, 3025873251, 3413358770, - 1673092091, 1493832846, 1721474883, 1479752146, 81598739, 1530418117, 2762019274, - 3478535046, 3944331100, 4116504755, 480712357, 448437372, 1456786415, 3696871260, - 2074607518, 3226459752, 2507199309, 2947648865, 3142741843, 2969151790, 1814889320, - 2268952501, 3367879697, 2784993892, 1836517944, 1565951586, 377207573, 2621366329, - 33407321, 1445768530, 2156093253, 1325490318, 619266142, 298945371, 2558835427, - 1386275152, 2662699426, 97967685, 1147537351, 1889670166, 3203610476, 1999783658, - 504495866, 3384227644, 2584671737, 552770393, 1562106652, 2168477293, 3633667948, - 1319147485, 1999467843, 2037219988, 2755203144, 1443760911, 1552954684, 2411684219, - 1579607443, 2301846628, 484007519, 491681421, 2498192461, 745333677, 318325395, - 2329860147, 1097040622, 1270325434, 208017379, 3458698828, 193599782, 4223250613, - 1640082589, 4049245262, 1861636211, 3768793871, 2982566292, 1607453458, 2058912455, - 1871072589, 2431936270, 627206620, 3267100287, 1186015034, 3904037207, 1700245615, - 216975964, 4145824263, 2422827462, 2789343849, 2836499116, 1299764439, 3468614543, - 2084839633, 639361502, 2176257299, 208153182, 1442063188, 4065931048, 2962998954, - 2954254860, 1136411459, 1083787490, 465529524, 1812626669, 3098695824, 3509461223, - 4241734851, 3818602366, 1673951193, 2963948505, 3747133182, 1433548371, 2475307467, - 3651623536, 1565224991, 3268902719, 2725776746, 2374226870, 1938402968, 2118790546, - 2677094853, 1594765197, 348828658, 3267956202, 610771792, 2947042710, 2633188904, - 1713124265, 1448040688, 2121022793, 2632220187, 2740076538, 1400362266, 300148175, - 14274926, 1988232112, 404006876, 3662575932, 3313508750, 1661081792, 2488118016, - 3936354252, 3100635752, 2631341293, 595192504, 85775513, 2447581620, 2417839883, - 2465820467, 1596069326, 4044569663, 430717074, 1093503127, 4191772516, 2670961612, - 1842297666, 2992320857, 3292586028, 997741285, 1121764918, 3025873251, 3413358770, - 222522839, 1493832846, 1721474883, 1479752146, 81598739, 3132900738, 2762019274, - 3478535046, 3944331100, 4116504755, 3429405501, 448437372, 1456786415, 3696871260, - 2074607518, 1492852861, 2507199309, 2947648865, 3142741843, 2969151790, 2186889362, - 2268952501, 3367879697, 2784993892, 1836517944, 3169157745, 377207573, 2621366329, - 33407321, 1445768530, 4266168148, 1325490318, 619266142, 298945371, 2558835427, - 1447045944, 2662699426, 97967685, 1147537351, 1889670166, 3354555370, 1999783658, - 504495866, 3384227644, 2584671737, 2489662408, 1562106652, 2168477293, 3633667948, - 1319147485, 3353555249, 2037219988, 2755203144, 1443760911, 1552954684, 4137514176, - 1579607443, 2301846628, 484007519, 491681421, 164627749, 745333677, 318325395, - 2329860147, 1097040622, 3587444362, 208017379, 3458698828, 193599782, 4223250613, - 1102471426, 4049245262, 1861636211, 3768793871, 2982566292, 1941698603, 2058912455, - 1871072589, 2431936270, 627206620, 2511999766, 1406946444, 1571284360, 1416792763, - 1774410400, 1655066897, 740531687, 2852637013, 1574342442, 3931672444, 2887289502, - 3588598337, 1607795590, 1893126336, 4113959952, 250670029, 89330705, 2198706475, - 133060312, 4033807246, 2161604768, 3871950931, 1820516188, 828316231, 3126087794, - 3740050736, 543577819, 1589693651, 4210480257, 3844498352, 96010254, 2888517657, - 2931088981, 2307454606, 2411141663, 4193964276, 918899600, 888509951, 3023902229, - 1371276096, 2107726407, 3863079906, 3849297291, 1512401618, 3098628219, 487705749, - 492891601, 345791371, 3230138831, 1022138839, 974682588, 3677932604, 2054641860, - 3303576494, 1416653965, 1119635666, 1907978487, 4269977208, 2047880336, 205698774, - 2401894999, 3253173123, 2603439113, 1295808319, 2965198050, 1718424301, 1605627562, - 2860017421, 1619060227, 1130717786, 2992070906, 2964091191, 3192265220, 3860528275, - 45139953, 3914023193, 1253834497, 3885013544, 3793695046, 3632364934, 4127361980, - 3323804519, 4117285262, 4171102020, 1863837906, 2848174924, 1731389076, 2514130112, - 3539384422, 2950752200, 1138137434, 4147328087, 3345958235, 2305097760, 974161669, - 1739611700, 2522036172, 1196649816, 2395301283, 911135539, 1029496076, 1786766951, - 1802412425, 3485017668, 2837835718, 1951207514, 1447650206, 2289702688, 2517625074, - 2408021138, 2022522416, 719777136, 417238676, 1865171065, 801820378, 2836941189, - 1148226009, 1713866138, 64608707, 1458585813, 3581572089, 2251042907, 1818903516, - 3278072806, 2838874249, 2665607605, 3360214955, 2185961451, 410342713, 364484774, - 2887998484, 2100888426, 1394314931, 1362560504, 3487221127, 3140021092, 3168968161, - 1613267484, 290430673, 2588210538, 2493788232, 2641638765, 2971195072, 2749469779, - 692014176, 3268150742, 387451740, 461249727, 3579417331, 3738405845, 385445455, - 1464799053, 2786433795, 3370371952, 675344511, 4049011269, 2196568686, 4166285481, - 2547135972, 119952106, - }, - { - 2307933966, 145940188, 4247815717, 2995341855, 3245382498, 1213200792, 232910392, - 2718014238, 918321585, 3583102265, 3176078796, 937696513, 266558688, 1520650260, - 3655025189, 1653323191, 538426778, 491545855, 4064663509, 2788350755, 3941259490, - 3471552693, 1851151228, 3279627338, 845228710, 29883500, 1503432309, 593880175, - 2488716480, 828058076, 3287933183, 3510981973, 3970051135, 3803049980, 898259836, - 2890959433, 234437380, 201835406, 1523016285, 2419577439, 2943482079, 4219300984, - 1490698759, 533411805, 1644926459, 4097374623, 265292490, 2694560848, 285667083, - 1563945375, 3128365011, 95277844, 2938824634, 2717708621, 3374928056, 3672802273, - 3445399260, 2422205637, 1106030557, 1269805720, 1781057614, 3491203689, 2454028630, - 2158698380, 2578431870, 3540412661, 2206372988, 3138025266, 474100503, 3310048546, - 126856999, 3144057206, 917199551, 3549528813, 343855771, 391118877, 1900257963, - 1616289477, 3663959751, 1887891784, 697303016, 1346369879, 3634838543, 909311683, - 3534738830, 2676838865, 3020679234, 1248902118, 1517698896, 414632197, 199589058, - 2922557451, 3915079510, 1309075563, 3836275459, 2549095941, 1643088840, 1153547003, - 2254144060, 3585420425, 915059870, 2410951596, 1876156254, 2384812180, 3893647829, - 4119002503, 1535078752, 3888310943, 1483731374, 3915992153, 3662664617, 1065246672, - 2307959656, 1845927873, 2075990232, 1346396900, 4218283385, 3427468026, 1518645158, - 3092538772, 3383570452, 1317710387, 3390054918, 4222595168, 2468387909, 3864538174, - 2442851586, 1858344050, 1537617445, 1090881039, 2055021834, 4011332463, 2797336692, - 280272261, 3350338577, 1682666744, 1256176165, 2017003515, 3666229067, 4288064377, - 3407437449, 2957152445, 3557139753, 4106922773, 2612653316, 3491950269, 1107293753, - 2926461368, 1433860998, 1975669351, 1680462513, 4283282673, 168788571, 57021447, - 3888396304, 2218068386, 2170981202, 1587568797, 2097820654, 1308061343, 4096726326, - 2307933966, 145940188, 4247815717, 2995341855, 2894586378, 1213200792, 232910392, - 2718014238, 918321585, 520434726, 3176078796, 937696513, 266558688, 1520650260, - 645408471, 1653323191, 538426778, 491545855, 4064663509, 2605358672, 3941259490, - 3471552693, 1851151228, 3279627338, 1290188176, 29883500, 1503432309, 593880175, - 2488716480, 1172244224, 3287933183, 3510981973, 3970051135, 3803049980, 3836242189, - 2890959433, 234437380, 201835406, 1523016285, 1720566850, 2943482079, 4219300984, - 1490698759, 533411805, 982587365, 4097374623, 265292490, 2694560848, 285667083, - 3905392425, 3128365011, 95277844, 2938824634, 2717708621, 262111126, 3672802273, - 3445399260, 2422205637, 1106030557, 233401560, 1781057614, 3491203689, 2454028630, - 2158698380, 3314008662, 3540412661, 2206372988, 3138025266, 474100503, 1150191741, - 126856999, 3144057206, 917199551, 3549528813, 84516590, 391118877, 1900257963, - 1616289477, 3663959751, 2831036790, 697303016, 1346369879, 3634838543, 909311683, - 2206291004, 2676838865, 3020679234, 1248902118, 1517698896, 882506847, 199589058, - 2922557451, 3915079510, 1309075563, 3675129276, 2549095941, 1643088840, 1153547003, - 2254144060, 1702669516, 915059870, 2410951596, 1876156254, 2384812180, 393602062, - 4119002503, 1535078752, 3888310943, 1483731374, 1135074988, 3662664617, 1065246672, - 2307959656, 1845927873, 883002610, 1346396900, 4218283385, 3427468026, 1518645158, - 1478839081, 3383570452, 1317710387, 3390054918, 4222595168, 3009846855, 3864538174, - 2442851586, 1858344050, 1537617445, 2419526192, 2055021834, 4011332463, 2797336692, - 280272261, 2937342669, 1682666744, 1256176165, 2017003515, 3666229067, 3563024742, - 3407437449, 2957152445, 3557139753, 4106922773, 610182860, 3491950269, 1107293753, - 2926461368, 1433860998, 2493047579, 1680462513, 4283282673, 168788571, 57021447, - 2151356582, 2218068386, 2170981202, 1587568797, 2097820654, 2738927570, 4096726326, - 2307933966, 145940188, 4247815717, 1887236689, 2894586378, 1213200792, 232910392, - 2718014238, 2028538736, 520434726, 3176078796, 937696513, 266558688, 305624632, - 645408471, 1653323191, 538426778, 491545855, 4188864445, 2605358672, 3941259490, - 3471552693, 1851151228, 1720039364, 1290188176, 29883500, 1503432309, 593880175, - 2595662526, 1172244224, 3287933183, 3510981973, 3970051135, 2763703998, 3836242189, - 2890959433, 234437380, 201835406, 2652280530, 1720566850, 2943482079, 4219300984, - 1490698759, 1968049758, 982587365, 4097374623, 265292490, 2694560848, 1165326939, - 3905392425, 3128365011, 95277844, 2938824634, 2521869983, 262111126, 3672802273, - 3445399260, 2422205637, 395183943, 233401560, 1781057614, 3491203689, 2454028630, - 249721174, 3314008662, 3540412661, 2206372988, 3138025266, 1644439373, 1150191741, - 126856999, 3144057206, 917199551, 1997133400, 84516590, 391118877, 1900257963, - 1616289477, 3843764922, 2831036790, 697303016, 1346369879, 3634838543, 1901125181, - 2206291004, 2676838865, 3020679234, 1248902118, 344347894, 882506847, 199589058, - 2922557451, 3915079510, 2919277604, 3675129276, 2549095941, 1643088840, 1153547003, - 3305575634, 1702669516, 915059870, 2410951596, 1876156254, 1416053196, 393602062, - 4119002503, 1535078752, 3888310943, 3993632377, 1135074988, 3662664617, 1065246672, - 2307959656, 1044670394, 883002610, 1346396900, 4218283385, 3427468026, 1792832168, - 1478839081, 3383570452, 1317710387, 3390054918, 1596709924, 3009846855, 3864538174, - 2442851586, 1858344050, 2428482265, 2419526192, 2055021834, 4011332463, 2797336692, - 424213503, 2937342669, 1682666744, 1256176165, 2017003515, 717473071, 3563024742, - 3407437449, 2957152445, 3557139753, 3319575432, 610182860, 3491950269, 1107293753, - 2926461368, 3052637648, 2493047579, 1680462513, 4283282673, 168788571, 1401253163, - 2151356582, 2218068386, 2170981202, 1587568797, 3994937670, 2738927570, 4096726326, - 2307933966, 145940188, 3928146647, 1887236689, 2894586378, 1213200792, 232910392, - 833120806, 2028538736, 520434726, 3176078796, 937696513, 3704968451, 305624632, - 645408471, 1653323191, 538426778, 939335571, 4188864445, 2605358672, 3941259490, - 3471552693, 2168499975, 1720039364, 1290188176, 29883500, 1503432309, 524387655, - 2595662526, 1172244224, 3287933183, 3510981973, 2444664749, 2763703998, 3836242189, - 2890959433, 234437380, 3272987579, 2652280530, 1720566850, 2943482079, 4219300984, - 1045589319, 1968049758, 982587365, 4097374623, 265292490, 1077412791, 1165326939, - 3905392425, 3128365011, 95277844, 2896038035, 2521869983, 262111126, 3672802273, - 3445399260, 4273256145, 395183943, 233401560, 1781057614, 3491203689, 8343453, - 249721174, 3314008662, 3540412661, 2206372988, 3738630867, 1644439373, 1150191741, - 126856999, 3144057206, 65169501, 1997133400, 84516590, 391118877, 1900257963, - 2914085557, 3843764922, 2831036790, 697303016, 1346369879, 2007568079, 1901125181, - 2206291004, 2676838865, 3020679234, 2097032931, 344347894, 882506847, 199589058, - 2922557451, 3740400148, 2919277604, 3675129276, 2549095941, 1643088840, 199560818, - 3305575634, 1702669516, 915059870, 2410951596, 117939268, 1416053196, 393602062, - 4119002503, 1535078752, 4281599711, 3993632377, 1135074988, 3662664617, 1065246672, - 2854253374, 1044670394, 883002610, 1346396900, 4218283385, 803910659, 1792832168, - 1478839081, 3383570452, 1317710387, 1311168874, 1596709924, 3009846855, 3864538174, - 2442851586, 1967982878, 2428482265, 2419526192, 2055021834, 4011332463, 2725198749, - 424213503, 2937342669, 1682666744, 1256176165, 713350501, 717473071, 3563024742, - 3407437449, 2957152445, 2363682828, 3319575432, 610182860, 3491950269, 1107293753, - 3429638328, 3052637648, 2493047579, 1680462513, 4283282673, 2672311163, 1401253163, - 2151356582, 2218068386, 2170981202, 431601500, 4193143261, 2985267149, 1556712183, - 4135181832, 285960576, 81711096, 57066962, 2646151573, 3692824605, 485132216, - 2799654118, 903527523, 1210637484, 3195346614, 599540837, 1410108963, 3723542120, - 1350764011, 1717225239, 239736775, 3946934722, 420024332, 589304817, 1331122625, - 4294403247, 2009397371, 844641869, 166387728, 4093361096, 2342369656, 3958170613, - 1660376297, 1259528150, 4240809115, 2875563845, 2613790323, 2869665108, 1414690635, - 944649070, 3539368342, 199532147, 2707660205, 2258475730, 771169023, 158544851, - 588872178, 2002019277, 4225148852, 641266809, 2133909450, 330112418, 1815776319, - 1949213618, 3868452239, 2702722715, 2491030937, 468812562, 3226259052, 199165016, - 436679774, 881956108, 1098105661, 68909298, 248572829, 339224422, 553849953, - 3054752668, 701934162, 1898925107, 749060575, 987950022, 4040401060, 684345838, - 3449205676, 2583450513, 433795092, 3559011048, 293161429, 3947766299, 3491895171, - 1651265910, 1216468759, 1625512737, 412235874, 893680794, 2582820523, 1514322840, - 2348781204, 2720801933, 3364999370, 2822073391, 2627166519, 3805500773, 177760590, - 2210728920, 3136345252, 3226658259, 3982978003, 86264452, 536816704, 3489051867, - 2161950016, 1375640747, 4116957650, 3676292350, 3001078542, 1379688752, 3059678152, - 3740664918, 475697670, 539253230, 1256048653, 3819847913, 141216227, 3888391528, - 3567424851, 4131097532, 2142453586, 3606575354, 3689715433, 2318212425, 3026095399, - 2451038695, 4052322172, 1861782452, 3032216562, 4078403318, 2636775961, 2188864067, - 3276459319, 2230349722, 3939784264, 831216291, 2483460713, 2571551493, 484276565, - 3173595164, 4177831244, 4132249231, 2116763555, 1420812998, 2121017321, 2855491215, - 1630144518, 2489688364, 411521312, 3713786536, 4177871972, 690465497, 855092147, - 4271606539, 1265108699, 3757106624, 3151574897, 670335437, 3099376310, 3946436509, - 1795346235, 4013409945, - }, - { - 650684252, 2220445579, 537394374, 571322423, 2781663439, 899394682, 364129622, - 328438826, 1219862153, 830435885, 3278649457, 3072225531, 2838645991, 3150905380, - 1251952499, 1751415553, 2034088483, 1437197870, 1907624878, 1786974150, 4207811086, - 768131803, 2713210999, 4004509777, 3510764535, 2740991637, 3000313526, 1355959320, - 938244439, 4093313692, 2476002145, 835527260, 2084758949, 4223775017, 91645393, - 2251723899, 3159477758, 2008655575, 912220875, 1525327655, 2067948386, 2006141522, - 450235614, 3945671083, 2852189452, 3804118704, 3302604345, 1712745267, 349281154, - 19331179, 3423301791, 416995358, 2049170698, 684574142, 3271042138, 3438668017, - 1645378852, 1995123150, 1835887948, 2347182898, 3828432892, 3710259931, 713144773, - 3246285450, 2196135622, 1611287338, 2845388948, 3690657633, 2403178686, 2946296994, - 2180908599, 3072014497, 3436535724, 2948908116, 3080353236, 1669938872, 3572731079, - 1100892983, 308060688, 3092946261, 2725115972, 887278263, 991869336, 3597899723, - 3454505181, 1108269267, 851855066, 1940998002, 3539084542, 3102161424, 965450940, - 1942363226, 1430246588, 1368971075, 4251556311, 642683738, 3035789355, 1829444044, - 4234626091, 671403403, 2809844786, 2251172733, 970188857, 3910072565, 1131847479, - 3397535176, 3290884849, 861868157, 2811422184, 3280310458, 3502085520, 1499698865, - 2446269873, 236680785, 1896103604, 1179896471, 83960622, 3303129336, 1191373247, - 177898275, 3077388457, 1022975703, 2535144448, 8680269, 3602435630, 1810825915, - 2293529378, 2307085218, 483894148, 2872435038, 2043868156, 3038491874, 3786518530, - 3606440668, 3336713377, 120183042, 86901386, 2233164457, 2881782972, 3135264768, - 2294460421, 2996668315, 658184098, 3558825846, 2386173040, 1950463910, 551627788, - 2464303444, 893474565, 3277869222, 2852725906, 1191310725, 2398932683, 4164956002, - 1689291769, 2619288187, 3429362702, 3205668166, 1668126623, 955771270, 2106753333, - 650684252, 2220445579, 537394374, 571322423, 2369694095, 899394682, 364129622, - 328438826, 1219862153, 4195985755, 3278649457, 3072225531, 2838645991, 3150905380, - 2389919, 1751415553, 2034088483, 1437197870, 1907624878, 1516966376, 4207811086, - 768131803, 2713210999, 4004509777, 1955929377, 2740991637, 3000313526, 1355959320, - 938244439, 4263287583, 2476002145, 835527260, 2084758949, 4223775017, 110659216, - 2251723899, 3159477758, 2008655575, 912220875, 2378803214, 2067948386, 2006141522, - 450235614, 3945671083, 4112321452, 3804118704, 3302604345, 1712745267, 349281154, - 3834044005, 3423301791, 416995358, 2049170698, 684574142, 3651360887, 3438668017, - 1645378852, 1995123150, 1835887948, 1022257616, 3828432892, 3710259931, 713144773, - 3246285450, 2485142597, 1611287338, 2845388948, 3690657633, 2403178686, 2201888000, - 2180908599, 3072014497, 3436535724, 2948908116, 1647734358, 1669938872, 3572731079, - 1100892983, 308060688, 592016509, 2725115972, 887278263, 991869336, 3597899723, - 819708104, 1108269267, 851855066, 1940998002, 3539084542, 3156419045, 965450940, - 1942363226, 1430246588, 1368971075, 224112021, 642683738, 3035789355, 1829444044, - 4234626091, 314715303, 2809844786, 2251172733, 970188857, 3910072565, 155628632, - 3397535176, 3290884849, 861868157, 2811422184, 1847583676, 3502085520, 1499698865, - 2446269873, 236680785, 3698448762, 1179896471, 83960622, 3303129336, 1191373247, - 1567908030, 3077388457, 1022975703, 2535144448, 8680269, 3979982957, 1810825915, - 2293529378, 2307085218, 483894148, 4003402870, 2043868156, 3038491874, 3786518530, - 3606440668, 3062185402, 120183042, 86901386, 2233164457, 2881782972, 3345668738, - 2294460421, 2996668315, 658184098, 3558825846, 2121278529, 1950463910, 551627788, - 2464303444, 893474565, 183176481, 2852725906, 1191310725, 2398932683, 4164956002, - 788617081, 2619288187, 3429362702, 3205668166, 1668126623, 29124108, 2106753333, - 650684252, 2220445579, 537394374, 725338795, 2369694095, 899394682, 364129622, - 328438826, 1727397396, 4195985755, 3278649457, 3072225531, 2838645991, 583924693, - 2389919, 1751415553, 2034088483, 1437197870, 1017611325, 1516966376, 4207811086, - 768131803, 2713210999, 761144580, 1955929377, 2740991637, 3000313526, 1355959320, - 840696976, 4263287583, 2476002145, 835527260, 2084758949, 3729075247, 110659216, - 2251723899, 3159477758, 2008655575, 4127907945, 2378803214, 2067948386, 2006141522, - 450235614, 3240776806, 4112321452, 3804118704, 3302604345, 1712745267, 1079549936, - 3834044005, 3423301791, 416995358, 2049170698, 3913510119, 3651360887, 3438668017, - 1645378852, 1995123150, 841590980, 1022257616, 3828432892, 3710259931, 713144773, - 1272133892, 2485142597, 1611287338, 2845388948, 3690657633, 3083851146, 2201888000, - 2180908599, 3072014497, 3436535724, 4162521870, 1647734358, 1669938872, 3572731079, - 1100892983, 986584939, 592016509, 2725115972, 887278263, 991869336, 2711883653, - 819708104, 1108269267, 851855066, 1940998002, 4050477073, 3156419045, 965450940, - 1942363226, 1430246588, 4285490865, 224112021, 642683738, 3035789355, 1829444044, - 4197159994, 314715303, 2809844786, 2251172733, 970188857, 3018833494, 155628632, - 3397535176, 3290884849, 861868157, 2883971818, 1847583676, 3502085520, 1499698865, - 2446269873, 2621709156, 3698448762, 1179896471, 83960622, 3303129336, 2192966710, - 1567908030, 3077388457, 1022975703, 2535144448, 95661399, 3979982957, 1810825915, - 2293529378, 2307085218, 485952375, 4003402870, 2043868156, 3038491874, 3786518530, - 575288835, 3062185402, 120183042, 86901386, 2233164457, 2864966512, 3345668738, - 2294460421, 2996668315, 658184098, 2892259673, 2121278529, 1950463910, 551627788, - 2464303444, 2699734841, 183176481, 2852725906, 1191310725, 2398932683, 3505505465, - 788617081, 2619288187, 3429362702, 3205668166, 2157859363, 29124108, 2106753333, - 650684252, 2220445579, 978263237, 725338795, 2369694095, 899394682, 364129622, - 3795063930, 1727397396, 4195985755, 3278649457, 3072225531, 1996768476, 583924693, - 2389919, 1751415553, 2034088483, 1069211024, 1017611325, 1516966376, 4207811086, - 768131803, 1365857736, 761144580, 1955929377, 2740991637, 3000313526, 1057560595, - 840696976, 4263287583, 2476002145, 835527260, 76517292, 3729075247, 110659216, - 2251723899, 3159477758, 3272987770, 4127907945, 2378803214, 2067948386, 2006141522, - 1223694226, 3240776806, 4112321452, 3804118704, 3302604345, 2218568154, 1079549936, - 3834044005, 3423301791, 416995358, 3661322119, 3913510119, 3651360887, 3438668017, - 1645378852, 3606917602, 841590980, 1022257616, 3828432892, 3710259931, 1270853142, - 1272133892, 2485142597, 1611287338, 2845388948, 131877212, 3083851146, 2201888000, - 2180908599, 3072014497, 2459348479, 4162521870, 1647734358, 1669938872, 3572731079, - 4285199726, 986584939, 592016509, 2725115972, 887278263, 3824306591, 2711883653, - 819708104, 1108269267, 851855066, 190839383, 4050477073, 3156419045, 965450940, - 1942363226, 1750931697, 4285490865, 224112021, 642683738, 3035789355, 1544088048, - 4197159994, 314715303, 2809844786, 2251172733, 3155072709, 3018833494, 155628632, - 3397535176, 3290884849, 4153861738, 2883971818, 1847583676, 3502085520, 1499698865, - 1780983485, 2621709156, 3698448762, 1179896471, 83960622, 3849402190, 2192966710, - 1567908030, 3077388457, 1022975703, 1639944917, 95661399, 3979982957, 1810825915, - 2293529378, 3477014442, 485952375, 4003402870, 2043868156, 3038491874, 1482314580, - 575288835, 3062185402, 120183042, 86901386, 3129494022, 2864966512, 3345668738, - 2294460421, 2996668315, 1986664970, 2892259673, 2121278529, 1950463910, 551627788, - 3105369079, 2699734841, 183176481, 2852725906, 1191310725, 3154591925, 3505505465, - 788617081, 2619288187, 3429362702, 4204415531, 1321048315, 4247243973, 3085535935, - 114618345, 2126710176, 1857709117, 3744103666, 304437872, 2388303947, 1802971382, - 2099900439, 2543837819, 593111133, 3788847386, 1479546758, 4095492150, 240996968, - 3423191009, 2666077260, 884572403, 2988847666, 928827215, 2549465610, 2773670136, - 708214104, 2594951780, 1076989709, 2850313793, 1401578686, 4100639899, 2353261688, - 1323066237, 31664438, 951240198, 3676836716, 3633113483, 3262159382, 981784748, - 1172850762, 3106238289, 3118297408, 4207023277, 3362324732, 844983306, 3790928628, - 4156848237, 2638267501, 1494090858, 3955182404, 1193294064, 4035152789, 2971914580, - 2865046609, 3782329083, 120288587, 3300482994, 4268540970, 4183426205, 3572724103, - 3287140971, 3038086532, 3210919007, 2171998100, 3958495101, 1589679371, 2880366694, - 827575211, 1343189406, 364332706, 866065087, 33080625, 4284492640, 2277479989, - 4110331130, 430538110, 3549886335, 3734345920, 3780943339, 638033279, 2684714509, - 945721631, 49994267, 2394351381, 1996532760, 3201422203, 3509459657, 4118609520, - 632454166, 696027759, 901486290, 1230453723, 4225865813, 4072619256, 3111686961, - 1487480830, 4112016561, 1577020285, 2765241900, 2496609620, 1731271292, 6970479, - 2936359283, 1541124937, 3705956773, 2349695021, 2247551804, 3759489710, 1321217706, - 379586757, 2008242014, 1138475935, 3044902216, 1917596533, 2905651936, 3320601534, - 1468557693, 4101437636, 374575138, 730079080, 995340259, 1430552870, 3860649629, - 541396702, 3413070856, 3052797396, 3591116740, 2811484252, 2464310183, 1597327051, - 3288232619, 1564716093, 2838386049, 264313861, 881377066, 4165178494, 1069189853, - 1045737884, 2072266205, 2700673629, 2338724235, 837702541, 2603464957, 1548182143, - 3565539962, 38172869, 1949065935, 3628598166, 2788698071, 3531182193, 1367529788, - 3902468811, 1215323634, 1117475027, 3901912129, 2678279671, 597953858, 4082485755, - 3696533122, 1078703353, - }, - { - 590004384, 3025338414, 1764374188, 20686172, 932343559, 1798441768, 1013577341, - 4275903797, 853441141, 1065980978, 3665193407, 1555165047, 2962781443, 1822487181, - 3329200135, 1527094489, 3805115799, 2252376033, 2137546519, 3632426270, 2439842864, - 2525211849, 602876448, 1488163727, 3169015136, 832084039, 81097112, 994974428, - 1945411347, 1020609213, 2863240894, 1639194881, 3078842449, 1885382385, 2595105518, - 3857547190, 3654577058, 3853111480, 2237941224, 625422255, 3292783340, 750206381, - 1002246874, 900879607, 820635221, 3318328110, 3980484559, 3924790669, 4260574943, - 3658381114, 3673068643, 1319175627, 3620071157, 3914274380, 3310864044, 1529070914, - 1760958838, 818806045, 3056976418, 2337737150, 2061530784, 1036243443, 2058675708, - 1932546035, 1604709219, 1317296740, 2505350414, 624826181, 2710208816, 2208469912, - 1930700024, 3769953790, 2092911082, 520309780, 3787727278, 684095804, 3697683979, - 111440289, 4043494885, 1571375993, 1828801775, 3589061974, 3016563679, 2026002784, - 3810490061, 2634997537, 2715287551, 1973545003, 3407971274, 3239387641, 2479429785, - 324785401, 2622755198, 1525605325, 3280412074, 2453630352, 726090704, 4170024046, - 248003549, 3319518538, 1331224401, 1203416669, 3497395173, 2465693133, 15303334, - 267163358, 627307819, 294350450, 3691559013, 2491765952, 839609873, 1598505629, - 3905396753, 583168080, 281403302, 1658629464, 1498139453, 2860737994, 148007837, - 1439496901, 3226624586, 1708925351, 195473107, 1150552649, 2856922985, 1853471286, - 1286593394, 2025932254, 1300583198, 3169702837, 1255226060, 3482666699, 1515557266, - 1964035766, 1604627993, 641427670, 450188959, 1095230428, 293179001, 1293554079, - 3022335608, 610535626, 1329467104, 3717935497, 1252385485, 441595535, 2937045243, - 2846877561, 668719121, 3604154741, 1150714166, 1689640190, 2219487087, 2445975095, - 3492083575, 377195836, 2727989292, 2460040634, 2910322481, 399050881, 3601292788, - 590004384, 3025338414, 1764374188, 20686172, 3576058865, 1798441768, 1013577341, - 4275903797, 853441141, 3862104007, 3665193407, 1555165047, 2962781443, 1822487181, - 1058917817, 1527094489, 3805115799, 2252376033, 2137546519, 780594798, 2439842864, - 2525211849, 602876448, 1488163727, 642430472, 832084039, 81097112, 994974428, - 1945411347, 2231598766, 2863240894, 1639194881, 3078842449, 1885382385, 2387524763, - 3857547190, 3654577058, 3853111480, 2237941224, 991026264, 3292783340, 750206381, - 1002246874, 900879607, 1178067772, 3318328110, 3980484559, 3924790669, 4260574943, - 1964983082, 3673068643, 1319175627, 3620071157, 3914274380, 992141498, 1529070914, - 1760958838, 818806045, 3056976418, 3295305429, 2061530784, 1036243443, 2058675708, - 1932546035, 3724542133, 1317296740, 2505350414, 624826181, 2710208816, 3359715256, - 1930700024, 3769953790, 2092911082, 520309780, 1979908015, 684095804, 3697683979, - 111440289, 4043494885, 3256907235, 1828801775, 3589061974, 3016563679, 2026002784, - 1967781780, 2634997537, 2715287551, 1973545003, 3407971274, 391604110, 2479429785, - 324785401, 2622755198, 1525605325, 462777294, 2453630352, 726090704, 4170024046, - 248003549, 3125444318, 1331224401, 1203416669, 3497395173, 2465693133, 1610778556, - 267163358, 627307819, 294350450, 3691559013, 3302305047, 839609873, 1598505629, - 3905396753, 583168080, 1502262581, 1658629464, 1498139453, 2860737994, 148007837, - 2973368511, 3226624586, 1708925351, 195473107, 1150552649, 522423348, 1853471286, - 1286593394, 2025932254, 1300583198, 555770116, 1255226060, 3482666699, 1515557266, - 1964035766, 877073175, 641427670, 450188959, 1095230428, 293179001, 4216364784, - 3022335608, 610535626, 1329467104, 3717935497, 1665384485, 441595535, 2937045243, - 2846877561, 668719121, 978801343, 1150714166, 1689640190, 2219487087, 2445975095, - 3819595050, 377195836, 2727989292, 2460040634, 2910322481, 1200428010, 3601292788, - 590004384, 3025338414, 1764374188, 3586255253, 3576058865, 1798441768, 1013577341, - 4275903797, 1511067357, 3862104007, 3665193407, 1555165047, 2962781443, 2749766525, - 1058917817, 1527094489, 3805115799, 2252376033, 817362043, 780594798, 2439842864, - 2525211849, 602876448, 2309049006, 642430472, 832084039, 81097112, 994974428, - 3148197354, 2231598766, 2863240894, 1639194881, 3078842449, 311769962, 2387524763, - 3857547190, 3654577058, 3853111480, 1888597091, 991026264, 3292783340, 750206381, - 1002246874, 2904195378, 1178067772, 3318328110, 3980484559, 3924790669, 4265386540, - 1964983082, 3673068643, 1319175627, 3620071157, 1635921454, 992141498, 1529070914, - 1760958838, 818806045, 3002614702, 3295305429, 2061530784, 1036243443, 2058675708, - 2534375036, 3724542133, 1317296740, 2505350414, 624826181, 3042995618, 3359715256, - 1930700024, 3769953790, 2092911082, 1870611696, 1979908015, 684095804, 3697683979, - 111440289, 1111193348, 3256907235, 1828801775, 3589061974, 3016563679, 2203918092, - 1967781780, 2634997537, 2715287551, 1973545003, 17967467, 391604110, 2479429785, - 324785401, 2622755198, 3993572289, 462777294, 2453630352, 726090704, 4170024046, - 813760479, 3125444318, 1331224401, 1203416669, 3497395173, 2528908686, 1610778556, - 267163358, 627307819, 294350450, 4252461657, 3302305047, 839609873, 1598505629, - 3905396753, 3407593947, 1502262581, 1658629464, 1498139453, 2860737994, 1137070983, - 2973368511, 3226624586, 1708925351, 195473107, 1973834367, 522423348, 1853471286, - 1286593394, 2025932254, 1636839834, 555770116, 1255226060, 3482666699, 1515557266, - 4244619305, 877073175, 641427670, 450188959, 1095230428, 710341587, 4216364784, - 3022335608, 610535626, 1329467104, 262034293, 1665384485, 441595535, 2937045243, - 2846877561, 1059914271, 978801343, 1150714166, 1689640190, 2219487087, 258315233, - 3819595050, 377195836, 2727989292, 2460040634, 1828274968, 1200428010, 3601292788, - 590004384, 3025338414, 3487643146, 3586255253, 3576058865, 1798441768, 1013577341, - 3609472816, 1511067357, 3862104007, 3665193407, 1555165047, 4188135767, 2749766525, - 1058917817, 1527094489, 3805115799, 1547526585, 817362043, 780594798, 2439842864, - 2525211849, 3949139098, 2309049006, 642430472, 832084039, 81097112, 2619711743, - 3148197354, 2231598766, 2863240894, 1639194881, 3018692935, 311769962, 2387524763, - 3857547190, 3654577058, 2418052942, 1888597091, 991026264, 3292783340, 750206381, - 2501986418, 2904195378, 1178067772, 3318328110, 3980484559, 655757623, 4265386540, - 1964983082, 3673068643, 1319175627, 1539823819, 1635921454, 992141498, 1529070914, - 1760958838, 1840073710, 3002614702, 3295305429, 2061530784, 1036243443, 2212957003, - 2534375036, 3724542133, 1317296740, 2505350414, 2754670042, 3042995618, 3359715256, - 1930700024, 3769953790, 3307920786, 1870611696, 1979908015, 684095804, 3697683979, - 326641529, 1111193348, 3256907235, 1828801775, 3589061974, 1408835557, 2203918092, - 1967781780, 2634997537, 2715287551, 1958610929, 17967467, 391604110, 2479429785, - 324785401, 3833051255, 3993572289, 462777294, 2453630352, 726090704, 1236380896, - 813760479, 3125444318, 1331224401, 1203416669, 728276857, 2528908686, 1610778556, - 267163358, 627307819, 4276734917, 4252461657, 3302305047, 839609873, 1598505629, - 3827653659, 3407593947, 1502262581, 1658629464, 1498139453, 3636064463, 1137070983, - 2973368511, 3226624586, 1708925351, 2288771247, 1973834367, 522423348, 1853471286, - 1286593394, 798364204, 1636839834, 555770116, 1255226060, 3482666699, 2385578475, - 4244619305, 877073175, 641427670, 450188959, 3502743047, 710341587, 4216364784, - 3022335608, 610535626, 2388448039, 262034293, 1665384485, 441595535, 2937045243, - 3028160550, 1059914271, 978801343, 1150714166, 1689640190, 169488023, 258315233, - 3819595050, 377195836, 2727989292, 837094660, 3531987448, 1901453576, 3312447598, - 1036467641, 2243300650, 3148869460, 1886274644, 4076707689, 257110870, 3118463831, - 1165161057, 1118846497, 3446934363, 1514176098, 1362957326, 2629874126, 791374320, - 1015673947, 4252955786, 2409207780, 3831311130, 1654475922, 3682733431, 780405105, - 4059616372, 503333525, 1471514828, 2526848791, 607539645, 730408454, 1574159005, - 1777808061, 1296178310, 1078855633, 878462103, 269337411, 750735378, 2599590920, - 4206153248, 939121991, 3061289971, 2543431563, 1684736054, 2319658494, 77300347, - 3222569207, 3882064339, 2201120493, 289098227, 3934209124, 2407620042, 2713079957, - 2812644841, 115993752, 2545688211, 774350907, 939749505, 2242588062, 960853876, - 296665594, 1367312411, 3370351589, 711706404, 3331136631, 1370376958, 2322438166, - 577115138, 1472236592, 4029835216, 1122502809, 3490426739, 1930206806, 2074277138, - 1360950220, 3797708387, 2007430804, 2257239461, 3889012648, 710165871, 763101711, - 728019024, 652403220, 2517020147, 1801290767, 1478810019, 1057288808, 2879821959, - 3916870020, 1480362189, 919816752, 375872647, 3236906236, 1504223782, 128306943, - 1355826533, 2656243649, 390454690, 3848250363, 377480950, 358651174, 1337795904, - 1925462532, 2421843219, 173144626, 886649902, 402617827, 932830871, 742712936, - 4033430386, 1409945926, 3617206544, 2383446356, 3452204096, 615486157, 720696019, - 1730134434, 3918468503, 1629431965, 2174079220, 325852294, 234479771, 1490297289, - 3579002992, 3538738636, 139386548, 3067789050, 2078261059, 3552654276, 1774602596, - 2105142163, 2768099869, 2265044995, 3680536732, 3601322356, 2848878442, 1166743022, - 3508176959, 2186695985, 550278868, 3324775634, 384537301, 1019044102, 3354263542, - 1942540686, 922714337, 3097711558, 3074228403, 3565076630, 3459053081, 4128383906, - 1114387332, 2101424539, 1192649508, 58778130, 1651798895, 1752063480, 1728826905, - 2225187635, 2463770127, - }, - { - 1978406995, 576106282, 2238958298, 2073551095, 624788087, 4231569260, 1853272808, - 238274694, 2389334758, 410188028, 2293786099, 4243662908, 2317700970, 4050493361, - 2348206908, 485250660, 1212732903, 169414736, 292623762, 1602229231, 2466348869, - 3063669700, 1872890881, 1887188929, 3447638989, 162521682, 1470651713, 4036975255, - 3423782623, 4043724693, 1686690883, 2610958712, 35940353, 78593759, 1565950713, - 1304303952, 2004267248, 1417268036, 3328228522, 789915977, 2567452041, 3564175714, - 1838409932, 1455795236, 22377452, 455201131, 3340286965, 184599544, 4102076073, - 4007870762, 1470247063, 1579231003, 3544385556, 3408973464, 3759098465, 3243598964, - 532452279, 1172265732, 3520978258, 2880513876, 41188252, 1663974668, 3444236420, - 338981290, 2140558860, 3310465688, 552673362, 3277110106, 948036400, 1346056406, - 3257468427, 4008294878, 3788890535, 2414511414, 3539325895, 3025695322, 3727849930, - 3922840362, 535899902, 665898223, 1456499692, 354208792, 247894771, 2093316680, - 2945209002, 1029298544, 976007759, 394966955, 1843302845, 3689202777, 1999949614, - 1070472810, 4233404701, 667526747, 2313963966, 3519400667, 1548274317, 3272402139, - 2570038689, 892260481, 3547254358, 1540409404, 3687395534, 3751445920, 546406228, - 2167638865, 4234783150, 806401261, 1351195286, 1085913868, 3109267901, 1882610112, - 1568734773, 239430641, 3971361190, 383932711, 149541490, 196701535, 108079452, - 888590964, 1708559652, 3196290573, 2115587458, 3198525248, 3580113911, 3098818120, - 4271558926, 3208851696, 3354604918, 3536923694, 1087345822, 2292802521, 3500230819, - 411564772, 2408049547, 1215342690, 1707182109, 774540619, 1613606757, 836141085, - 1061962136, 348765795, 2852610966, 3526215991, 2708801073, 3467537935, 472234793, - 3944263763, 1782219410, 502724699, 3525703395, 1756411033, 1358811278, 3938603279, - 3701976555, 3259537961, 628617330, 1553932236, 1974037630, 2090519666, 2185028543, - 1978406995, 576106282, 2238958298, 2073551095, 638634424, 4231569260, 1853272808, - 238274694, 2389334758, 3808551433, 2293786099, 4243662908, 2317700970, 4050493361, - 957981276, 485250660, 1212732903, 169414736, 292623762, 1956197178, 2466348869, - 3063669700, 1872890881, 1887188929, 1162224455, 162521682, 1470651713, 4036975255, - 3423782623, 3243414978, 1686690883, 2610958712, 35940353, 78593759, 1648686849, - 1304303952, 2004267248, 1417268036, 3328228522, 3740797237, 2567452041, 3564175714, - 1838409932, 1455795236, 1045087636, 455201131, 3340286965, 184599544, 4102076073, - 2685677331, 1470247063, 1579231003, 3544385556, 3408973464, 3832799869, 3243598964, - 532452279, 1172265732, 3520978258, 531684354, 41188252, 1663974668, 3444236420, - 338981290, 1286622338, 3310465688, 552673362, 3277110106, 948036400, 2987864230, - 3257468427, 4008294878, 3788890535, 2414511414, 2613137548, 3025695322, 3727849930, - 3922840362, 535899902, 3288883992, 1456499692, 354208792, 247894771, 2093316680, - 3775770224, 1029298544, 976007759, 394966955, 1843302845, 1484214934, 1999949614, - 1070472810, 4233404701, 667526747, 3708951530, 3519400667, 1548274317, 3272402139, - 2570038689, 3457725296, 3547254358, 1540409404, 3687395534, 3751445920, 181641144, - 2167638865, 4234783150, 806401261, 1351195286, 3457819598, 3109267901, 1882610112, - 1568734773, 239430641, 4037392309, 383932711, 149541490, 196701535, 108079452, - 1724276622, 1708559652, 3196290573, 2115587458, 3198525248, 3784683125, 3098818120, - 4271558926, 3208851696, 3354604918, 149872004, 1087345822, 2292802521, 3500230819, - 411564772, 4068437023, 1215342690, 1707182109, 774540619, 1613606757, 1062624488, - 1061962136, 348765795, 2852610966, 3526215991, 1518538195, 3467537935, 472234793, - 3944263763, 1782219410, 1835413488, 3525703395, 1756411033, 1358811278, 3938603279, - 1054245423, 3259537961, 628617330, 1553932236, 1974037630, 2030751433, 2185028543, - 1978406995, 576106282, 2238958298, 3877268821, 638634424, 4231569260, 1853272808, - 238274694, 2482404724, 3808551433, 2293786099, 4243662908, 2317700970, 1955227186, - 957981276, 485250660, 1212732903, 169414736, 1333246101, 1956197178, 2466348869, - 3063669700, 1872890881, 3662049503, 1162224455, 162521682, 1470651713, 4036975255, - 3593925064, 3243414978, 1686690883, 2610958712, 35940353, 2530174792, 1648686849, - 1304303952, 2004267248, 1417268036, 1299827381, 3740797237, 2567452041, 3564175714, - 1838409932, 4221368409, 1045087636, 455201131, 3340286965, 184599544, 486448047, - 2685677331, 1470247063, 1579231003, 3544385556, 1404931688, 3832799869, 3243598964, - 532452279, 1172265732, 3373048034, 531684354, 41188252, 1663974668, 3444236420, - 1375188728, 1286622338, 3310465688, 552673362, 3277110106, 655980467, 2987864230, - 3257468427, 4008294878, 3788890535, 763995173, 2613137548, 3025695322, 3727849930, - 3922840362, 1850434657, 3288883992, 1456499692, 354208792, 247894771, 3440471938, - 3775770224, 1029298544, 976007759, 394966955, 3298245949, 1484214934, 1999949614, - 1070472810, 4233404701, 3788558253, 3708951530, 3519400667, 1548274317, 3272402139, - 3117201719, 3457725296, 3547254358, 1540409404, 3687395534, 3871454027, 181641144, - 2167638865, 4234783150, 806401261, 1627904858, 3457819598, 3109267901, 1882610112, - 1568734773, 3178105921, 4037392309, 383932711, 149541490, 196701535, 424324376, - 1724276622, 1708559652, 3196290573, 2115587458, 2946026327, 3784683125, 3098818120, - 4271558926, 3208851696, 2551504859, 149872004, 1087345822, 2292802521, 3500230819, - 3055410013, 4068437023, 1215342690, 1707182109, 774540619, 2466902579, 1062624488, - 1061962136, 348765795, 2852610966, 355211123, 1518538195, 3467537935, 472234793, - 3944263763, 3159176627, 1835413488, 3525703395, 1756411033, 1358811278, 2153206130, - 1054245423, 3259537961, 628617330, 1553932236, 1741202495, 2030751433, 2185028543, - 1978406995, 576106282, 2832311581, 3877268821, 638634424, 4231569260, 1853272808, - 3103974717, 2482404724, 3808551433, 2293786099, 4243662908, 2607780401, 1955227186, - 957981276, 485250660, 1212732903, 3214649174, 1333246101, 1956197178, 2466348869, - 3063669700, 2428387069, 3662049503, 1162224455, 162521682, 1470651713, 3563435961, - 3593925064, 3243414978, 1686690883, 2610958712, 1021669488, 2530174792, 1648686849, - 1304303952, 2004267248, 1150095671, 1299827381, 3740797237, 2567452041, 3564175714, - 1992360540, 4221368409, 1045087636, 455201131, 3340286965, 3795860292, 486448047, - 2685677331, 1470247063, 1579231003, 3012017918, 1404931688, 3832799869, 3243598964, - 532452279, 2740401823, 3373048034, 531684354, 41188252, 1663974668, 1239982773, - 1375188728, 1286622338, 3310465688, 552673362, 2159084435, 655980467, 2987864230, - 3257468427, 4008294878, 1526518186, 763995173, 2613137548, 3025695322, 3727849930, - 4161669345, 1850434657, 3288883992, 1456499692, 354208792, 1648970767, 3440471938, - 3775770224, 1029298544, 976007759, 292829454, 3298245949, 1484214934, 1999949614, - 1070472810, 949984087, 3788558253, 3708951530, 3519400667, 1548274317, 3691975282, - 3117201719, 3457725296, 3547254358, 1540409404, 3414085332, 3871454027, 181641144, - 2167638865, 4234783150, 487427004, 1627904858, 3457819598, 3109267901, 1882610112, - 2942538550, 3178105921, 4037392309, 383932711, 149541490, 528605550, 424324376, - 1724276622, 1708559652, 3196290573, 2042399752, 2946026327, 3784683125, 3098818120, - 4271558926, 2493686919, 2551504859, 149872004, 1087345822, 2292802521, 3257357826, - 3055410013, 4068437023, 1215342690, 1707182109, 1101368233, 2466902579, 1062624488, - 1061962136, 348765795, 377675640, 355211123, 1518538195, 3467537935, 472234793, - 1918362523, 3159176627, 1835413488, 3525703395, 1756411033, 490591069, 2153206130, - 1054245423, 3259537961, 628617330, 2464143505, 3547421156, 4181103091, 1646291356, - 2711273600, 2961799099, 1443009342, 2191618308, 1193143275, 1858488142, 3741304147, - 1479629752, 214641634, 1601114903, 3032545707, 5784133, 1466424840, 2251379876, - 4054080092, 2965144328, 644228426, 1397556958, 422190032, 3059134799, 3779253493, - 1314537880, 867798895, 3819721559, 3588436937, 670021879, 1070365654, 3339455790, - 2963659516, 1662488399, 2336157317, 3427798652, 2782719134, 1317842084, 1576308528, - 1129452059, 3400565954, 84977051, 3689257381, 3289717503, 3535165628, 3982356490, - 173255911, 1929987033, 4221790572, 3473317939, 749060417, 2711561754, 316719217, - 2359410057, 2014271053, 1432982162, 2107582322, 1899811989, 1394115707, 1134266213, - 2334994542, 2475488907, 3238562415, 2410379210, 4147209396, 2446286513, 2194020199, - 3068194593, 797186100, 1299000541, 1870322719, 2944499140, 1045779179, 2735528787, - 3057750264, 2607876894, 1595833743, 3327636115, 3520489322, 3864068029, 3153522810, - 2609437702, 1360208295, 2062444770, 3927110355, 1524755299, 1708215998, 3587488663, - 2813888113, 686192293, 1078633032, 3066910876, 793688350, 3613674912, 387713910, - 2660476731, 3032509241, 2353038709, 2212424333, 2110412913, 3631228061, 2765134272, - 4025821789, 3324834269, 187577732, 1568270802, 2098502315, 2472645526, 2986813860, - 1621191378, 3891512282, 1561648319, 2690491944, 3075246584, 3202791012, 315381589, - 3645907425, 3532420114, 802256935, 1270128258, 2695868207, 4075358890, 3888212208, - 510396943, 3683116722, 3943939501, 146061942, 733291914, 1402325031, 672641124, - 2817168601, 2622398925, 3641379870, 2969146913, 4232866548, 1694492034, 3065141682, - 234404736, 1921499010, 2300706258, 1304904939, 207802178, 2674605425, 2688377241, - 2674991105, 2585496531, 2358858923, 2578793432, 3275116043, 228073476, 2936443283, - 3713102344, 1629243323, 209348683, 3730808488, 275442226, 223820143, 2365614109, - 3017206322, 1906208795, - }, - { - 1545504510, 1985586093, 2005504076, 2487099791, 2348737867, 2254755902, 3789154730, - 3268946922, 99552511, 1369361877, 1888041043, 3105269579, 4044127396, 2380045264, - 2970234287, 293292961, 1811276320, 1083136897, 3016497500, 950611584, 2165628367, - 4140133899, 2402926185, 990501164, 2185997143, 1769871204, 721625457, 567446962, - 1695515231, 1848699963, 4163520111, 2316975723, 4268269680, 1021066723, 517434635, - 3827063239, 3483118065, 760366769, 3072996795, 3548263896, 2131401627, 4167855065, - 410255606, 1992500865, 1322267629, 1599293552, 2389387938, 3721625360, 216375429, - 2002236178, 1834631738, 1585275126, 3879559071, 2517667239, 1397456303, 4095227658, - 589002062, 137665950, 3933018338, 1519132173, 3566494128, 3914066872, 3233332246, - 855336825, 1882502420, 1081015168, 4148374722, 1683880703, 1161266344, 99374978, - 733926790, 3520260556, 3643143173, 927318029, 398003191, 3472026294, 3518018860, - 2319507998, 2650129369, 3781620600, 1294634949, 3977318486, 3068540117, 3732334866, - 740308004, 1988900647, 2936479173, 2348744493, 1357856242, 3842428732, 3746094733, - 214260739, 3493892012, 2358001919, 1775614809, 952871363, 1216985499, 2706067772, - 1008517818, 4189424856, 1260334069, 2420035836, 311831945, 3409272605, 4266242510, - 3590716427, 537257045, 3153762469, 1620749663, 3338743851, 3644831936, 3243426619, - 783551642, 1305153827, 2026979662, 3164955857, 4082645339, 1633544228, 3389303153, - 440623817, 204979344, 1674764841, 633231391, 4180702701, 1953210184, 2534954734, - 4252100558, 2993632630, 4050264705, 678445398, 1502035091, 302442688, 493504779, - 2321459487, 1141171231, 1507727159, 672678623, 4046722895, 65675127, 2936731189, - 441159654, 832039862, 2252252769, 3090962795, 2839688755, 645344032, 2921087914, - 2264738834, 2341060101, 778789539, 737962654, 2859693559, 2784310535, 493247978, - 185832691, 3321631011, 641506549, 2652806878, 480335604, 2908694258, 984807024, - 1545504510, 1985586093, 2005504076, 2487099791, 127488455, 2254755902, 3789154730, - 3268946922, 99552511, 2160330513, 1888041043, 3105269579, 4044127396, 2380045264, - 3185912634, 293292961, 1811276320, 1083136897, 3016497500, 116883339, 2165628367, - 4140133899, 2402926185, 990501164, 4099344218, 1769871204, 721625457, 567446962, - 1695515231, 1218419978, 4163520111, 2316975723, 4268269680, 1021066723, 237254804, - 3827063239, 3483118065, 760366769, 3072996795, 1020639813, 2131401627, 4167855065, - 410255606, 1992500865, 1887858126, 1599293552, 2389387938, 3721625360, 216375429, - 2096265248, 1834631738, 1585275126, 3879559071, 2517667239, 3267338158, 4095227658, - 589002062, 137665950, 3933018338, 3823062902, 3566494128, 3914066872, 3233332246, - 855336825, 3240858503, 1081015168, 4148374722, 1683880703, 1161266344, 4034899335, - 733926790, 3520260556, 3643143173, 927318029, 2130442867, 3472026294, 3518018860, - 2319507998, 2650129369, 253769320, 1294634949, 3977318486, 3068540117, 3732334866, - 3100107703, 1988900647, 2936479173, 2348744493, 1357856242, 477065277, 3746094733, - 214260739, 3493892012, 2358001919, 52055911, 952871363, 1216985499, 2706067772, - 1008517818, 2820619262, 1260334069, 2420035836, 311831945, 3409272605, 2066128794, - 3590716427, 537257045, 3153762469, 1620749663, 2261931254, 3644831936, 3243426619, - 783551642, 1305153827, 3937339872, 3164955857, 4082645339, 1633544228, 3389303153, - 3304461891, 204979344, 1674764841, 633231391, 4180702701, 2649553051, 2534954734, - 4252100558, 2993632630, 4050264705, 3777379050, 1502035091, 302442688, 493504779, - 2321459487, 1795212504, 1507727159, 672678623, 4046722895, 65675127, 2810951967, - 441159654, 832039862, 2252252769, 3090962795, 3317253399, 645344032, 2921087914, - 2264738834, 2341060101, 1431934790, 737962654, 2859693559, 2784310535, 493247978, - 555655767, 3321631011, 641506549, 2652806878, 480335604, 1837415425, 984807024, - 1545504510, 1985586093, 2005504076, 2274320195, 127488455, 2254755902, 3789154730, - 3268946922, 3812459919, 2160330513, 1888041043, 3105269579, 4044127396, 2341347785, - 3185912634, 293292961, 1811276320, 1083136897, 825098089, 116883339, 2165628367, - 4140133899, 2402926185, 4124720284, 4099344218, 1769871204, 721625457, 567446962, - 3598160577, 1218419978, 4163520111, 2316975723, 4268269680, 923374392, 237254804, - 3827063239, 3483118065, 760366769, 2263405553, 1020639813, 2131401627, 4167855065, - 410255606, 3382265961, 1887858126, 1599293552, 2389387938, 3721625360, 3440586186, - 2096265248, 1834631738, 1585275126, 3879559071, 711626863, 3267338158, 4095227658, - 589002062, 137665950, 1190761134, 3823062902, 3566494128, 3914066872, 3233332246, - 3844456625, 3240858503, 1081015168, 4148374722, 1683880703, 589447946, 4034899335, - 733926790, 3520260556, 3643143173, 3202263729, 2130442867, 3472026294, 3518018860, - 2319507998, 3458685425, 253769320, 1294634949, 3977318486, 3068540117, 702365700, - 3100107703, 1988900647, 2936479173, 2348744493, 969926974, 477065277, 3746094733, - 214260739, 3493892012, 2890740482, 52055911, 952871363, 1216985499, 2706067772, - 1079370138, 2820619262, 1260334069, 2420035836, 311831945, 701108525, 2066128794, - 3590716427, 537257045, 3153762469, 2900214585, 2261931254, 3644831936, 3243426619, - 783551642, 3143067452, 3937339872, 3164955857, 4082645339, 1633544228, 1680728882, - 3304461891, 204979344, 1674764841, 633231391, 689425572, 2649553051, 2534954734, - 4252100558, 2993632630, 865432399, 3777379050, 1502035091, 302442688, 493504779, - 1282312650, 1795212504, 1507727159, 672678623, 4046722895, 976003271, 2810951967, - 441159654, 832039862, 2252252769, 726554843, 3317253399, 645344032, 2921087914, - 2264738834, 1325395107, 1431934790, 737962654, 2859693559, 2784310535, 3876486226, - 555655767, 3321631011, 641506549, 2652806878, 3848380198, 1837415425, 984807024, - 1545504510, 1985586093, 3711682090, 2274320195, 127488455, 2254755902, 3789154730, - 1595223697, 3812459919, 2160330513, 1888041043, 3105269579, 2773455385, 2341347785, - 3185912634, 293292961, 1811276320, 3280464626, 825098089, 116883339, 2165628367, - 4140133899, 3092114881, 4124720284, 4099344218, 1769871204, 721625457, 1514083147, - 3598160577, 1218419978, 4163520111, 2316975723, 200993429, 923374392, 237254804, - 3827063239, 3483118065, 677187089, 2263405553, 1020639813, 2131401627, 4167855065, - 1892382552, 3382265961, 1887858126, 1599293552, 2389387938, 4153928364, 3440586186, - 2096265248, 1834631738, 1585275126, 3348317504, 711626863, 3267338158, 4095227658, - 589002062, 3125839176, 1190761134, 3823062902, 3566494128, 3914066872, 1320578396, - 3844456625, 3240858503, 1081015168, 4148374722, 258762412, 589447946, 4034899335, - 733926790, 3520260556, 4290301810, 3202263729, 2130442867, 3472026294, 3518018860, - 2904238635, 3458685425, 253769320, 1294634949, 3977318486, 2517006218, 702365700, - 3100107703, 1988900647, 2936479173, 3227096174, 969926974, 477065277, 3746094733, - 214260739, 3868449115, 2890740482, 52055911, 952871363, 1216985499, 2857823043, - 1079370138, 2820619262, 1260334069, 2420035836, 1843837226, 701108525, 2066128794, - 3590716427, 537257045, 1202524172, 2900214585, 2261931254, 3644831936, 3243426619, - 2113758468, 3143067452, 3937339872, 3164955857, 4082645339, 3987431298, 1680728882, - 3304461891, 204979344, 1674764841, 2684386058, 689425572, 2649553051, 2534954734, - 4252100558, 3511996574, 865432399, 3777379050, 1502035091, 302442688, 970989610, - 1282312650, 1795212504, 1507727159, 672678623, 3080995547, 976003271, 2810951967, - 441159654, 832039862, 2670291295, 726554843, 3317253399, 645344032, 2921087914, - 3039207936, 1325395107, 1431934790, 737962654, 2859693559, 2452474228, 3876486226, - 555655767, 3321631011, 641506549, 712394572, 931322445, 3691485988, 77755644, - 3585967569, 1546642657, 1074481665, 1211742891, 2405208503, 1015438825, 3187019083, - 2194891243, 1305917012, 3737279586, 2633137983, 1924729261, 72781059, 1412697099, - 3828782214, 1637665425, 4170514983, 2248277352, 3793164712, 2365683667, 1287488796, - 3240061130, 2411573225, 3237771995, 901649504, 4107276625, 1613775409, 741888560, - 332459303, 850991886, 3249391248, 3550484151, 3689717953, 233288631, 2496730550, - 3221264250, 3172144573, 1429937065, 1776357872, 1084763904, 1993209913, 4142869218, - 3130780078, 18180577, 2819625557, 1978393449, 372704074, 3919523286, 1777756963, - 188652529, 411213996, 62282979, 3775037518, 2534579861, 2966280971, 3863833471, - 3228893189, 3123894696, 362579125, 1232030882, 575379775, 1019196436, 1914161190, - 3649246842, 2192095564, 2368224476, 138396720, 1299868479, 507152626, 2129033575, - 3801624222, 623352301, 1551535796, 3848329776, 2727905150, 1109499603, 3222756581, - 3914846131, 3207366497, 3216028717, 3712661572, 1970542, 1320230637, 2583706801, - 1341029904, 1903168049, 1244252579, 1885511879, 2426625042, 3082846847, 3858784104, - 2263210027, 130350645, 956540733, 776729371, 2266749094, 2220603773, 2556170531, - 263980324, 802194348, 697108594, 3634984969, 4251738712, 1831444758, 1209156358, - 3089957258, 4195548426, 3641578987, 990686800, 2391278490, 2233755358, 1739784005, - 2458544650, 340925249, 2442887806, 3503407512, 3058778909, 3619026333, 2289286518, - 1296212011, 3879317178, 1210295163, 3113210467, 1578990986, 641384071, 2437977832, - 1689385197, 1323268226, 861337916, 3532905860, 3735971843, 2294673483, 1032787575, - 1868992735, 4260308791, 2091311463, 2354047234, 1005300697, 29821726, 2790044161, - 3154591207, 1370229266, 3464848205, 3855301526, 544374401, 101012897, 4214903025, - 1310520049, 14884434, 1438288148, 2118574986, 2360002070, 512167778, 4186534704, - 3633828199, 493600836, - }, - { - 2932801042, 4101748508, 3363559072, 1213475638, 2400369070, 1726749444, 3175844814, - 2600020277, 3779799804, 1886667522, 1228105891, 589138388, 3960459504, 450669757, - 3773736740, 2107201112, 1437834675, 3618095315, 3662453347, 968349971, 1891706458, - 2333451375, 4242907074, 3265111057, 3648168902, 4137035018, 105573058, 2075999861, - 1053920954, 3768713177, 1836088599, 2015103258, 2649187541, 2717894301, 534937136, - 3492326400, 2406499346, 617315838, 1384748442, 519804615, 524657043, 832148261, - 156272480, 394759604, 2428809631, 3401589884, 2588359262, 3826333418, 2427993050, - 3254067543, 2570694144, 2876613091, 2883884893, 613070434, 1599903665, 3476967713, - 1729385632, 207879231, 1256308247, 2538975486, 2550001448, 1820975095, 915640692, - 1633749116, 1294669585, 3257901643, 3193347552, 3369630539, 285165240, 2337727802, - 1854640523, 1034379307, 1206304638, 889104297, 3084078942, 3485609519, 3903898589, - 4274630316, 3290195566, 2071163950, 775170461, 551343738, 164916146, 1678786363, - 123960948, 2721608023, 3463122611, 1525791510, 1531697627, 1457848578, 665433501, - 1784274031, 3436850186, 3976095421, 383031580, 2146948399, 3137780800, 410458873, - 381977170, 4264728702, 1515223147, 3358033956, 139804933, 438534588, 901342240, - 1536972976, 184570377, 681864510, 844333847, 2515362910, 917461167, 2538721219, - 4268394152, 680292330, 3420438710, 3784725677, 1983802086, 4165891809, 2369490764, - 3808530114, 3391499460, 2509287180, 970129219, 2492785859, 3611863290, 1303524794, - 2991964551, 1828774928, 3950385781, 3251583775, 14901408, 1890180396, 1306701779, - 3161784071, 637842485, 2830070006, 3867491336, 1594948357, 2579795132, 479188700, - 806498245, 3905876458, 3499065005, 3168076042, 769094339, 3769363696, 1241457026, - 1073618847, 251335726, 2574341631, 2534047421, 3151952274, 534046859, 3264754113, - 1325368288, 2131927230, 3229420672, 336348290, 3768781638, 2593952436, 849969290, - 2932801042, 4101748508, 3363559072, 1213475638, 1710895496, 1726749444, 3175844814, - 2600020277, 3779799804, 4044580435, 1228105891, 589138388, 3960459504, 450669757, - 4253882965, 2107201112, 1437834675, 3618095315, 3662453347, 3625360228, 1891706458, - 2333451375, 4242907074, 3265111057, 3638586625, 4137035018, 105573058, 2075999861, - 1053920954, 3014895241, 1836088599, 2015103258, 2649187541, 2717894301, 701652515, - 3492326400, 2406499346, 617315838, 1384748442, 1142040801, 524657043, 832148261, - 156272480, 394759604, 944890908, 3401589884, 2588359262, 3826333418, 2427993050, - 337891051, 2570694144, 2876613091, 2883884893, 613070434, 659063916, 3476967713, - 1729385632, 207879231, 1256308247, 311608860, 2550001448, 1820975095, 915640692, - 1633749116, 1772334285, 3257901643, 3193347552, 3369630539, 285165240, 2627441892, - 1854640523, 1034379307, 1206304638, 889104297, 2289660031, 3485609519, 3903898589, - 4274630316, 3290195566, 3572160580, 775170461, 551343738, 164916146, 1678786363, - 3109616684, 2721608023, 3463122611, 1525791510, 1531697627, 3660976089, 665433501, - 1784274031, 3436850186, 3976095421, 1696775162, 2146948399, 3137780800, 410458873, - 381977170, 1669455215, 1515223147, 3358033956, 139804933, 438534588, 1738237971, - 1536972976, 184570377, 681864510, 844333847, 770765754, 917461167, 2538721219, - 4268394152, 680292330, 1993152157, 3784725677, 1983802086, 4165891809, 2369490764, - 3411542022, 3391499460, 2509287180, 970129219, 2492785859, 1869391890, 1303524794, - 2991964551, 1828774928, 3950385781, 4139486157, 14901408, 1890180396, 1306701779, - 3161784071, 174545194, 2830070006, 3867491336, 1594948357, 2579795132, 4132973523, - 806498245, 3905876458, 3499065005, 3168076042, 538076966, 3769363696, 1241457026, - 1073618847, 251335726, 2085586137, 2534047421, 3151952274, 534046859, 3264754113, - 643987981, 2131927230, 3229420672, 336348290, 3768781638, 3468816701, 849969290, - 2932801042, 4101748508, 3363559072, 2524943673, 1710895496, 1726749444, 3175844814, - 2600020277, 3677241699, 4044580435, 1228105891, 589138388, 3960459504, 3903077887, - 4253882965, 2107201112, 1437834675, 3618095315, 2362822379, 3625360228, 1891706458, - 2333451375, 4242907074, 2289503940, 3638586625, 4137035018, 105573058, 2075999861, - 1299938293, 3014895241, 1836088599, 2015103258, 2649187541, 3727003343, 701652515, - 3492326400, 2406499346, 617315838, 1627975589, 1142040801, 524657043, 832148261, - 156272480, 3658645823, 944890908, 3401589884, 2588359262, 3826333418, 3645806126, - 337891051, 2570694144, 2876613091, 2883884893, 2866570997, 659063916, 3476967713, - 1729385632, 207879231, 298556768, 311608860, 2550001448, 1820975095, 915640692, - 1014996737, 1772334285, 3257901643, 3193347552, 3369630539, 96395889, 2627441892, - 1854640523, 1034379307, 1206304638, 2546521293, 2289660031, 3485609519, 3903898589, - 4274630316, 2360048518, 3572160580, 775170461, 551343738, 164916146, 2068601014, - 3109616684, 2721608023, 3463122611, 1525791510, 1228011534, 3660976089, 665433501, - 1784274031, 3436850186, 1620580129, 1696775162, 2146948399, 3137780800, 410458873, - 2753059283, 1669455215, 1515223147, 3358033956, 139804933, 2786429190, 1738237971, - 1536972976, 184570377, 681864510, 358796749, 770765754, 917461167, 2538721219, - 4268394152, 2355846025, 1993152157, 3784725677, 1983802086, 4165891809, 360259050, - 3411542022, 3391499460, 2509287180, 970129219, 4055494275, 1869391890, 1303524794, - 2991964551, 1828774928, 3508750618, 4139486157, 14901408, 1890180396, 1306701779, - 3684762156, 174545194, 2830070006, 3867491336, 1594948357, 702781070, 4132973523, - 806498245, 3905876458, 3499065005, 1372989388, 538076966, 3769363696, 1241457026, - 1073618847, 3579114424, 2085586137, 2534047421, 3151952274, 534046859, 1882037168, - 643987981, 2131927230, 3229420672, 336348290, 555833786, 3468816701, 849969290, - 2932801042, 4101748508, 1095934625, 2524943673, 1710895496, 1726749444, 3175844814, - 2287140069, 3677241699, 4044580435, 1228105891, 589138388, 1596938176, 3903077887, - 4253882965, 2107201112, 1437834675, 2605388022, 2362822379, 3625360228, 1891706458, - 2333451375, 174003035, 2289503940, 3638586625, 4137035018, 105573058, 697023108, - 1299938293, 3014895241, 1836088599, 2015103258, 4128339205, 3727003343, 701652515, - 3492326400, 2406499346, 426422678, 1627975589, 1142040801, 524657043, 832148261, - 2461054373, 3658645823, 944890908, 3401589884, 2588359262, 3184255074, 3645806126, - 337891051, 2570694144, 2876613091, 187151044, 2866570997, 659063916, 3476967713, - 1729385632, 2811989057, 298556768, 311608860, 2550001448, 1820975095, 1806779934, - 1014996737, 1772334285, 3257901643, 3193347552, 2145947779, 96395889, 2627441892, - 1854640523, 1034379307, 2748996070, 2546521293, 2289660031, 3485609519, 3903898589, - 452746826, 2360048518, 3572160580, 775170461, 551343738, 669098691, 2068601014, - 3109616684, 2721608023, 3463122611, 22889155, 1228011534, 3660976089, 665433501, - 1784274031, 227705324, 1620580129, 1696775162, 2146948399, 3137780800, 4267814323, - 2753059283, 1669455215, 1515223147, 3358033956, 2806778033, 2786429190, 1738237971, - 1536972976, 184570377, 3310279262, 358796749, 770765754, 917461167, 2538721219, - 2247224091, 2355846025, 1993152157, 3784725677, 1983802086, 2399541755, 360259050, - 3411542022, 3391499460, 2509287180, 2335541531, 4055494275, 1869391890, 1303524794, - 2991964551, 392724462, 3508750618, 4139486157, 14901408, 1890180396, 2513331299, - 3684762156, 174545194, 2830070006, 3867491336, 1887131931, 702781070, 4132973523, - 806498245, 3905876458, 2263606492, 1372989388, 538076966, 3769363696, 1241457026, - 170472774, 3579114424, 2085586137, 2534047421, 3151952274, 1488165272, 1882037168, - 643987981, 2131927230, 3229420672, 1158405862, 1469009373, 4117356830, 4063868500, - 2006417445, 2976934394, 2683607933, 3174943272, 2099974138, 2250858961, 205251124, - 84783688, 1551294676, 224349432, 1893741756, 3680361724, 561624088, 251553631, - 1654870642, 2195380145, 866503297, 1814519294, 905566144, 727763043, 1910034093, - 1876316198, 3031876716, 2783769690, 2649650479, 2024342098, 2170858649, 2186613759, - 2688207487, 881594599, 1010953695, 2768977700, 3341020856, 2446339960, 2648757147, - 1317083878, 3301541769, 3574285525, 3331294407, 712581268, 3612116700, 3510601489, - 2569879282, 3772968052, 332485239, 280920979, 716834274, 1863623285, 654670865, - 1706917935, 1598315563, 2486805657, 2295746319, 635609792, 55141757, 4089183045, - 145257162, 1921789879, 2833550514, 3798992859, 1532875864, 3668053062, 2749191097, - 3412220447, 3383752088, 3191842833, 4167387125, 2438940746, 1453011669, 2747298308, - 1057877757, 399006034, 132680506, 31671249, 1070386969, 2415113777, 3720335676, - 3416473189, 1476808053, 785398955, 3335661823, 315496929, 1421907623, 1802371914, - 3049258946, 1773374729, 382902076, 3262814446, 1774244917, 4064677234, 2281551331, - 3019541390, 2445483046, 3059154103, 2147309319, 566587847, 216051987, 521013398, - 2721884570, 3325443529, 1921922591, 1643064709, 1155714395, 1737031844, 2117338012, - 1876262536, 3589621009, 3800806613, 1102108318, 1376914700, 539544394, 799741508, - 1192097712, 2894663754, 567276527, 106814343, 3985577014, 422246623, 126568764, - 4008211389, 4037889581, 2185357423, 2239644921, 2116447019, 1249715620, 2095747493, - 4063243162, 3059330950, 1045571624, 1150656233, 3024439196, 3981904623, 1743764595, - 4220253496, 3322182853, 2132911849, 2074342674, 198749193, 574306951, 3563262292, - 3832626833, 2349475213, 182567249, 1530390173, 2066055611, 2609802571, 1392638962, - 1495846580, 2356952332, 4029921749, 1731839848, 527880959, 1204112231, 938004695, - 294300378, 1855457892, - }, - { - 1438083560, 1727969469, 703174449, 1296281193, 1386452240, 3304170302, 3048300096, - 277697908, 2675939661, 3382564518, 1639425457, 2210719281, 3173605115, 1685375802, - 1317820682, 1960916541, 4230888182, 1924357010, 3322827982, 1663716994, 976583570, - 4146230815, 525755678, 3608894680, 1715438458, 1519478303, 2845291872, 1115405802, - 2468673244, 2289739992, 46988928, 2559411080, 2466723374, 2995303634, 3871022237, - 1794652692, 2424766096, 2849910020, 978542234, 1667051478, 3393290740, 1508376445, - 4090541488, 1314139749, 1271060027, 3272019878, 4032394060, 757805987, 619143288, - 1165760536, 225099797, 871754591, 2065691940, 2016593817, 1705071529, 2559080067, - 2048856253, 3217759224, 2691334730, 1576829868, 3356759591, 1570481357, 1097065360, - 852561431, 3559721965, 1403648739, 1772347635, 1196457607, 462142253, 761176322, - 2209893444, 217724244, 3356132814, 2838131962, 3571552868, 1197135963, 3239010986, - 2612283238, 2606429155, 2194090162, 4256137634, 935551404, 3057660021, 866672836, - 1119670384, 1757615349, 649402076, 2814108193, 3312658713, 2627947214, 2982267121, - 486762785, 2746076238, 2134737126, 4106010468, 3151832629, 2419694200, 2803791741, - 2100250718, 3171079849, 1874606681, 1884940331, 926257211, 1940082331, 1024435222, - 609478334, 2501896844, 518643063, 4285619138, 1054300997, 4024681853, 2287236199, - 2891891855, 1519666047, 1919500932, 3880316442, 1994336737, 1025147784, 3433493260, - 1647319600, 3298872174, 3744513628, 2918990402, 2649193481, 234630674, 1963357481, - 1118148435, 2658522312, 2563194501, 2238556876, 1210050812, 748709882, 3894824022, - 2575692519, 436044710, 3465014792, 3686094502, 2963529475, 3251316066, 2834750227, - 789471563, 853201732, 4119014483, 1312738151, 2018934495, 542908921, 732294449, - 2519981401, 1663929229, 4041419972, 3038382188, 3182489020, 353453260, 4074472601, - 1187952022, 2118553383, 1068338764, 3699144039, 3129056770, 1419222328, 2666827910, - 1438083560, 1727969469, 703174449, 1296281193, 2134413940, 3304170302, 3048300096, - 277697908, 2675939661, 3817858752, 1639425457, 2210719281, 3173605115, 1685375802, - 2587083472, 1960916541, 4230888182, 1924357010, 3322827982, 2582901426, 976583570, - 4146230815, 525755678, 3608894680, 524232549, 1519478303, 2845291872, 1115405802, - 2468673244, 591800699, 46988928, 2559411080, 2466723374, 2995303634, 2307625850, - 1794652692, 2424766096, 2849910020, 978542234, 1284927074, 3393290740, 1508376445, - 4090541488, 1314139749, 3508281898, 3272019878, 4032394060, 757805987, 619143288, - 1846615167, 225099797, 871754591, 2065691940, 2016593817, 1193455869, 2559080067, - 2048856253, 3217759224, 2691334730, 2665708717, 3356759591, 1570481357, 1097065360, - 852561431, 1652864273, 1403648739, 1772347635, 1196457607, 462142253, 1222855287, - 2209893444, 217724244, 3356132814, 2838131962, 3060983219, 1197135963, 3239010986, - 2612283238, 2606429155, 4171729370, 4256137634, 935551404, 3057660021, 866672836, - 75618353, 1757615349, 649402076, 2814108193, 3312658713, 3975515213, 2982267121, - 486762785, 2746076238, 2134737126, 3251020123, 3151832629, 2419694200, 2803791741, - 2100250718, 624531676, 1874606681, 1884940331, 926257211, 1940082331, 3678479182, - 609478334, 2501896844, 518643063, 4285619138, 1725899979, 4024681853, 2287236199, - 2891891855, 1519666047, 702508101, 3880316442, 1994336737, 1025147784, 3433493260, - 4212959134, 3298872174, 3744513628, 2918990402, 2649193481, 1782150764, 1963357481, - 1118148435, 2658522312, 2563194501, 3330122355, 1210050812, 748709882, 3894824022, - 2575692519, 637240921, 3465014792, 3686094502, 2963529475, 3251316066, 1510158901, - 789471563, 853201732, 4119014483, 1312738151, 3018953017, 542908921, 732294449, - 2519981401, 1663929229, 2696317636, 3038382188, 3182489020, 353453260, 4074472601, - 4249950407, 2118553383, 1068338764, 3699144039, 3129056770, 2334590922, 2666827910, - 1438083560, 1727969469, 703174449, 1679528518, 2134413940, 3304170302, 3048300096, - 277697908, 3417107827, 3817858752, 1639425457, 2210719281, 3173605115, 1858788112, - 2587083472, 1960916541, 4230888182, 1924357010, 3692988029, 2582901426, 976583570, - 4146230815, 525755678, 1122319464, 524232549, 1519478303, 2845291872, 1115405802, - 205855120, 591800699, 46988928, 2559411080, 2466723374, 3358512221, 2307625850, - 1794652692, 2424766096, 2849910020, 2865273283, 1284927074, 3393290740, 1508376445, - 4090541488, 2453941323, 3508281898, 3272019878, 4032394060, 757805987, 3191753865, - 1846615167, 225099797, 871754591, 2065691940, 1301630578, 1193455869, 2559080067, - 2048856253, 3217759224, 3858428004, 2665708717, 3356759591, 1570481357, 1097065360, - 3550687085, 1652864273, 1403648739, 1772347635, 1196457607, 2158802672, 1222855287, - 2209893444, 217724244, 3356132814, 1954043011, 3060983219, 1197135963, 3239010986, - 2612283238, 2156334822, 4171729370, 4256137634, 935551404, 3057660021, 3331206175, - 75618353, 1757615349, 649402076, 2814108193, 1313890357, 3975515213, 2982267121, - 486762785, 2746076238, 2023213803, 3251020123, 3151832629, 2419694200, 2803791741, - 392313450, 624531676, 1874606681, 1884940331, 926257211, 3369012310, 3678479182, - 609478334, 2501896844, 518643063, 3638013610, 1725899979, 4024681853, 2287236199, - 2891891855, 429282096, 702508101, 3880316442, 1994336737, 1025147784, 1217486411, - 4212959134, 3298872174, 3744513628, 2918990402, 1279832521, 1782150764, 1963357481, - 1118148435, 2658522312, 2379123622, 3330122355, 1210050812, 748709882, 3894824022, - 3987054169, 637240921, 3465014792, 3686094502, 2963529475, 2167876400, 1510158901, - 789471563, 853201732, 4119014483, 1746447311, 3018953017, 542908921, 732294449, - 2519981401, 1908715414, 2696317636, 3038382188, 3182489020, 353453260, 2132930364, - 4249950407, 2118553383, 1068338764, 3699144039, 433893434, 2334590922, 2666827910, - 1438083560, 1727969469, 1154725669, 1679528518, 2134413940, 3304170302, 3048300096, - 31944135, 3417107827, 3817858752, 1639425457, 2210719281, 4203237786, 1858788112, - 2587083472, 1960916541, 4230888182, 2712081548, 3692988029, 2582901426, 976583570, - 4146230815, 3948659885, 1122319464, 524232549, 1519478303, 2845291872, 2881616509, - 205855120, 591800699, 46988928, 2559411080, 3645011109, 3358512221, 2307625850, - 1794652692, 2424766096, 3667888476, 2865273283, 1284927074, 3393290740, 1508376445, - 1605429636, 2453941323, 3508281898, 3272019878, 4032394060, 3904681057, 3191753865, - 1846615167, 225099797, 871754591, 696516502, 1301630578, 1193455869, 2559080067, - 2048856253, 2589248412, 3858428004, 2665708717, 3356759591, 1570481357, 1884333722, - 3550687085, 1652864273, 1403648739, 1772347635, 3418430008, 2158802672, 1222855287, - 2209893444, 217724244, 4164333189, 1954043011, 3060983219, 1197135963, 3239010986, - 2300947859, 2156334822, 4171729370, 4256137634, 935551404, 1258856668, 3331206175, - 75618353, 1757615349, 649402076, 772455867, 1313890357, 3975515213, 2982267121, - 486762785, 3671941628, 2023213803, 3251020123, 3151832629, 2419694200, 4264015999, - 392313450, 624531676, 1874606681, 1884940331, 2460787316, 3369012310, 3678479182, - 609478334, 2501896844, 2131090271, 3638013610, 1725899979, 4024681853, 2287236199, - 455349830, 429282096, 702508101, 3880316442, 1994336737, 1727894434, 1217486411, - 4212959134, 3298872174, 3744513628, 1120563681, 1279832521, 1782150764, 1963357481, - 1118148435, 3362151087, 2379123622, 3330122355, 1210050812, 748709882, 2506587900, - 3987054169, 637240921, 3465014792, 3686094502, 1265652315, 2167876400, 1510158901, - 789471563, 853201732, 3472479264, 1746447311, 3018953017, 542908921, 732294449, - 659090240, 1908715414, 2696317636, 3038382188, 3182489020, 174113867, 2132930364, - 4249950407, 2118553383, 1068338764, 4115132848, 1714842877, 1153237667, 1015943026, - 2014412384, 2478393613, 1340079052, 167685322, 1848482402, 3252973254, 638064461, - 1599254200, 2525050247, 2813349060, 2415037971, 3274852801, 3415369586, 3216396500, - 3147792606, 438338168, 2326605175, 2846648724, 3871841623, 287840506, 3218295001, - 2562000356, 574276928, 418096348, 1798854554, 1913561074, 2025706546, 41907788, - 3535708035, 1240819558, 208810147, 4062740265, 451865782, 2652508890, 3579720859, - 1243967909, 2191937647, 2473947838, 1847359263, 2496539569, 4061942257, 1372849161, - 2016697844, 1827460131, 1135062647, 1255573479, 3506657283, 3699699807, 3087913374, - 1196140869, 4095306490, 830793530, 1289366065, 3268392251, 4119035690, 1631012325, - 3410799501, 1470209122, 3057922764, 2895379380, 2654121201, 1984999545, 2258412956, - 4267137150, 3396740662, 2480013857, 3845856317, 3669454152, 2438423716, 3191341994, - 1571280634, 1423782557, 3279999352, 1886288620, 205278284, 793062897, 112852083, - 69164746, 2218046933, 4206182754, 3021072495, 2157753215, 2875773583, 1453706073, - 168681204, 3905840714, 4098714445, 3410804508, 1737239929, 1613207828, 2987997090, - 1869303136, 3348561687, 3391148819, 1680062950, 4150476788, 2340622122, 11331065, - 2250669421, 3003852975, 2145739501, 1627177260, 994260425, 1479134620, 2315299915, - 1268765340, 285960682, 3801150032, 3948820512, 1677682247, 1735541155, 1914753931, - 1965156079, 1875233710, 681418791, 2077804400, 1963479724, 2447942398, 269798686, - 2740088859, 1974178779, 3373487761, 2879779843, 157827737, 3855390825, 2779173093, - 2359181541, 3508102362, 4001266348, 3949912729, 3232414439, 472195874, 57835121, - 1854343116, 3020785997, 2024437594, 2182964208, 3379376555, 1213864603, 307833006, - 1029130725, 545051507, 4001695571, 2258480284, 896286117, 355474524, 2514583184, - 2997458384, 3278715462, 1675341954, 3603020014, 2318410671, 2152785892, 4285597912, - 35655711, 2087100216, - }, - { - 1671155, 472949658, 148656515, 1640075411, 930771231, 1601854390, 471598090, - 2013359012, 3708325970, 1688441844, 736452516, 100585026, 1154373750, 4029833741, - 3409420465, 192349301, 3804215437, 909027311, 2896874106, 3567276364, 1319305666, - 3858990362, 3155018279, 3756192170, 3567813642, 228734829, 577956164, 2078807284, - 1005987081, 1464380935, 112604551, 3865074232, 3776350052, 1112767766, 2947509331, - 910887552, 4127297396, 851240323, 3136588838, 1639013085, 1154068086, 639126620, - 2501600773, 3174842042, 3456593672, 80596481, 126970446, 2184239961, 1448001095, - 689252599, 1087028487, 2905348107, 2502009404, 2156595397, 2149975474, 2201723284, - 3908202640, 754508313, 2321393187, 787043244, 2575809693, 4172462501, 2322897687, - 1899992264, 1854136781, 3575249683, 2939319477, 901605762, 676398674, 2849283587, - 2992300101, 1513271778, 2797164148, 1914019034, 1889341710, 2739211008, 1954453463, - 3279391005, 2899313529, 1412533980, 1291505093, 2884603001, 564097935, 3552741248, - 2809901827, 1263126330, 860214490, 2168366043, 2681035029, 3226888214, 2902522885, - 554804421, 1571065517, 3322453053, 4144256215, 126415290, 980853251, 1531963815, - 3237470129, 1465444883, 2031491001, 2205009469, 1046577915, 828927962, 2170245718, - 1090142292, 1667375106, 2522840205, 4047872402, 3862734726, 91588630, 3122782857, - 929883614, 694999008, 1472139068, 1246663706, 3500613893, 4200173807, 186199942, - 3890621040, 229752655, 1011692880, 2791828564, 2677625011, 791005643, 1754509337, - 2321492983, 3512328605, 1294405891, 2845189858, 434175992, 3155484007, 2306406482, - 3197931140, 22971924, 1521633702, 2366802562, 399245037, 2833224222, 2507478835, - 3231711673, 3784114896, 1927919696, 783802899, 3408133710, 2278711709, 3001078924, - 1223320630, 3246830042, 943189685, 4062534962, 1039971013, 2342241593, 3551623946, - 322017346, 3585779636, 81127429, 3549929990, 2886997195, 1746081951, 4169018554, - 1671155, 472949658, 148656515, 1640075411, 3772042754, 1601854390, 471598090, - 2013359012, 3708325970, 321630853, 736452516, 100585026, 1154373750, 4029833741, - 1926754199, 192349301, 3804215437, 909027311, 2896874106, 1138131968, 1319305666, - 3858990362, 3155018279, 3756192170, 2489094664, 228734829, 577956164, 2078807284, - 1005987081, 2678967510, 112604551, 3865074232, 3776350052, 1112767766, 626049886, - 910887552, 4127297396, 851240323, 3136588838, 2142891352, 1154068086, 639126620, - 2501600773, 3174842042, 3342870442, 80596481, 126970446, 2184239961, 1448001095, - 3399719246, 1087028487, 2905348107, 2502009404, 2156595397, 14860817, 2201723284, - 3908202640, 754508313, 2321393187, 90540547, 2575809693, 4172462501, 2322897687, - 1899992264, 56239065, 3575249683, 2939319477, 901605762, 676398674, 412461711, - 2992300101, 1513271778, 2797164148, 1914019034, 3660190396, 2739211008, 1954453463, - 3279391005, 2899313529, 4193503742, 1291505093, 2884603001, 564097935, 3552741248, - 2124229268, 1263126330, 860214490, 2168366043, 2681035029, 4086980935, 2902522885, - 554804421, 1571065517, 3322453053, 1821678887, 126415290, 980853251, 1531963815, - 3237470129, 2099629264, 2031491001, 2205009469, 1046577915, 828927962, 3447807375, - 1090142292, 1667375106, 2522840205, 4047872402, 2255362927, 91588630, 3122782857, - 929883614, 694999008, 4135967848, 1246663706, 3500613893, 4200173807, 186199942, - 4182379872, 229752655, 1011692880, 2791828564, 2677625011, 397062412, 1754509337, - 2321492983, 3512328605, 1294405891, 1028843071, 434175992, 3155484007, 2306406482, - 3197931140, 3217107401, 1521633702, 2366802562, 399245037, 2833224222, 76017436, - 3231711673, 3784114896, 1927919696, 783802899, 2157090897, 2278711709, 3001078924, - 1223320630, 3246830042, 1197195551, 4062534962, 1039971013, 2342241593, 3551623946, - 63853850, 3585779636, 81127429, 3549929990, 2886997195, 1335910186, 4169018554, - 1671155, 472949658, 148656515, 3600963048, 3772042754, 1601854390, 471598090, - 2013359012, 1181513377, 321630853, 736452516, 100585026, 1154373750, 2323956092, - 1926754199, 192349301, 3804215437, 909027311, 2993842723, 1138131968, 1319305666, - 3858990362, 3155018279, 2288945270, 2489094664, 228734829, 577956164, 2078807284, - 1924581773, 2678967510, 112604551, 3865074232, 3776350052, 2127459222, 626049886, - 910887552, 4127297396, 851240323, 547797457, 2142891352, 1154068086, 639126620, - 2501600773, 2391654498, 3342870442, 80596481, 126970446, 2184239961, 824575673, - 3399719246, 1087028487, 2905348107, 2502009404, 740197255, 14860817, 2201723284, - 3908202640, 754508313, 4133980283, 90540547, 2575809693, 4172462501, 2322897687, - 831222037, 56239065, 3575249683, 2939319477, 901605762, 1998632674, 412461711, - 2992300101, 1513271778, 2797164148, 969149327, 3660190396, 2739211008, 1954453463, - 3279391005, 1267183547, 4193503742, 1291505093, 2884603001, 564097935, 3378471970, - 2124229268, 1263126330, 860214490, 2168366043, 867190357, 4086980935, 2902522885, - 554804421, 1571065517, 497580674, 1821678887, 126415290, 980853251, 1531963815, - 2259090956, 2099629264, 2031491001, 2205009469, 1046577915, 30458798, 3447807375, - 1090142292, 1667375106, 2522840205, 748518306, 2255362927, 91588630, 3122782857, - 929883614, 1016302700, 4135967848, 1246663706, 3500613893, 4200173807, 4149573092, - 4182379872, 229752655, 1011692880, 2791828564, 2890696349, 397062412, 1754509337, - 2321492983, 3512328605, 3005148093, 1028843071, 434175992, 3155484007, 2306406482, - 1417194283, 3217107401, 1521633702, 2366802562, 399245037, 665389310, 76017436, - 3231711673, 3784114896, 1927919696, 37004463, 2157090897, 2278711709, 3001078924, - 1223320630, 1281902891, 1197195551, 4062534962, 1039971013, 2342241593, 836721481, - 63853850, 3585779636, 81127429, 3549929990, 2541553478, 1335910186, 4169018554, - 1671155, 472949658, 2086411677, 3600963048, 3772042754, 1601854390, 471598090, - 3297781744, 1181513377, 321630853, 736452516, 100585026, 2296508711, 2323956092, - 1926754199, 192349301, 3804215437, 314399580, 2993842723, 1138131968, 1319305666, - 3858990362, 584746730, 2288945270, 2489094664, 228734829, 577956164, 3868048239, - 1924581773, 2678967510, 112604551, 3865074232, 2091950990, 2127459222, 626049886, - 910887552, 4127297396, 2494071916, 547797457, 2142891352, 1154068086, 639126620, - 1159991153, 2391654498, 3342870442, 80596481, 126970446, 2276453681, 824575673, - 3399719246, 1087028487, 2905348107, 874278393, 740197255, 14860817, 2201723284, - 3908202640, 1189317351, 4133980283, 90540547, 2575809693, 4172462501, 746169572, - 831222037, 56239065, 3575249683, 2939319477, 4148988439, 1998632674, 412461711, - 2992300101, 1513271778, 1078781767, 969149327, 3660190396, 2739211008, 1954453463, - 369522045, 1267183547, 4193503742, 1291505093, 2884603001, 2820350438, 3378471970, - 2124229268, 1263126330, 860214490, 793306335, 867190357, 4086980935, 2902522885, - 554804421, 1472297125, 497580674, 1821678887, 126415290, 980853251, 1628231485, - 2259090956, 2099629264, 2031491001, 2205009469, 2562996945, 30458798, 3447807375, - 1090142292, 1667375106, 3513508401, 748518306, 2255362927, 91588630, 3122782857, - 435869165, 1016302700, 4135967848, 1246663706, 3500613893, 4156110437, 4149573092, - 4182379872, 229752655, 1011692880, 1150278253, 2890696349, 397062412, 1754509337, - 2321492983, 1126835971, 3005148093, 1028843071, 434175992, 3155484007, 4169948411, - 1417194283, 3217107401, 1521633702, 2366802562, 1629830655, 665389310, 76017436, - 3231711673, 3784114896, 2523153991, 37004463, 2157090897, 2278711709, 3001078924, - 3770048208, 1281902891, 1197195551, 4062534962, 1039971013, 2710590100, 836721481, - 63853850, 3585779636, 81127429, 3850118466, 1883009417, 1027645619, 2766570701, - 529436174, 4182542040, 2027954186, 1551970336, 2476537298, 1601343216, 3847258834, - 14764974, 2173280370, 4148127270, 2818930089, 4238274314, 1291010651, 276452076, - 192067464, 4086351393, 37573517, 48008720, 1641547972, 3144774960, 2159884108, - 4260412239, 4072883650, 801704944, 2475958420, 2719220408, 555871884, 3338968445, - 1704817873, 1960791083, 3785650808, 948722806, 3591229899, 1776225011, 4086658524, - 2675451845, 308053697, 3514232055, 2575301108, 1970226110, 3926325352, 770275431, - 1432667716, 671201644, 1008866625, 1151827040, 11061406, 3492749345, 2398090284, - 2479688660, 2275263177, 2452696627, 3239880878, 3206200433, 1520851097, 1517432473, - 1468198490, 1756343506, 2477348626, 3684701600, 3173720911, 1034531154, 4092116810, - 3546516359, 2085136160, 643024588, 1462240654, 1877398196, 3615581878, 1419408410, - 3581360976, 1731324772, 1377343320, 3848152825, 2213533588, 2484549569, 2043594863, - 224490427, 1298974897, 4279011954, 3970331393, 3795364604, 285230552, 2893090686, - 2399312639, 2638905215, 3481427245, 3477537504, 2609821731, 867675919, 3395750357, - 1969593211, 2390932014, 3164333009, 3032345429, 3054196992, 1655295657, 193598641, - 1267960637, 1599091894, 3377410805, 1529073346, 1949183620, 1575927573, 1493246650, - 2285478895, 797817618, 1736047766, 1537439339, 1422940895, 2210817855, 2888194544, - 800138109, 1689425315, 87966703, 3800446188, 137301285, 3334431104, 1776710491, - 4010349050, 2577018472, 3083459223, 672158271, 3379478560, 2445459713, 918903140, - 2577376693, 273150303, 2300393435, 3529750006, 3941920515, 2590879584, 2005940914, - 2533952036, 2918638361, 1907638097, 959011520, 1477207871, 2141548481, 2065858781, - 3145892196, 3679867589, 1295127682, 1325838381, 3482593404, 1212565985, 3404887017, - 709111097, 1714185234, 561489165, 3545430079, 359778601, 3034684349, 2235482356, - 2263913966, 1397371482, - }, - { - 170295791, 2753410803, 2200994594, 14686027, 3460333923, 1523230564, 393272614, - 1632665034, 2139771608, 2436912103, 375335282, 667585308, 3651645415, 1403132103, - 4146144245, 786890392, 1349234364, 1278024517, 84921263, 3758850381, 4213552796, - 2355655048, 1636349912, 172797504, 2490691729, 1233059003, 2593048824, 942056581, - 953415060, 4250104075, 787552244, 1995239637, 2482815609, 767530774, 773778243, - 841396894, 2718419035, 3363828032, 737774143, 4128182656, 2335090807, 1421795969, - 2322011430, 2808330380, 2207840656, 1646731611, 492284258, 2339383764, 3439685708, - 2316859204, 4055048437, 1700143892, 2980557654, 1353917552, 548777318, 1077538998, - 2650679367, 2853583947, 2721899692, 4253535213, 3375043688, 3489699354, 2401362855, - 3391605246, 914273272, 3060460082, 1409014396, 3313834796, 461914731, 82334736, - 3200344474, 2743316601, 842885927, 613943741, 96056919, 3116963503, 305659983, - 132158360, 239064402, 849530381, 543215927, 4250983939, 2719881954, 1950301886, - 2760008207, 853237881, 3875675156, 1753566841, 1446648300, 1663885236, 2155720472, - 1902508987, 4246118829, 383661834, 2420221467, 156828838, 2919782856, 499968148, - 2538550321, 65231340, 1589837081, 3654438263, 467304037, 1000159563, 622643461, - 1410713407, 491953742, 1003597552, 1972701846, 1534343952, 1934888620, 4214562113, - 4154375443, 3612899079, 2132948514, 2599819225, 2676649952, 3147375990, 533258319, - 3323553423, 4203909276, 668602384, 3979162921, 2360530772, 162121513, 8968884, - 3647746035, 2830313226, 1736955603, 78142012, 1643270604, 1571637938, 4065571991, - 2071640825, 2715113082, 3826814783, 1067370024, 1810581550, 2354204343, 3798962263, - 1664654967, 3740539785, 3746164996, 4280983219, 3313400832, 3305556349, 4226011346, - 839676594, 1785445494, 1248107478, 904240268, 3484988721, 2290931247, 2109493967, - 3895901626, 1494555863, 3251796061, 40877237, 2914051470, 2810210896, 1428826975, - 170295791, 2753410803, 2200994594, 14686027, 3263438011, 1523230564, 393272614, - 1632665034, 2139771608, 1847095655, 375335282, 667585308, 3651645415, 1403132103, - 1888152231, 786890392, 1349234364, 1278024517, 84921263, 317409190, 4213552796, - 2355655048, 1636349912, 172797504, 891435579, 1233059003, 2593048824, 942056581, - 953415060, 1606837225, 787552244, 1995239637, 2482815609, 767530774, 723338833, - 841396894, 2718419035, 3363828032, 737774143, 1043554448, 2335090807, 1421795969, - 2322011430, 2808330380, 2754923978, 1646731611, 492284258, 2339383764, 3439685708, - 3985616488, 4055048437, 1700143892, 2980557654, 1353917552, 588678041, 1077538998, - 2650679367, 2853583947, 2721899692, 992549416, 3375043688, 3489699354, 2401362855, - 3391605246, 2111206241, 3060460082, 1409014396, 3313834796, 461914731, 749987143, - 3200344474, 2743316601, 842885927, 613943741, 1572013294, 3116963503, 305659983, - 132158360, 239064402, 2802105766, 543215927, 4250983939, 2719881954, 1950301886, - 1025784309, 853237881, 3875675156, 1753566841, 1446648300, 2265992307, 2155720472, - 1902508987, 4246118829, 383661834, 1291267638, 156828838, 2919782856, 499968148, - 2538550321, 2108151330, 1589837081, 3654438263, 467304037, 1000159563, 611554173, - 1410713407, 491953742, 1003597552, 1972701846, 1548061756, 1934888620, 4214562113, - 4154375443, 3612899079, 3599839935, 2599819225, 2676649952, 3147375990, 533258319, - 4213499273, 4203909276, 668602384, 3979162921, 2360530772, 197252548, 8968884, - 3647746035, 2830313226, 1736955603, 791687787, 1643270604, 1571637938, 4065571991, - 2071640825, 2026290282, 3826814783, 1067370024, 1810581550, 2354204343, 2679791787, - 1664654967, 3740539785, 3746164996, 4280983219, 1690075221, 3305556349, 4226011346, - 839676594, 1785445494, 935893161, 904240268, 3484988721, 2290931247, 2109493967, - 1497667362, 1494555863, 3251796061, 40877237, 2914051470, 1936503212, 1428826975, - 170295791, 2753410803, 2200994594, 3416506072, 3263438011, 1523230564, 393272614, - 1632665034, 3223475136, 1847095655, 375335282, 667585308, 3651645415, 1628711405, - 1888152231, 786890392, 1349234364, 1278024517, 3955811679, 317409190, 4213552796, - 2355655048, 1636349912, 2875036620, 891435579, 1233059003, 2593048824, 942056581, - 2852399035, 1606837225, 787552244, 1995239637, 2482815609, 3849697041, 723338833, - 841396894, 2718419035, 3363828032, 2914796626, 1043554448, 2335090807, 1421795969, - 2322011430, 1088985845, 2754923978, 1646731611, 492284258, 2339383764, 2345741058, - 3985616488, 4055048437, 1700143892, 2980557654, 3047950756, 588678041, 1077538998, - 2650679367, 2853583947, 1902113580, 992549416, 3375043688, 3489699354, 2401362855, - 2822431025, 2111206241, 3060460082, 1409014396, 3313834796, 3661696135, 749987143, - 3200344474, 2743316601, 842885927, 3902266797, 1572013294, 3116963503, 305659983, - 132158360, 2399116869, 2802105766, 543215927, 4250983939, 2719881954, 1909593430, - 1025784309, 853237881, 3875675156, 1753566841, 315928539, 2265992307, 2155720472, - 1902508987, 4246118829, 4054781820, 1291267638, 156828838, 2919782856, 499968148, - 2746436642, 2108151330, 1589837081, 3654438263, 467304037, 2376244866, 611554173, - 1410713407, 491953742, 1003597552, 961109680, 1548061756, 1934888620, 4214562113, - 4154375443, 3318608531, 3599839935, 2599819225, 2676649952, 3147375990, 3197943734, - 4213499273, 4203909276, 668602384, 3979162921, 4241359084, 197252548, 8968884, - 3647746035, 2830313226, 2057817762, 791687787, 1643270604, 1571637938, 4065571991, - 961587641, 2026290282, 3826814783, 1067370024, 1810581550, 1525669339, 2679791787, - 1664654967, 3740539785, 3746164996, 3971185743, 1690075221, 3305556349, 4226011346, - 839676594, 4017546432, 935893161, 904240268, 3484988721, 2290931247, 2887434676, - 1497667362, 1494555863, 3251796061, 40877237, 675451622, 1936503212, 1428826975, - 170295791, 2753410803, 13691728, 3416506072, 3263438011, 1523230564, 393272614, - 2875584734, 3223475136, 1847095655, 375335282, 667585308, 192306502, 1628711405, - 1888152231, 786890392, 1349234364, 511851370, 3955811679, 317409190, 4213552796, - 2355655048, 131052067, 2875036620, 891435579, 1233059003, 2593048824, 2915307792, - 2852399035, 1606837225, 787552244, 1995239637, 886016481, 3849697041, 723338833, - 841396894, 2718419035, 1765948302, 2914796626, 1043554448, 2335090807, 1421795969, - 4270899906, 1088985845, 2754923978, 1646731611, 492284258, 1723935335, 2345741058, - 3985616488, 4055048437, 1700143892, 2254566160, 3047950756, 588678041, 1077538998, - 2650679367, 1004539894, 1902113580, 992549416, 3375043688, 3489699354, 2030140735, - 2822431025, 2111206241, 3060460082, 1409014396, 3053214877, 3661696135, 749987143, - 3200344474, 2743316601, 398855857, 3902266797, 1572013294, 3116963503, 305659983, - 1626072332, 2399116869, 2802105766, 543215927, 4250983939, 1149058742, 1909593430, - 1025784309, 853237881, 3875675156, 2709854504, 315928539, 2265992307, 2155720472, - 1902508987, 4065691077, 4054781820, 1291267638, 156828838, 2919782856, 1004764391, - 2746436642, 2108151330, 1589837081, 3654438263, 2380382984, 2376244866, 611554173, - 1410713407, 491953742, 3149407591, 961109680, 1548061756, 1934888620, 4214562113, - 1555853416, 3318608531, 3599839935, 2599819225, 2676649952, 1902647993, 3197943734, - 4213499273, 4203909276, 668602384, 2188341510, 4241359084, 197252548, 8968884, - 3647746035, 629654524, 2057817762, 791687787, 1643270604, 1571637938, 3066487639, - 961587641, 2026290282, 3826814783, 1067370024, 2223613942, 1525669339, 2679791787, - 1664654967, 3740539785, 3902060288, 3971185743, 1690075221, 3305556349, 4226011346, - 3135081672, 4017546432, 935893161, 904240268, 3484988721, 2448752416, 2887434676, - 1497667362, 1494555863, 3251796061, 1037186927, 1608759110, 3873834254, 59242551, - 487334743, 2580513180, 3704829028, 3859157573, 3452402004, 783668920, 2394905786, - 3179497902, 2576105629, 1552362163, 2138613992, 224944469, 3876873579, 3402518289, - 1709606949, 4255868112, 1249055439, 3395879908, 2957760102, 346905231, 590629983, - 1171021480, 4051081465, 3913643946, 3115845768, 1021908139, 2556028362, 3828177651, - 2870156105, 899722025, 661756192, 3775551864, 1288569751, 3751947667, 3064664685, - 2559273148, 2660772417, 2448044253, 3054357327, 3434913868, 1444728572, 3010819186, - 3010362527, 1709131033, 3425689752, 2849921358, 3518017065, 3845809665, 3245724553, - 1008739837, 3274032925, 2567688974, 1981389077, 1108638127, 470206543, 1097339633, - 1714430226, 2321268672, 1149373331, 294569671, 4264586290, 4270574127, 2522456947, - 230975563, 131504269, 541738544, 1380704847, 2946408074, 282744860, 246858261, - 2037373985, 1769191691, 2174871838, 2097427065, 492251656, 1252290304, 3616248100, - 3213248383, 1847973756, 647347869, 3015847616, 299045987, 866593289, 2009367463, - 2448831631, 337965200, 1210654808, 1694878225, 853507918, 3373825966, 4262812941, - 4279525028, 338822858, 1038097567, 3996799911, 755960212, 149304151, 1599868486, - 4021605447, 3040297322, 3891899828, 1711866076, 900840696, 3675688669, 3070862438, - 2611308185, 2359948129, 1158552196, 2094484627, 3077606843, 2119537593, 427023787, - 3632076073, 2670551310, 3396099733, 1066081183, 1817788918, 324769315, 656687887, - 202117575, 3106428593, 3730407212, 1661316263, 1215084998, 2025391552, 664352483, - 1914686594, 9439399, 2548190484, 3127972014, 4008228378, 2645735658, 2191361716, - 2211450148, 1863406291, 1179298131, 241880428, 2330159770, 3490494273, 1337382890, - 747522461, 1060348557, 3618051469, 991193538, 1604905367, 2595102954, 1460144089, - 3990194961, 44265425, 896268152, 9333748, 2850675977, 941433385, 2483544989, - 3443750079, 2488690792, - }, - { - 824297644, 239464654, 4133652405, 1611614045, 102133367, 1780659362, 114934718, - 3793050817, 3286619856, 1323742990, 3487325492, 468742651, 271433491, 3474195023, - 479173886, 3282693508, 978269731, 1826990521, 3664994445, 1943608646, 2356793330, - 2228748670, 4238523810, 2467714013, 1732683390, 2345218001, 3371637369, 1073602848, - 844797255, 3881048480, 509186599, 1399427071, 3815270778, 1505666412, 2616384981, - 2990167853, 3716581225, 3063486812, 1568307898, 3262882991, 1455926070, 3011806226, - 3803364927, 849372289, 2382885729, 3071102985, 3838244574, 3219174218, 847830757, - 1414310383, 3679389549, 1558413907, 2211822428, 339810803, 1051648907, 76928699, - 3174194320, 3920525151, 2010088097, 4111092791, 3537133983, 1701410561, 3036563175, - 4010986440, 1749862952, 159833659, 3406940095, 1041601178, 4005001553, 1663515026, - 1728511107, 1496728329, 2359970426, 530862749, 3797637507, 2550923758, 1450321218, - 21682904, 936804838, 3832989199, 3063256293, 3991708711, 986539283, 3775232150, - 2867283706, 747477232, 946349345, 1010022077, 188204104, 2526787171, 2816843760, - 1776005940, 2819738500, 1155856699, 2191793692, 3802193350, 1163036922, 645032560, - 3122679267, 3311719932, 3757073707, 2464258247, 1360425558, 387981241, 1714916540, - 411019237, 2248466094, 2878213113, 2742600760, 2763650927, 2526526309, 1093836264, - 3819986000, 3754388150, 1731831799, 1441137152, 1625850961, 1182084155, 1596226376, - 2389499892, 3923360808, 2439159233, 1623373213, 2513747479, 3651587995, 1040867254, - 4208484711, 3489019765, 2141904813, 3666280633, 970464748, 2970978888, 1376163015, - 1218588624, 2721249823, 707915046, 4262557484, 3237019195, 744279211, 364567144, - 1997174860, 3215512870, 2758022574, 2677818352, 4198422061, 3016017869, 2243997977, - 1029293722, 1820056287, 1090825999, 4135403724, 299239527, 874620372, 2995368704, - 3219627293, 2431393692, 3470601754, 1809177571, 37446335, 1619184385, 675901368, - 824297644, 239464654, 4133652405, 1611614045, 1918718045, 1780659362, 114934718, - 3793050817, 3286619856, 3566342809, 3487325492, 468742651, 271433491, 3474195023, - 77797025, 3282693508, 978269731, 1826990521, 3664994445, 1455182612, 2356793330, - 2228748670, 4238523810, 2467714013, 1081984526, 2345218001, 3371637369, 1073602848, - 844797255, 4125413817, 509186599, 1399427071, 3815270778, 1505666412, 891823593, - 2990167853, 3716581225, 3063486812, 1568307898, 1753181930, 1455926070, 3011806226, - 3803364927, 849372289, 4211525266, 3071102985, 3838244574, 3219174218, 847830757, - 774013898, 3679389549, 1558413907, 2211822428, 339810803, 2282783575, 76928699, - 3174194320, 3920525151, 2010088097, 3894905215, 3537133983, 1701410561, 3036563175, - 4010986440, 676262036, 159833659, 3406940095, 1041601178, 4005001553, 3470687799, - 1728511107, 1496728329, 2359970426, 530862749, 3081565689, 2550923758, 1450321218, - 21682904, 936804838, 951873872, 3063256293, 3991708711, 986539283, 3775232150, - 487381835, 747477232, 946349345, 1010022077, 188204104, 2898848241, 2816843760, - 1776005940, 2819738500, 1155856699, 2432683643, 3802193350, 1163036922, 645032560, - 3122679267, 22749078, 3757073707, 2464258247, 1360425558, 387981241, 3652130062, - 411019237, 2248466094, 2878213113, 2742600760, 811608089, 2526526309, 1093836264, - 3819986000, 3754388150, 415809552, 1441137152, 1625850961, 1182084155, 1596226376, - 202609936, 3923360808, 2439159233, 1623373213, 2513747479, 4149563237, 1040867254, - 4208484711, 3489019765, 2141904813, 718806958, 970464748, 2970978888, 1376163015, - 1218588624, 2307367700, 707915046, 4262557484, 3237019195, 744279211, 1876395939, - 1997174860, 3215512870, 2758022574, 2677818352, 2276158677, 3016017869, 2243997977, - 1029293722, 1820056287, 3605618012, 4135403724, 299239527, 874620372, 2995368704, - 872126519, 2431393692, 3470601754, 1809177571, 37446335, 2365355125, 675901368, - 824297644, 239464654, 4133652405, 8139161, 1918718045, 1780659362, 114934718, - 3793050817, 2424418256, 3566342809, 3487325492, 468742651, 271433491, 542129690, - 77797025, 3282693508, 978269731, 1826990521, 2963435579, 1455182612, 2356793330, - 2228748670, 4238523810, 2373300657, 1081984526, 2345218001, 3371637369, 1073602848, - 2948610237, 4125413817, 509186599, 1399427071, 3815270778, 2870251133, 891823593, - 2990167853, 3716581225, 3063486812, 2347504584, 1753181930, 1455926070, 3011806226, - 3803364927, 3956554065, 4211525266, 3071102985, 3838244574, 3219174218, 2018597841, - 774013898, 3679389549, 1558413907, 2211822428, 56072605, 2282783575, 76928699, - 3174194320, 3920525151, 268031035, 3894905215, 3537133983, 1701410561, 3036563175, - 366935627, 676262036, 159833659, 3406940095, 1041601178, 4125224603, 3470687799, - 1728511107, 1496728329, 2359970426, 3570997128, 3081565689, 2550923758, 1450321218, - 21682904, 604517910, 951873872, 3063256293, 3991708711, 986539283, 2414780630, - 487381835, 747477232, 946349345, 1010022077, 3820353604, 2898848241, 2816843760, - 1776005940, 2819738500, 1192624235, 2432683643, 3802193350, 1163036922, 645032560, - 4050277201, 22749078, 3757073707, 2464258247, 1360425558, 1933406988, 3652130062, - 411019237, 2248466094, 2878213113, 37869698, 811608089, 2526526309, 1093836264, - 3819986000, 3999750910, 415809552, 1441137152, 1625850961, 1182084155, 1186617400, - 202609936, 3923360808, 2439159233, 1623373213, 4226729056, 4149563237, 1040867254, - 4208484711, 3489019765, 3728140516, 718806958, 970464748, 2970978888, 1376163015, - 1307011711, 2307367700, 707915046, 4262557484, 3237019195, 4014387080, 1876395939, - 1997174860, 3215512870, 2758022574, 1696763772, 2276158677, 3016017869, 2243997977, - 1029293722, 1444214949, 3605618012, 4135403724, 299239527, 874620372, 1524158085, - 872126519, 2431393692, 3470601754, 1809177571, 163166369, 2365355125, 675901368, - 824297644, 239464654, 1626558353, 8139161, 1918718045, 1780659362, 114934718, - 1885224714, 2424418256, 3566342809, 3487325492, 468742651, 1101039917, 542129690, - 77797025, 3282693508, 978269731, 3659653445, 2963435579, 1455182612, 2356793330, - 2228748670, 539062188, 2373300657, 1081984526, 2345218001, 3371637369, 2825652803, - 2948610237, 4125413817, 509186599, 1399427071, 3197034620, 2870251133, 891823593, - 2990167853, 3716581225, 3773712182, 2347504584, 1753181930, 1455926070, 3011806226, - 3260276773, 3956554065, 4211525266, 3071102985, 3838244574, 201639236, 2018597841, - 774013898, 3679389549, 1558413907, 2830702673, 56072605, 2282783575, 76928699, - 3174194320, 1677734845, 268031035, 3894905215, 3537133983, 1701410561, 4240866153, - 366935627, 676262036, 159833659, 3406940095, 4245889153, 4125224603, 3470687799, - 1728511107, 1496728329, 3650277906, 3570997128, 3081565689, 2550923758, 1450321218, - 3392011930, 604517910, 951873872, 3063256293, 3991708711, 2876003834, 2414780630, - 487381835, 747477232, 946349345, 982266944, 3820353604, 2898848241, 2816843760, - 1776005940, 3677715064, 1192624235, 2432683643, 3802193350, 1163036922, 1226669337, - 4050277201, 22749078, 3757073707, 2464258247, 4197532785, 1933406988, 3652130062, - 411019237, 2248466094, 3209426720, 37869698, 811608089, 2526526309, 1093836264, - 535856568, 3999750910, 415809552, 1441137152, 1625850961, 2181491119, 1186617400, - 202609936, 3923360808, 2439159233, 1823827533, 4226729056, 4149563237, 1040867254, - 4208484711, 1101917521, 3728140516, 718806958, 970464748, 2970978888, 1574663259, - 1307011711, 2307367700, 707915046, 4262557484, 2164217930, 4014387080, 1876395939, - 1997174860, 3215512870, 1335157953, 1696763772, 2276158677, 3016017869, 2243997977, - 324788481, 1444214949, 3605618012, 4135403724, 299239527, 4190629945, 1524158085, - 872126519, 2431393692, 3470601754, 3701018280, 671547257, 4029965023, 1026428282, - 1584875796, 3537698406, 3731126476, 2419795330, 993551117, 2126319514, 3557113304, - 1014047757, 1407120210, 1977537539, 1338958570, 3249585389, 3661503659, 4240815680, - 1866933898, 3205442033, 4247144816, 1422846419, 3847421981, 1383632066, 3589322376, - 1816906043, 1310944471, 3646822098, 799529013, 3350558751, 2552899295, 4281235599, - 4069668296, 4123814877, 3289565353, 1512974699, 111908081, 2535556715, 333570815, - 3638041929, 1942569446, 20945397, 3784826827, 200406456, 2640512138, 38390336, - 436784052, 3062106345, 1675333627, 709613078, 3479720979, 2726065658, 4072312748, - 797389139, 3492082903, 3792395750, 983473383, 2984788349, 2030282907, 2246686378, - 2451087141, 1799640566, 2182694041, 3226819076, 3573153299, 3658670545, 1197013516, - 777601408, 4271704548, 1192713934, 1628497069, 681025927, 4078910773, 619496169, - 1534725146, 1881987408, 2283881479, 1090218673, 4169123978, 2352195985, 2640116078, - 3869558100, 2859177954, 3329803656, 4048903941, 1636589748, 2095007175, 4169840880, - 2953611537, 2413740464, 3029624235, 778662441, 422412779, 412103280, 1701569571, - 564088645, 469973310, 254302146, 3963642101, 555781470, 2983576224, 1757897888, - 1420763962, 2176323176, 916790568, 3057610889, 196828641, 1435167402, 325046353, - 1337309066, 2691769282, 3572566918, 2910149226, 3659418019, 2511762503, 3869838339, - 1413312151, 1939339596, 801124461, 760477862, 2416958233, 3439465675, 3561763524, - 1760392811, 1582406751, 1203071257, 755811399, 2675585013, 1150664766, 3515765747, - 3419135844, 2076543342, 1191918544, 3644819073, 2195875022, 2909071148, 3385707813, - 1151273265, 1467337419, 3570589492, 3742049917, 1609858615, 2964509119, 3747960348, - 2825858640, 101501715, 1234710482, 750428334, 2870070395, 416615350, 4054039387, - 3807926874, 3035407103, 1644560291, 2490941295, 963796562, 3233132139, 2590859502, - 2845243609, 964355909, - }, - { - 2882980002, 2211288683, 872766101, 3713771728, 1429983118, 2069599564, 827699420, - 1288565883, 2985727214, 3873174741, 2138389854, 3915615927, 2759028650, 3120611541, - 385953581, 189931252, 2044235060, 4214733958, 1899137741, 1973215178, 494148492, - 1550568689, 3646957712, 3764784141, 1114556979, 1411407684, 1194906295, 1718808623, - 1809627046, 1413570172, 180837718, 2588730975, 1481586714, 2836300053, 1967135375, - 4010897189, 3392273121, 3466021198, 1182364160, 1364130321, 1795412556, 330320182, - 1165093128, 2125767818, 904192995, 51833064, 232302906, 1834422179, 476731510, - 3484170517, 2373156680, 2610500049, 1688364249, 463611489, 3759685710, 62038708, - 2357334250, 1230002441, 520303451, 3009758047, 1882263827, 2524779298, 1736323157, - 3883037541, 1103650182, 1137565179, 3112310886, 3524287283, 3064002681, 4106308847, - 3180534967, 2463036338, 1859639515, 1319061987, 354419222, 4108171950, 601260554, - 705389180, 4081137445, 3461353436, 399768111, 3963945521, 2094962544, 630762046, - 369047181, 3495709267, 3525452874, 314919391, 2152657907, 881476500, 3565507827, - 2594931381, 579458905, 1767988684, 2678728511, 3416503939, 4150612567, 1015748208, - 2059142720, 2725183490, 2998421769, 1644667445, 4221112143, 456578131, 3881530201, - 190710543, 1721255927, 2274887963, 187713135, 2209254952, 2185750138, 2992229399, - 482133467, 2758198810, 15147949, 536333711, 2296185346, 1103433779, 1573407789, - 1357843567, 2927153963, 4157295398, 533935893, 3567030810, 1900900411, 509578395, - 3810017456, 2134110040, 3347323570, 3497032747, 201278263, 3933249682, 3849960474, - 2509123202, 3445521167, 1355284593, 2444811561, 2751112324, 1116246614, 511213077, - 3412599909, 1712118363, 54054007, 442729047, 3077267414, 1532701769, 181534938, - 1278069867, 3847149992, 2305860479, 4146252420, 2047690303, 361856758, 452490341, - 636885000, 1733216839, 3788548638, 1094285639, 1349356222, 2760444511, 976767752, - 2882980002, 2211288683, 872766101, 3713771728, 895830110, 2069599564, 827699420, - 1288565883, 2985727214, 3377496544, 2138389854, 3915615927, 2759028650, 3120611541, - 3254971483, 189931252, 2044235060, 4214733958, 1899137741, 2095055586, 494148492, - 1550568689, 3646957712, 3764784141, 2869825005, 1411407684, 1194906295, 1718808623, - 1809627046, 907760376, 180837718, 2588730975, 1481586714, 2836300053, 639229964, - 4010897189, 3392273121, 3466021198, 1182364160, 3006792787, 1795412556, 330320182, - 1165093128, 2125767818, 253264555, 51833064, 232302906, 1834422179, 476731510, - 4284481518, 2373156680, 2610500049, 1688364249, 463611489, 4133115610, 62038708, - 2357334250, 1230002441, 520303451, 1497001150, 1882263827, 2524779298, 1736323157, - 3883037541, 3541909847, 1137565179, 3112310886, 3524287283, 3064002681, 3193060438, - 3180534967, 2463036338, 1859639515, 1319061987, 111871878, 4108171950, 601260554, - 705389180, 4081137445, 742999102, 399768111, 3963945521, 2094962544, 630762046, - 3219207950, 3495709267, 3525452874, 314919391, 2152657907, 720863934, 3565507827, - 2594931381, 579458905, 1767988684, 3958525287, 3416503939, 4150612567, 1015748208, - 2059142720, 4227838648, 2998421769, 1644667445, 4221112143, 456578131, 302729329, - 190710543, 1721255927, 2274887963, 187713135, 1293706587, 2185750138, 2992229399, - 482133467, 2758198810, 2514965671, 536333711, 2296185346, 1103433779, 1573407789, - 2237639577, 2927153963, 4157295398, 533935893, 3567030810, 3793156627, 509578395, - 3810017456, 2134110040, 3347323570, 1358364, 201278263, 3933249682, 3849960474, - 2509123202, 628476542, 1355284593, 2444811561, 2751112324, 1116246614, 3421170828, - 3412599909, 1712118363, 54054007, 442729047, 325825294, 1532701769, 181534938, - 1278069867, 3847149992, 2785457372, 4146252420, 2047690303, 361856758, 452490341, - 1099532083, 1733216839, 3788548638, 1094285639, 1349356222, 3047068265, 976767752, - 2882980002, 2211288683, 872766101, 366378371, 895830110, 2069599564, 827699420, - 1288565883, 962962884, 3377496544, 2138389854, 3915615927, 2759028650, 3742489931, - 3254971483, 189931252, 2044235060, 4214733958, 3073407497, 2095055586, 494148492, - 1550568689, 3646957712, 758370067, 2869825005, 1411407684, 1194906295, 1718808623, - 636166267, 907760376, 180837718, 2588730975, 1481586714, 705382583, 639229964, - 4010897189, 3392273121, 3466021198, 3815622040, 3006792787, 1795412556, 330320182, - 1165093128, 2956382339, 253264555, 51833064, 232302906, 1834422179, 3665645898, - 4284481518, 2373156680, 2610500049, 1688364249, 2565987890, 4133115610, 62038708, - 2357334250, 1230002441, 2397198293, 1497001150, 1882263827, 2524779298, 1736323157, - 817630445, 3541909847, 1137565179, 3112310886, 3524287283, 1356492703, 3193060438, - 3180534967, 2463036338, 1859639515, 3963974342, 111871878, 4108171950, 601260554, - 705389180, 1776439965, 742999102, 399768111, 3963945521, 2094962544, 2007137733, - 3219207950, 3495709267, 3525452874, 314919391, 3877039785, 720863934, 3565507827, - 2594931381, 579458905, 2919403199, 3958525287, 3416503939, 4150612567, 1015748208, - 960765392, 4227838648, 2998421769, 1644667445, 4221112143, 2402062799, 302729329, - 190710543, 1721255927, 2274887963, 3958481548, 1293706587, 2185750138, 2992229399, - 482133467, 3838280, 2514965671, 536333711, 2296185346, 1103433779, 3675282065, - 2237639577, 2927153963, 4157295398, 533935893, 4172021805, 3793156627, 509578395, - 3810017456, 2134110040, 3608998517, 1358364, 201278263, 3933249682, 3849960474, - 2445690023, 628476542, 1355284593, 2444811561, 2751112324, 507378026, 3421170828, - 3412599909, 1712118363, 54054007, 770634305, 325825294, 1532701769, 181534938, - 1278069867, 4055596097, 2785457372, 4146252420, 2047690303, 361856758, 3439427065, - 1099532083, 1733216839, 3788548638, 1094285639, 1633234274, 3047068265, 976767752, - 2882980002, 2211288683, 3763615153, 366378371, 895830110, 2069599564, 827699420, - 2457443913, 962962884, 3377496544, 2138389854, 3915615927, 3290989016, 3742489931, - 3254971483, 189931252, 2044235060, 4275822963, 3073407497, 2095055586, 494148492, - 1550568689, 1043420085, 758370067, 2869825005, 1411407684, 1194906295, 676378812, - 636166267, 907760376, 180837718, 2588730975, 2971715054, 705382583, 639229964, - 4010897189, 3392273121, 795184546, 3815622040, 3006792787, 1795412556, 330320182, - 1990804460, 2956382339, 253264555, 51833064, 232302906, 836875615, 3665645898, - 4284481518, 2373156680, 2610500049, 98106795, 2565987890, 4133115610, 62038708, - 2357334250, 2761212145, 2397198293, 1497001150, 1882263827, 2524779298, 2381031747, - 817630445, 3541909847, 1137565179, 3112310886, 2501374726, 1356492703, 3193060438, - 3180534967, 2463036338, 3671733096, 3963974342, 111871878, 4108171950, 601260554, - 1017043724, 1776439965, 742999102, 399768111, 3963945521, 2177838102, 2007137733, - 3219207950, 3495709267, 3525452874, 3254054416, 3877039785, 720863934, 3565507827, - 2594931381, 1994293489, 2919403199, 3958525287, 3416503939, 4150612567, 1976960210, - 960765392, 4227838648, 2998421769, 1644667445, 2896792687, 2402062799, 302729329, - 190710543, 1721255927, 2914584080, 3958481548, 1293706587, 2185750138, 2992229399, - 810756083, 3838280, 2514965671, 536333711, 2296185346, 1776509588, 3675282065, - 2237639577, 2927153963, 4157295398, 2048779551, 4172021805, 3793156627, 509578395, - 3810017456, 3042185034, 3608998517, 1358364, 201278263, 3933249682, 3551449718, - 2445690023, 628476542, 1355284593, 2444811561, 3480611728, 507378026, 3421170828, - 3412599909, 1712118363, 1268921331, 770634305, 325825294, 1532701769, 181534938, - 2645357587, 4055596097, 2785457372, 4146252420, 2047690303, 1994855609, 3439427065, - 1099532083, 1733216839, 3788548638, 3516588243, 4058132193, 3940172101, 4043964688, - 3377150021, 1381463736, 3320280180, 931260821, 2754727582, 1286176949, 1661126244, - 2301263887, 2255977851, 1122646603, 1767549201, 162324152, 425506096, 3777762686, - 13687528, 710105607, 1092739920, 2930179533, 568855389, 2476208631, 964360978, - 2011445117, 3887128674, 2799005525, 2479086439, 814368438, 2018629666, 909662384, - 231589584, 1422241284, 4035938208, 3570985552, 660700421, 603857869, 567385627, - 3232044670, 291307502, 947817625, 3466590280, 3080261993, 947835229, 2925888682, - 1817591844, 2652420575, 4150903445, 4055627313, 1715025966, 505331227, 1863531052, - 2928506098, 947547681, 1117344443, 781457023, 607542746, 241559360, 3797150797, - 105381589, 361541961, 3393121650, 3840152184, 2873171161, 3030026082, 1115171192, - 1718221281, 96787532, 2556617898, 1237726058, 2876298621, 1052881200, 461661595, - 2632346030, 1775614319, 2454951319, 3691637824, 4018448825, 1610472965, 3076493165, - 1364200430, 2011206580, 1066672050, 706141458, 2064189273, 346938484, 2964350202, - 3731612957, 2506635528, 2007045393, 3312126930, 2602035453, 988876930, 2960173442, - 559685520, 2719943441, 891699839, 1151651090, 1223301894, 3666960271, 1330825927, - 1681770552, 38877327, 3803211467, 4000053051, 3552560459, 3510286057, 2606732870, - 721190747, 1933504723, 3110735238, 2333178561, 1577381363, 595257962, 4120745072, - 960219089, 2591080970, 3354222743, 47827627, 3759509914, 304815919, 2643673615, - 1381570381, 2103367217, 2440936991, 2376721005, 1483630814, 3137202706, 3075255640, - 1743649605, 3649754571, 2550788713, 4281983459, 904183710, 4243944530, 2742129811, - 3363501626, 3670239155, 4233018118, 2615012385, 1420298161, 1251344091, 2172588631, - 1243035186, 1724496237, 762022558, 8747231, 334416849, 1219880856, 187900356, - 2527057367, 1730455958, 3240238410, 906024910, 2351575735, 4207748622, 936139767, - 1984289988, 285939331, - }, - { - 4246897171, 2217508286, 4117450683, 4110626546, 3753823387, 3977667932, 623718443, - 2276396692, 3772091798, 2272323453, 710314822, 3733316262, 1497955597, 700242668, - 3582720207, 1247731879, 336477088, 532374143, 1123157198, 123828173, 272472192, - 2142741093, 2557920990, 4209595119, 2807266578, 1516814248, 4250883502, 1967663703, - 215335417, 1252724071, 4267389372, 94668579, 1980152960, 968677393, 1237744359, - 63833646, 2488747616, 700459471, 744977323, 40829823, 955400639, 37187948, - 53133706, 2014551043, 1664982537, 3342787122, 1549278321, 1245110464, 3424539081, - 2180485253, 2757636973, 3590044052, 2712703548, 1366894959, 1777449151, 1538653374, - 168718075, 2435805251, 588815465, 3166271130, 3164200096, 417809976, 623036767, - 340121872, 1792214783, 56330125, 3268029211, 1117100306, 345899179, 1547071836, - 3657965225, 4109701299, 664937685, 2627187961, 149301108, 1764003230, 3177910586, - 3081492846, 2295419724, 2553420882, 1506534805, 971284719, 3224921758, 3336906843, - 1507395478, 1224379418, 4117299702, 1973783225, 3609783242, 4186900040, 3715175536, - 3904547465, 459692505, 3546328518, 3071448159, 1300375875, 1805392236, 3072717072, - 99113127, 4281059076, 1658649136, 1974081931, 3940966682, 2092428023, 4014384840, - 1546542514, 1130620125, 4117533767, 3372991735, 3537429957, 2704347564, 2300583688, - 915286167, 1553874575, 3466388216, 701000054, 349103195, 1554395274, 3140941933, - 2874072684, 2630572105, 2794301280, 321399291, 1158058020, 3570908149, 122802750, - 3012686842, 2588402967, 3420589812, 581016671, 193235885, 1558092297, 1233353728, - 1080743465, 3292663441, 2188057155, 2715412992, 4274317234, 1657504087, 2554269340, - 1079741964, 922252155, 569761460, 3215661310, 2450710288, 2491078689, 632504591, - 2169581755, 2552457727, 2554414735, 3347573916, 681756629, 801451286, 3504956478, - 1308297539, 3602650700, 3530372129, 4117441036, 1827438812, 2852602217, 570161747, - 4246897171, 2217508286, 4117450683, 4110626546, 756072139, 3977667932, 623718443, - 2276396692, 3772091798, 3829898369, 710314822, 3733316262, 1497955597, 700242668, - 757539371, 1247731879, 336477088, 532374143, 1123157198, 2374238409, 272472192, - 2142741093, 2557920990, 4209595119, 1632439709, 1516814248, 4250883502, 1967663703, - 215335417, 1267642920, 4267389372, 94668579, 1980152960, 968677393, 2252616933, - 63833646, 2488747616, 700459471, 744977323, 2711054317, 955400639, 37187948, - 53133706, 2014551043, 1664498234, 3342787122, 1549278321, 1245110464, 3424539081, - 496150741, 2757636973, 3590044052, 2712703548, 1366894959, 2066534443, 1538653374, - 168718075, 2435805251, 588815465, 318307195, 3164200096, 417809976, 623036767, - 340121872, 3426055217, 56330125, 3268029211, 1117100306, 345899179, 979486044, - 3657965225, 4109701299, 664937685, 2627187961, 2747102301, 1764003230, 3177910586, - 3081492846, 2295419724, 1088606857, 1506534805, 971284719, 3224921758, 3336906843, - 984983218, 1224379418, 4117299702, 1973783225, 3609783242, 1044785427, 3715175536, - 3904547465, 459692505, 3546328518, 2096978494, 1300375875, 1805392236, 3072717072, - 99113127, 972796497, 1658649136, 1974081931, 3940966682, 2092428023, 2914458983, - 1546542514, 1130620125, 4117533767, 3372991735, 947968718, 2704347564, 2300583688, - 915286167, 1553874575, 2124709798, 701000054, 349103195, 1554395274, 3140941933, - 2569019225, 2630572105, 2794301280, 321399291, 1158058020, 4051601694, 122802750, - 3012686842, 2588402967, 3420589812, 1738150581, 193235885, 1558092297, 1233353728, - 1080743465, 1527068788, 2188057155, 2715412992, 4274317234, 1657504087, 1543089352, - 1079741964, 922252155, 569761460, 3215661310, 2869922986, 2491078689, 632504591, - 2169581755, 2552457727, 2807462748, 3347573916, 681756629, 801451286, 3504956478, - 3400676931, 3602650700, 3530372129, 4117441036, 1827438812, 4056234054, 570161747, - 4246897171, 2217508286, 4117450683, 3321376103, 756072139, 3977667932, 623718443, - 2276396692, 1340008665, 3829898369, 710314822, 3733316262, 1497955597, 2098292377, - 757539371, 1247731879, 336477088, 532374143, 2210327641, 2374238409, 272472192, - 2142741093, 2557920990, 3502520226, 1632439709, 1516814248, 4250883502, 1967663703, - 499168780, 1267642920, 4267389372, 94668579, 1980152960, 2695928666, 2252616933, - 63833646, 2488747616, 700459471, 4181471443, 2711054317, 955400639, 37187948, - 53133706, 441944403, 1664498234, 3342787122, 1549278321, 1245110464, 2271611585, - 496150741, 2757636973, 3590044052, 2712703548, 3009817799, 2066534443, 1538653374, - 168718075, 2435805251, 734763537, 318307195, 3164200096, 417809976, 623036767, - 4002728646, 3426055217, 56330125, 3268029211, 1117100306, 1435987728, 979486044, - 3657965225, 4109701299, 664937685, 815527474, 2747102301, 1764003230, 3177910586, - 3081492846, 63383766, 1088606857, 1506534805, 971284719, 3224921758, 2331024939, - 984983218, 1224379418, 4117299702, 1973783225, 3998070267, 1044785427, 3715175536, - 3904547465, 459692505, 2582830990, 2096978494, 1300375875, 1805392236, 3072717072, - 321154403, 972796497, 1658649136, 1974081931, 3940966682, 3789726976, 2914458983, - 1546542514, 1130620125, 4117533767, 3440681546, 947968718, 2704347564, 2300583688, - 915286167, 474021937, 2124709798, 701000054, 349103195, 1554395274, 702752814, - 2569019225, 2630572105, 2794301280, 321399291, 2406346046, 4051601694, 122802750, - 3012686842, 2588402967, 1782259321, 1738150581, 193235885, 1558092297, 1233353728, - 3935919190, 1527068788, 2188057155, 2715412992, 4274317234, 1722541048, 1543089352, - 1079741964, 922252155, 569761460, 3384000986, 2869922986, 2491078689, 632504591, - 2169581755, 3451609034, 2807462748, 3347573916, 681756629, 801451286, 2643408064, - 3400676931, 3602650700, 3530372129, 4117441036, 3635077251, 4056234054, 570161747, - 4246897171, 2217508286, 2364796923, 3321376103, 756072139, 3977667932, 623718443, - 3792539489, 1340008665, 3829898369, 710314822, 3733316262, 876419217, 2098292377, - 757539371, 1247731879, 336477088, 3307300788, 2210327641, 2374238409, 272472192, - 2142741093, 4142392723, 3502520226, 1632439709, 1516814248, 4250883502, 3551852862, - 499168780, 1267642920, 4267389372, 94668579, 1177286958, 2695928666, 2252616933, - 63833646, 2488747616, 3571573975, 4181471443, 2711054317, 955400639, 37187948, - 1485050393, 441944403, 1664498234, 3342787122, 1549278321, 518707274, 2271611585, - 496150741, 2757636973, 3590044052, 305206687, 3009817799, 2066534443, 1538653374, - 168718075, 1914032206, 734763537, 318307195, 3164200096, 417809976, 2062496275, - 4002728646, 3426055217, 56330125, 3268029211, 1878869053, 1435987728, 979486044, - 3657965225, 4109701299, 1558853775, 815527474, 2747102301, 1764003230, 3177910586, - 681877401, 63383766, 1088606857, 1506534805, 971284719, 2546285777, 2331024939, - 984983218, 1224379418, 4117299702, 539292757, 3998070267, 1044785427, 3715175536, - 3904547465, 3854154565, 2582830990, 2096978494, 1300375875, 1805392236, 2586804198, - 321154403, 972796497, 1658649136, 1974081931, 1718873863, 3789726976, 2914458983, - 1546542514, 1130620125, 477866180, 3440681546, 947968718, 2704347564, 2300583688, - 56071603, 474021937, 2124709798, 701000054, 349103195, 2431577249, 702752814, - 2569019225, 2630572105, 2794301280, 211758134, 2406346046, 4051601694, 122802750, - 3012686842, 2470642374, 1782259321, 1738150581, 193235885, 1558092297, 852353933, - 3935919190, 1527068788, 2188057155, 2715412992, 543290606, 1722541048, 1543089352, - 1079741964, 922252155, 1146820965, 3384000986, 2869922986, 2491078689, 632504591, - 2936494996, 3451609034, 2807462748, 3347573916, 681756629, 3428474076, 2643408064, - 3400676931, 3602650700, 3530372129, 3558016488, 304167301, 3073812276, 1253385329, - 801639697, 1346336854, 3880416830, 1110804934, 2500585706, 1294233475, 1964132477, - 1625651370, 2732590160, 310054807, 3350133555, 800839525, 3435579932, 2120216654, - 407780291, 1228117799, 513334510, 1423091447, 3698882838, 2556406643, 1536483608, - 998695315, 1619514015, 4197375975, 892985909, 993665758, 4160405430, 2379977763, - 1423742790, 4286808034, 479280944, 3611297256, 3481820363, 1261889958, 455298115, - 3955764756, 2406161837, 185873336, 3382956716, 3556168427, 3988426650, 2917586591, - 1248672474, 2925146191, 1416331075, 290755159, 2845168299, 3301422441, 3771816588, - 491352430, 2461746382, 1591975949, 604909111, 3595669760, 4079314041, 258321046, - 1352583874, 999018951, 3150079914, 113122510, 743303046, 3205496412, 4267738054, - 2567402806, 2181107494, 3266354249, 1941487496, 2742084900, 3758785335, 732694221, - 2052988791, 1759288229, 1094292464, 1582835026, 2817864273, 666443657, 419482443, - 2877435004, 2944696351, 2523539432, 301119182, 998264713, 2314419254, 3610447393, - 1139414242, 1486351830, 3207929489, 384633091, 4056367270, 2348418835, 3773781885, - 1963929818, 804929680, 1511023454, 3915948102, 1371942526, 2586212526, 130122933, - 2030859646, 3730011315, 118408868, 632704878, 3559959612, 2926361713, 1401386286, - 599210027, 2315051975, 157809758, 1148939942, 3060024350, 1464284678, 3209480975, - 3961060416, 3481639206, 4113344379, 3475766200, 130581501, 1844026536, 2661594012, - 3145812007, 3233175620, 2549419093, 2612966733, 1348260920, 740167863, 226231218, - 2631972701, 2148020402, 3399479414, 1074946996, 30872114, 1342415612, 1071408471, - 1141719547, 332346805, 1473336719, 4207932404, 3668838170, 3154502882, 3892070442, - 2812790310, 13931822, 1150258251, 2369539473, 640926011, 2991135002, 2410382633, - 548200125, 3977740663, 1245837867, 2378569399, 1561469990, 2437445882, 214387770, - 3329587833, 281635893, - }, - { - 1720103319, 2201367526, 1415072072, 2446588589, 2195586017, 3817930623, 653121934, - 2766514657, 765921436, 630082485, 2990883045, 3304472999, 471385134, 4097977544, - 3749829028, 3587534772, 1064359851, 800061060, 2844220510, 389838005, 3681318140, - 1515923235, 1885079324, 713031018, 1962734763, 2288160004, 1983331336, 1247350521, - 4208372034, 1444837930, 3549494305, 4169715512, 701313302, 1118275019, 3118975645, - 4153969630, 3516491181, 3601057044, 2509222288, 223064937, 899123842, 2574531231, - 1386928111, 3790651401, 1300768348, 2038833061, 3736517792, 3850203561, 1679542285, - 3391273474, 3862995487, 3118056386, 47128429, 2977525950, 3236389548, 1937040839, - 4223233198, 2105119262, 721111284, 331726226, 68419013, 2575393464, 3648293304, - 1448878851, 4186783614, 3696899986, 1270877069, 3351263117, 3918639273, 1472902162, - 2767482392, 3549853842, 2353191576, 3353325530, 3072485271, 2689121900, 2335686695, - 246689858, 2946177636, 1677728066, 1455723263, 3447540996, 2143976172, 1779511280, - 3667361203, 1575502035, 849872082, 3527265600, 1443266215, 1320668722, 458373857, - 3862342513, 699597603, 685707268, 948502001, 2501058653, 2254562046, 2210683894, - 29088679, 1456231200, 2764392560, 4138068372, 3094591474, 1093749152, 1668875176, - 3133003149, 4128702884, 652852832, 2211671337, 2231125160, 131729558, 3845605816, - 3769660625, 1696592453, 728353643, 2751201502, 3496971733, 3349166522, 1005919830, - 3411089601, 3754493523, 1994945529, 1604309774, 2083609686, 833983349, 2600153513, - 1677348112, 207321473, 1051990507, 2135039620, 4239461390, 1574144998, 1070761856, - 1990807569, 112704720, 2506523299, 2827487353, 4130754901, 1943274185, 3913701053, - 1014850621, 3662772872, 4115124063, 1760146762, 3254829227, 800302547, 3602066837, - 975658158, 2880018391, 714134831, 2696483406, 2351365577, 2811011071, 3505407160, - 54109504, 424967367, 3759525737, 1726627246, 1110539071, 2339755764, 3356877114, - 1720103319, 2201367526, 1415072072, 2446588589, 2499136377, 3817930623, 653121934, - 2766514657, 765921436, 3794433488, 2990883045, 3304472999, 471385134, 4097977544, - 3618516788, 3587534772, 1064359851, 800061060, 2844220510, 2319780070, 3681318140, - 1515923235, 1885079324, 713031018, 11705290, 2288160004, 1983331336, 1247350521, - 4208372034, 2508892029, 3549494305, 4169715512, 701313302, 1118275019, 1430522809, - 4153969630, 3516491181, 3601057044, 2509222288, 1917025539, 899123842, 2574531231, - 1386928111, 3790651401, 1219040401, 2038833061, 3736517792, 3850203561, 1679542285, - 671522957, 3862995487, 3118056386, 47128429, 2977525950, 2762831063, 1937040839, - 4223233198, 2105119262, 721111284, 1386688457, 68419013, 2575393464, 3648293304, - 1448878851, 466405406, 3696899986, 1270877069, 3351263117, 3918639273, 94103836, - 2767482392, 3549853842, 2353191576, 3353325530, 349361794, 2689121900, 2335686695, - 246689858, 2946177636, 3232050945, 1455723263, 3447540996, 2143976172, 1779511280, - 542837628, 1575502035, 849872082, 3527265600, 1443266215, 1867394883, 458373857, - 3862342513, 699597603, 685707268, 4210562190, 2501058653, 2254562046, 2210683894, - 29088679, 3647972960, 2764392560, 4138068372, 3094591474, 1093749152, 312511475, - 3133003149, 4128702884, 652852832, 2211671337, 145492343, 131729558, 3845605816, - 3769660625, 1696592453, 4223421915, 2751201502, 3496971733, 3349166522, 1005919830, - 1656802049, 3754493523, 1994945529, 1604309774, 2083609686, 3032348100, 2600153513, - 1677348112, 207321473, 1051990507, 3349078950, 4239461390, 1574144998, 1070761856, - 1990807569, 2970449178, 2506523299, 2827487353, 4130754901, 1943274185, 445467699, - 1014850621, 3662772872, 4115124063, 1760146762, 3738518624, 800302547, 3602066837, - 975658158, 2880018391, 1553758240, 2696483406, 2351365577, 2811011071, 3505407160, - 1259180427, 424967367, 3759525737, 1726627246, 1110539071, 2863575420, 3356877114, - 1720103319, 2201367526, 1415072072, 1463388387, 2499136377, 3817930623, 653121934, - 2766514657, 526940162, 3794433488, 2990883045, 3304472999, 471385134, 594057325, - 3618516788, 3587534772, 1064359851, 800061060, 1001523010, 2319780070, 3681318140, - 1515923235, 1885079324, 255576756, 11705290, 2288160004, 1983331336, 1247350521, - 1108575113, 2508892029, 3549494305, 4169715512, 701313302, 524281295, 1430522809, - 4153969630, 3516491181, 3601057044, 1816283752, 1917025539, 899123842, 2574531231, - 1386928111, 1530966640, 1219040401, 2038833061, 3736517792, 3850203561, 1855689726, - 671522957, 3862995487, 3118056386, 47128429, 1718476461, 2762831063, 1937040839, - 4223233198, 2105119262, 176166283, 1386688457, 68419013, 2575393464, 3648293304, - 4069820559, 466405406, 3696899986, 1270877069, 3351263117, 1645545933, 94103836, - 2767482392, 3549853842, 2353191576, 4163887784, 349361794, 2689121900, 2335686695, - 246689858, 1246040634, 3232050945, 1455723263, 3447540996, 2143976172, 2111249329, - 542837628, 1575502035, 849872082, 3527265600, 1836050084, 1867394883, 458373857, - 3862342513, 699597603, 3139537113, 4210562190, 2501058653, 2254562046, 2210683894, - 3997617191, 3647972960, 2764392560, 4138068372, 3094591474, 2664795910, 312511475, - 3133003149, 4128702884, 652852832, 1658020144, 145492343, 131729558, 3845605816, - 3769660625, 2822578949, 4223421915, 2751201502, 3496971733, 3349166522, 1582873482, - 1656802049, 3754493523, 1994945529, 1604309774, 1113569720, 3032348100, 2600153513, - 1677348112, 207321473, 3169983987, 3349078950, 4239461390, 1574144998, 1070761856, - 1308776367, 2970449178, 2506523299, 2827487353, 4130754901, 1403493846, 445467699, - 1014850621, 3662772872, 4115124063, 340210579, 3738518624, 800302547, 3602066837, - 975658158, 3367770843, 1553758240, 2696483406, 2351365577, 2811011071, 4162875353, - 1259180427, 424967367, 3759525737, 1726627246, 1341806135, 2863575420, 3356877114, - 1720103319, 2201367526, 2232383995, 1463388387, 2499136377, 3817930623, 653121934, - 1756183481, 526940162, 3794433488, 2990883045, 3304472999, 2185125572, 594057325, - 3618516788, 3587534772, 1064359851, 2933544964, 1001523010, 2319780070, 3681318140, - 1515923235, 4147783641, 255576756, 11705290, 2288160004, 1983331336, 956739400, - 1108575113, 2508892029, 3549494305, 4169715512, 142273913, 524281295, 1430522809, - 4153969630, 3516491181, 986032639, 1816283752, 1917025539, 899123842, 2574531231, - 1508271110, 1530966640, 1219040401, 2038833061, 3736517792, 458417668, 1855689726, - 671522957, 3862995487, 3118056386, 284266432, 1718476461, 2762831063, 1937040839, - 4223233198, 1605514069, 176166283, 1386688457, 68419013, 2575393464, 3650747541, - 4069820559, 466405406, 3696899986, 1270877069, 678590674, 1645545933, 94103836, - 2767482392, 3549853842, 398179945, 4163887784, 349361794, 2689121900, 2335686695, - 3853658293, 1246040634, 3232050945, 1455723263, 3447540996, 2657693810, 2111249329, - 542837628, 1575502035, 849872082, 2061659800, 1836050084, 1867394883, 458373857, - 3862342513, 730568629, 3139537113, 4210562190, 2501058653, 2254562046, 449510786, - 3997617191, 3647972960, 2764392560, 4138068372, 1939679536, 2664795910, 312511475, - 3133003149, 4128702884, 4057510355, 1658020144, 145492343, 131729558, 3845605816, - 3235632110, 2822578949, 4223421915, 2751201502, 3496971733, 4258920219, 1582873482, - 1656802049, 3754493523, 1994945529, 1073499993, 1113569720, 3032348100, 2600153513, - 1677348112, 3152835240, 3169983987, 3349078950, 4239461390, 1574144998, 2548972357, - 1308776367, 2970449178, 2506523299, 2827487353, 2908066033, 1403493846, 445467699, - 1014850621, 3662772872, 1685925089, 340210579, 3738518624, 800302547, 3602066837, - 2264692610, 3367770843, 1553758240, 2696483406, 2351365577, 1686022564, 4162875353, - 1259180427, 424967367, 3759525737, 70326173, 3028074555, 2568586198, 2513473964, - 2923109510, 2265392251, 3760490867, 147487099, 386755149, 2152759137, 2716532213, - 1153507474, 627929575, 847454712, 2426916452, 3861548980, 209825268, 1090299778, - 1876886461, 976019203, 4290216337, 2278290065, 3302814528, 1567440061, 1874857224, - 3794588915, 3218569451, 2335365199, 1959651923, 3366000689, 2374428382, 2126784887, - 4123272655, 274837369, 1413111935, 1754627204, 1863684635, 4170025739, 2150019850, - 4250751856, 3601214212, 2024081043, 334808859, 3921757513, 3870643644, 2864810945, - 1004431888, 4283279830, 873365350, 2479791433, 3393478881, 3373502257, 1882140107, - 2546676519, 1208428915, 268043238, 2292710623, 770651064, 2330160036, 2476488258, - 2496037992, 118721504, 2289499985, 987994743, 3610346256, 3371795927, 2681434550, - 2213200417, 3729194378, 1657623395, 402983380, 3618058500, 3487743585, 965523531, - 819256729, 2544660729, 3273986506, 60894411, 1779152929, 3598159279, 3429317853, - 2246402362, 3761392367, 3921798306, 947928110, 2394097908, 4004330264, 1180759989, - 1624349051, 1750929499, 3889184770, 2052097704, 4092981046, 2913733578, 4241980897, - 1127407450, 950788009, 2105033320, 473205730, 981905310, 2888856914, 798112239, - 3377889612, 2273659507, 1157471194, 4269212574, 3575306012, 116024754, 1432668659, - 1079598649, 3882002482, 3838480186, 823643071, 1244220618, 1227720039, 1343395654, - 4277277976, 2612321540, 3013674017, 3658064522, 2573775167, 142767236, 2545708383, - 1740478937, 809036862, 1492188594, 1294286248, 1093543858, 2944418375, 2981996479, - 4067464923, 3071157685, 1938984450, 81707323, 337713546, 1849381296, 3447450393, - 3551106302, 3394545269, 3167744716, 1815294624, 3244728913, 2462138247, 2286711732, - 3023116169, 707366723, 1314169762, 1511231537, 2227622993, 2876600706, 4271030726, - 2020521540, 2966596767, 3964589247, 1291306737, 883851756, 1355819080, 2834319249, - 3825063450, 4205423325, - }, - { - 525214560, 1972466543, 1542775297, 3030388145, 2623763324, 1445252054, 2315649878, - 2940376435, 1322155857, 2007925719, 899111545, 3946601974, 720416639, 566341007, - 3830971140, 2379218430, 946001131, 324551023, 3792134824, 2419222364, 2507004728, - 4050415702, 2934667964, 3435655480, 3738151878, 340092998, 429296098, 3804978739, - 1547120540, 976306993, 1134820236, 288696971, 292350374, 423348923, 4250561112, - 1380146522, 646098313, 3081299572, 3633231429, 2348008746, 3250735726, 3495239618, - 1083361876, 2660545988, 97607299, 741626628, 2451882102, 607936604, 1566190301, - 3752644837, 1626575269, 2569947980, 120166892, 1936167922, 2964570009, 2601765059, - 2550590348, 1491574373, 1916644920, 2955888714, 3900360190, 396836243, 2417234534, - 4219822777, 3017031315, 3848370775, 4113753945, 1038708316, 1227041843, 1287656330, - 594136009, 1679465955, 1127853612, 445673212, 2491164616, 4234959779, 3670094401, - 2810998507, 2091885715, 4213376041, 3724691332, 1428205363, 2351471476, 1863345709, - 3172242044, 1435176883, 925973933, 3166951436, 2056462416, 489417029, 4029854347, - 3002516723, 1597712463, 1200457469, 3909654542, 1352519428, 13398705, 3919269221, - 371331154, 332347636, 3726033518, 2407091731, 2926199215, 3054175446, 3208807730, - 584793525, 2706493003, 561190823, 2412132195, 2488492462, 3149885896, 3512276852, - 2843032269, 2485506176, 4025325347, 4152622551, 4022346903, 331746013, 197533993, - 3658414685, 2670729696, 3290854172, 2251426444, 3569225076, 2466203243, 658184940, - 518096293, 52156682, 2398958685, 745491615, 3723004242, 2847276077, 1857504125, - 633035220, 4057593658, 2783467746, 3122875931, 446601186, 2786851490, 261950076, - 2843506874, 745391893, 1404094021, 2234513997, 315083019, 645865358, 2862243948, - 1204315994, 3701151065, 663411328, 1924727700, 1905843757, 1483930049, 449616818, - 3793968150, 1840668755, 1671024110, 4079375869, 4171670660, 2585904968, 3886777251, - 525214560, 1972466543, 1542775297, 3030388145, 2530126952, 1445252054, 2315649878, - 2940376435, 1322155857, 1599103627, 899111545, 3946601974, 720416639, 566341007, - 4070101360, 2379218430, 946001131, 324551023, 3792134824, 2445126690, 2507004728, - 4050415702, 2934667964, 3435655480, 2968121571, 340092998, 429296098, 3804978739, - 1547120540, 3901803457, 1134820236, 288696971, 292350374, 423348923, 1589814289, - 1380146522, 646098313, 3081299572, 3633231429, 670777956, 3250735726, 3495239618, - 1083361876, 2660545988, 4050232394, 741626628, 2451882102, 607936604, 1566190301, - 1132827700, 1626575269, 2569947980, 120166892, 1936167922, 1280520333, 2601765059, - 2550590348, 1491574373, 1916644920, 1073889810, 3900360190, 396836243, 2417234534, - 4219822777, 1754651820, 3848370775, 4113753945, 1038708316, 1227041843, 464826842, - 594136009, 1679465955, 1127853612, 445673212, 4198686893, 4234959779, 3670094401, - 2810998507, 2091885715, 416103731, 3724691332, 1428205363, 2351471476, 1863345709, - 2637470915, 1435176883, 925973933, 3166951436, 2056462416, 2546319147, 4029854347, - 3002516723, 1597712463, 1200457469, 681365672, 1352519428, 13398705, 3919269221, - 371331154, 742849231, 3726033518, 2407091731, 2926199215, 3054175446, 1323833820, - 584793525, 2706493003, 561190823, 2412132195, 3747238187, 3149885896, 3512276852, - 2843032269, 2485506176, 3817319503, 4152622551, 4022346903, 331746013, 197533993, - 99009902, 2670729696, 3290854172, 2251426444, 3569225076, 4199909720, 658184940, - 518096293, 52156682, 2398958685, 1648201186, 3723004242, 2847276077, 1857504125, - 633035220, 1394668680, 2783467746, 3122875931, 446601186, 2786851490, 2590549096, - 2843506874, 745391893, 1404094021, 2234513997, 347299411, 645865358, 2862243948, - 1204315994, 3701151065, 4028305509, 1924727700, 1905843757, 1483930049, 449616818, - 2251238906, 1840668755, 1671024110, 4079375869, 4171670660, 4080554282, 3886777251, - 525214560, 1972466543, 1542775297, 3280177496, 2530126952, 1445252054, 2315649878, - 2940376435, 2094983509, 1599103627, 899111545, 3946601974, 720416639, 1446566513, - 4070101360, 2379218430, 946001131, 324551023, 2945613775, 2445126690, 2507004728, - 4050415702, 2934667964, 2815036731, 2968121571, 340092998, 429296098, 3804978739, - 3298867574, 3901803457, 1134820236, 288696971, 292350374, 3280367987, 1589814289, - 1380146522, 646098313, 3081299572, 2536311658, 670777956, 3250735726, 3495239618, - 1083361876, 3726225049, 4050232394, 741626628, 2451882102, 607936604, 3460165725, - 1132827700, 1626575269, 2569947980, 120166892, 2961109404, 1280520333, 2601765059, - 2550590348, 1491574373, 755823086, 1073889810, 3900360190, 396836243, 2417234534, - 3036027780, 1754651820, 3848370775, 4113753945, 1038708316, 3784147349, 464826842, - 594136009, 1679465955, 1127853612, 2128970592, 4198686893, 4234959779, 3670094401, - 2810998507, 421961324, 416103731, 3724691332, 1428205363, 2351471476, 3407618159, - 2637470915, 1435176883, 925973933, 3166951436, 1274860184, 2546319147, 4029854347, - 3002516723, 1597712463, 671480036, 681365672, 1352519428, 13398705, 3919269221, - 1150967289, 742849231, 3726033518, 2407091731, 2926199215, 3106945136, 1323833820, - 584793525, 2706493003, 561190823, 2013357219, 3747238187, 3149885896, 3512276852, - 2843032269, 3595347994, 3817319503, 4152622551, 4022346903, 331746013, 367216863, - 99009902, 2670729696, 3290854172, 2251426444, 3130148315, 4199909720, 658184940, - 518096293, 52156682, 3004378899, 1648201186, 3723004242, 2847276077, 1857504125, - 253542783, 1394668680, 2783467746, 3122875931, 446601186, 1228837642, 2590549096, - 2843506874, 745391893, 1404094021, 1324404436, 347299411, 645865358, 2862243948, - 1204315994, 1455458347, 4028305509, 1924727700, 1905843757, 1483930049, 330348422, - 2251238906, 1840668755, 1671024110, 4079375869, 606568968, 4080554282, 3886777251, - 525214560, 1972466543, 1703103913, 3280177496, 2530126952, 1445252054, 2315649878, - 3946153427, 2094983509, 1599103627, 899111545, 3946601974, 2053673584, 1446566513, - 4070101360, 2379218430, 946001131, 4184236551, 2945613775, 2445126690, 2507004728, - 4050415702, 3890831500, 2815036731, 2968121571, 340092998, 429296098, 228493148, - 3298867574, 3901803457, 1134820236, 288696971, 2321943990, 3280367987, 1589814289, - 1380146522, 646098313, 1765624343, 2536311658, 670777956, 3250735726, 3495239618, - 1772431608, 3726225049, 4050232394, 741626628, 2451882102, 3386124330, 3460165725, - 1132827700, 1626575269, 2569947980, 860947846, 2961109404, 1280520333, 2601765059, - 2550590348, 2298495740, 755823086, 1073889810, 3900360190, 396836243, 2702634902, - 3036027780, 1754651820, 3848370775, 4113753945, 3836550212, 3784147349, 464826842, - 594136009, 1679465955, 1500399122, 2128970592, 4198686893, 4234959779, 3670094401, - 1632934875, 421961324, 416103731, 3724691332, 1428205363, 2330377177, 3407618159, - 2637470915, 1435176883, 925973933, 2558479866, 1274860184, 2546319147, 4029854347, - 3002516723, 1331271216, 671480036, 681365672, 1352519428, 13398705, 1532459856, - 1150967289, 742849231, 3726033518, 2407091731, 1766120506, 3106945136, 1323833820, - 584793525, 2706493003, 3817434387, 2013357219, 3747238187, 3149885896, 3512276852, - 203757, 3595347994, 3817319503, 4152622551, 4022346903, 3438004885, 367216863, - 99009902, 2670729696, 3290854172, 1092092654, 3130148315, 4199909720, 658184940, - 518096293, 982576981, 3004378899, 1648201186, 3723004242, 2847276077, 33113683, - 253542783, 1394668680, 2783467746, 3122875931, 3109404671, 1228837642, 2590549096, - 2843506874, 745391893, 809710525, 1324404436, 347299411, 645865358, 2862243948, - 3652256751, 1455458347, 4028305509, 1924727700, 1905843757, 2035132481, 330348422, - 2251238906, 1840668755, 1671024110, 3593348393, 4151905045, 3398483770, 611142788, - 1798029112, 2747225670, 2894981396, 2117120651, 3087941624, 416876364, 700011792, - 63929447, 822005210, 3483417647, 3513365134, 3071572873, 1925919001, 2778688996, - 3079943255, 1252316311, 91270196, 3469862149, 156659741, 1342755036, 3821302858, - 1790046971, 289329863, 1357914395, 4143182690, 2590503919, 3242437796, 1341085928, - 2685277054, 727602392, 2581493226, 3216496864, 2171373196, 3767765187, 1895767358, - 1029452326, 851913526, 1746266839, 3370323171, 648118190, 3244211645, 2623946928, - 3859087079, 384443034, 2026989771, 802104797, 2201121552, 725742304, 1673563239, - 4045658814, 2682764476, 3032306650, 2725871420, 3467522540, 534803010, 1135606913, - 871336950, 937160030, 3384357161, 641566845, 2267407903, 331847343, 787968740, - 2673012251, 2066357778, 2740382722, 1638377946, 2260504282, 3513172717, 238548903, - 2203496688, 630532448, 3702112076, 2635952931, 3344713216, 139406056, 2369004628, - 3547213209, 2944858950, 1231203228, 616949630, 2619739101, 89360251, 2364353701, - 1025345607, 4177965685, 62274372, 3059207586, 3303376016, 2919795870, 3676526103, - 2689781822, 1062293263, 2684605838, 863975243, 723728777, 1057919510, 1708017843, - 4264127977, 3013938022, 3958746896, 328415103, 1117948849, 751056929, 2442147201, - 1781170563, 765377308, 961452970, 4247303973, 2233034754, 86997820, 3495561473, - 3075957349, 3152032365, 1220657606, 708134514, 26714613, 3749542051, 1640668224, - 2252760600, 1635050662, 947216628, 3612773344, 4089189500, 3647048119, 979491227, - 4149824933, 3160885292, 2808843788, 998859510, 3903167193, 1728999561, 3673946130, - 279338980, 2507635299, 1614929524, 302060483, 2874453052, 3798613814, 2013436766, - 3514754020, 2923162106, 2658720327, 3498579091, 3292220096, 3796129102, 1907288796, - 2820663603, 4276052248, 247755133, 2088596201, 3154955976, 3309397641, 3606171919, - 1356791029, 1030266022, - }, - { - 3868946146, 1938156793, 1877502872, 1408917625, 1549117911, 2465501566, 4218547770, - 2942249332, 2731789075, 2366036899, 1312641799, 2243363271, 2238839307, 384814263, - 1552361757, 3521369641, 431721717, 3089625732, 1023760034, 53847139, 2240881978, - 3178046414, 145135653, 1580878781, 3500228040, 3360910006, 3285542950, 3330062556, - 2870158227, 1481496810, 4222704363, 2973046526, 435155769, 3234730070, 3306545960, - 2539776908, 3991420334, 125389349, 2397544348, 2504790975, 886432257, 1804136430, - 1506551086, 219847214, 890282686, 1489840806, 2536942497, 87527661, 1822718904, - 3984956867, 2334419518, 4065487054, 992104547, 1566792845, 1068226712, 2622731799, - 921431708, 2833392639, 640267449, 324907409, 3911698049, 2108189994, 1623761598, - 52771719, 467926435, 2811768106, 3760723083, 906402727, 3438479463, 2064004404, - 988123982, 563076447, 2979641383, 1366086397, 2078608605, 3868491514, 1077957067, - 615363273, 1388831706, 1586480552, 4216838311, 3587550780, 2057048927, 2814838921, - 2454041809, 180612020, 930406098, 4286819113, 2756562967, 3404265234, 3844482428, - 467484533, 4122644954, 3517116598, 1887163240, 4217569180, 4191149652, 2756931330, - 3702787956, 152166773, 146763911, 536678737, 481385008, 3681433244, 1194909733, - 3713568496, 3927837202, 846842608, 687314083, 1144793694, 1062075916, 3017627145, - 1296695243, 981862419, 2363304726, 3242788356, 3359957762, 4249190787, 1697910336, - 3286799886, 1063822293, 3246091430, 743808559, 2137668568, 2812072749, 2303791182, - 3161789548, 2911126624, 4087873192, 1813622227, 1272618849, 1882292328, 3861455677, - 2921641470, 3079812494, 2814569163, 1975646942, 2826176621, 1896904368, 831552834, - 2935863403, 449217054, 3688067832, 1048877596, 1613227043, 553867520, 3682575786, - 3058863948, 4200858129, 4131625, 2434123776, 2235627905, 2905358693, 3429312266, - 3363231514, 1182242507, 2792234422, 1843330053, 4192875151, 1088813348, 357805687, - 3868946146, 1938156793, 1877502872, 1408917625, 30638250, 2465501566, 4218547770, - 2942249332, 2731789075, 448998968, 1312641799, 2243363271, 2238839307, 384814263, - 2229663001, 3521369641, 431721717, 3089625732, 1023760034, 790771414, 2240881978, - 3178046414, 145135653, 1580878781, 847577516, 3360910006, 3285542950, 3330062556, - 2870158227, 112738978, 4222704363, 2973046526, 435155769, 3234730070, 1135073835, - 2539776908, 3991420334, 125389349, 2397544348, 1243128255, 886432257, 1804136430, - 1506551086, 219847214, 875051553, 1489840806, 2536942497, 87527661, 1822718904, - 1883615145, 2334419518, 4065487054, 992104547, 1566792845, 1037132511, 2622731799, - 921431708, 2833392639, 640267449, 504304037, 3911698049, 2108189994, 1623761598, - 52771719, 3969520254, 2811768106, 3760723083, 906402727, 3438479463, 3707538496, - 988123982, 563076447, 2979641383, 1366086397, 3577913613, 3868491514, 1077957067, - 615363273, 1388831706, 903353909, 4216838311, 3587550780, 2057048927, 2814838921, - 3532304828, 180612020, 930406098, 4286819113, 2756562967, 1950528802, 3844482428, - 467484533, 4122644954, 3517116598, 139409766, 4217569180, 4191149652, 2756931330, - 3702787956, 504815033, 146763911, 536678737, 481385008, 3681433244, 2166865052, - 3713568496, 3927837202, 846842608, 687314083, 135403542, 1062075916, 3017627145, - 1296695243, 981862419, 2405232584, 3242788356, 3359957762, 4249190787, 1697910336, - 3517294012, 1063822293, 3246091430, 743808559, 2137668568, 2962825355, 2303791182, - 3161789548, 2911126624, 4087873192, 2344237973, 1272618849, 1882292328, 3861455677, - 2921641470, 1062672856, 2814569163, 1975646942, 2826176621, 1896904368, 3172875195, - 2935863403, 449217054, 3688067832, 1048877596, 983648949, 553867520, 3682575786, - 3058863948, 4200858129, 2552994282, 2434123776, 2235627905, 2905358693, 3429312266, - 461707508, 1182242507, 2792234422, 1843330053, 4192875151, 2557078297, 357805687, - 3868946146, 1938156793, 1877502872, 1178921294, 30638250, 2465501566, 4218547770, - 2942249332, 2597087237, 448998968, 1312641799, 2243363271, 2238839307, 3465588695, - 2229663001, 3521369641, 431721717, 3089625732, 2420359327, 790771414, 2240881978, - 3178046414, 145135653, 3411014139, 847577516, 3360910006, 3285542950, 3330062556, - 4257518865, 112738978, 4222704363, 2973046526, 435155769, 1154160505, 1135073835, - 2539776908, 3991420334, 125389349, 1396475349, 1243128255, 886432257, 1804136430, - 1506551086, 3727497731, 875051553, 1489840806, 2536942497, 87527661, 2521823325, - 1883615145, 2334419518, 4065487054, 992104547, 3431387970, 1037132511, 2622731799, - 921431708, 2833392639, 780276883, 504304037, 3911698049, 2108189994, 1623761598, - 1832564202, 3969520254, 2811768106, 3760723083, 906402727, 2319993554, 3707538496, - 988123982, 563076447, 2979641383, 3703509163, 3577913613, 3868491514, 1077957067, - 615363273, 3925135746, 903353909, 4216838311, 3587550780, 2057048927, 2129250845, - 3532304828, 180612020, 930406098, 4286819113, 571849466, 1950528802, 3844482428, - 467484533, 4122644954, 3696836546, 139409766, 4217569180, 4191149652, 2756931330, - 84389584, 504815033, 146763911, 536678737, 481385008, 281139563, 2166865052, - 3713568496, 3927837202, 846842608, 2123715146, 135403542, 1062075916, 3017627145, - 1296695243, 4206227732, 2405232584, 3242788356, 3359957762, 4249190787, 2766470555, - 3517294012, 1063822293, 3246091430, 743808559, 2821229002, 2962825355, 2303791182, - 3161789548, 2911126624, 503886017, 2344237973, 1272618849, 1882292328, 3861455677, - 4158985014, 1062672856, 2814569163, 1975646942, 2826176621, 4118784229, 3172875195, - 2935863403, 449217054, 3688067832, 3556237148, 983648949, 553867520, 3682575786, - 3058863948, 3200838331, 2552994282, 2434123776, 2235627905, 2905358693, 4178312045, - 461707508, 1182242507, 2792234422, 1843330053, 3597816691, 2557078297, 357805687, - 3868946146, 1938156793, 2168462050, 1178921294, 30638250, 2465501566, 4218547770, - 4101101381, 2597087237, 448998968, 1312641799, 2243363271, 313553894, 3465588695, - 2229663001, 3521369641, 431721717, 737541534, 2420359327, 790771414, 2240881978, - 3178046414, 326569272, 3411014139, 847577516, 3360910006, 3285542950, 3098408987, - 4257518865, 112738978, 4222704363, 2973046526, 3668411828, 1154160505, 1135073835, - 2539776908, 3991420334, 2902976896, 1396475349, 1243128255, 886432257, 1804136430, - 2162242501, 3727497731, 875051553, 1489840806, 2536942497, 2238214198, 2521823325, - 1883615145, 2334419518, 4065487054, 1081167745, 3431387970, 1037132511, 2622731799, - 921431708, 2612105434, 780276883, 504304037, 3911698049, 2108189994, 2518535877, - 1832564202, 3969520254, 2811768106, 3760723083, 2894544992, 2319993554, 3707538496, - 988123982, 563076447, 719340658, 3703509163, 3577913613, 3868491514, 1077957067, - 2371417985, 3925135746, 903353909, 4216838311, 3587550780, 3146473377, 2129250845, - 3532304828, 180612020, 930406098, 1054512059, 571849466, 1950528802, 3844482428, - 467484533, 1437844285, 3696836546, 139409766, 4217569180, 4191149652, 1161452915, - 84389584, 504815033, 146763911, 536678737, 3965987378, 281139563, 2166865052, - 3713568496, 3927837202, 2566873330, 2123715146, 135403542, 1062075916, 3017627145, - 3204726297, 4206227732, 2405232584, 3242788356, 3359957762, 2338319494, 2766470555, - 3517294012, 1063822293, 3246091430, 1531757306, 2821229002, 2962825355, 2303791182, - 3161789548, 2778326467, 503886017, 2344237973, 1272618849, 1882292328, 1725075819, - 4158985014, 1062672856, 2814569163, 1975646942, 3822868823, 4118784229, 3172875195, - 2935863403, 449217054, 2465297154, 3556237148, 983648949, 553867520, 3682575786, - 4023654874, 3200838331, 2552994282, 2434123776, 2235627905, 3063253867, 4178312045, - 461707508, 1182242507, 2792234422, 3673318927, 1249828417, 2772427670, 1052324962, - 3106530204, 2843183862, 630633945, 4140139503, 1659674037, 1096812757, 1376150732, - 2328468653, 1410746620, 4025107990, 3335632421, 2754906610, 1615859006, 285467698, - 4013475548, 1287384555, 1191111485, 1999165134, 2396354947, 1628158236, 3586708909, - 228664781, 2501369720, 2516229872, 2977432606, 1745869751, 750661412, 1142144084, - 2705268946, 1728488244, 589587862, 3604281130, 3217245915, 2061424631, 1918958878, - 1162850007, 438550637, 1774088146, 3237803593, 827476363, 404982536, 2344744845, - 3416436851, 369597250, 287618335, 1349740180, 3489688427, 417859991, 3229729092, - 3214122057, 3955335849, 3014669381, 2178319957, 1259991234, 2689513541, 2628816894, - 3734652479, 4202568782, 3149274749, 497295490, 3427602420, 3229774907, 59257138, - 856364156, 429586733, 1800559699, 1300239050, 1311125646, 257421988, 3749074142, - 1648939149, 1914174865, 105489877, 3599116888, 2695725484, 1543985792, 3210070699, - 1867126432, 3088920410, 953084407, 2185095866, 1427606476, 1572442276, 3322674991, - 3578824788, 1156246244, 2938200612, 3409545464, 215820858, 2279282461, 3861049095, - 1589517366, 208707366, 2741570297, 440313302, 864288468, 1564945290, 1050929272, - 3037450392, 1101323242, 1200278943, 3005564105, 3847988630, 3251750599, 2608433412, - 3106720723, 1522694503, 3857782840, 4282681349, 2229263718, 4106780914, 125648941, - 1933617693, 2971178569, 3537872030, 448962137, 652123777, 2393871920, 3938047691, - 244410098, 3110791961, 3122318189, 877378106, 3683644255, 4279094311, 3638987055, - 667681197, 1679868535, 1938378101, 1331340184, 734163051, 3409564713, 955108672, - 3969637663, 156515523, 1871394552, 590275639, 3237133664, 898438533, 2291347006, - 644781653, 3575493549, 1206698159, 2484805619, 2931447110, 2411269190, 3866437145, - 161562563, 3077166456, 792874130, 3193406610, 2500233218, 596837225, 3667458052, - 3239960816, 2271901243, - }, - { - 3975736867, 2402230281, 4092718962, 3100052505, 3277909563, 2827154828, 1067483357, - 3495429909, 426635932, 2702495453, 725679489, 3705541400, 1308182381, 27549785, - 3000675918, 2982141597, 1090931027, 755020243, 3986354189, 2529541113, 452574019, - 2384876926, 2147764179, 1360907484, 2072364695, 3034185952, 2765119653, 3279755577, - 3828140333, 582568392, 4228353628, 701214306, 2460043371, 3943376509, 2443090800, - 2481277520, 859309333, 2928621220, 1933644685, 3803162893, 3310629548, 2361261213, - 790233558, 2517540072, 2823327610, 2952921690, 3295251862, 1089451775, 2637751681, - 1648031370, 1343061717, 2355026672, 67684812, 4019593497, 2636283634, 1051433451, - 51111285, 15338687, 3779021741, 3987886044, 70037785, 2009147353, 4236701871, - 928261128, 2185183571, 2793993680, 2975111058, 3730415022, 3316612678, 823585671, - 4153354125, 509071385, 2056228251, 4034784810, 96820040, 169863045, 932848332, - 2282651407, 747279843, 1387211022, 2410099142, 3394315084, 3191572807, 4073182500, - 3768455462, 3712420663, 3000991259, 249137656, 2477445202, 3952155443, 392730170, - 4208559971, 24751401, 661761054, 1574175475, 2715927647, 985309803, 2570053358, - 619269634, 3890591314, 1129119636, 3133886450, 328788870, 3449809720, 1380118080, - 2719792059, 691527418, 3487733607, 3819095050, 3367871088, 709089170, 1057897966, - 1938975941, 4082466714, 251564920, 3083496965, 1040123365, 295024253, 2788334176, - 3430095934, 3641758945, 2029993123, 3231254260, 150555625, 2270671577, 2032382533, - 2088497043, 1392075576, 644811077, 2122632989, 3224165725, 1571908345, 2558692460, - 1493305706, 4064652450, 448105905, 699188129, 2017324335, 4286307548, 2415725473, - 3976741021, 3526784185, 2882973520, 3420335125, 2034028744, 1425242390, 982315917, - 2614735561, 2439972944, 2518992720, 3792239985, 3260669732, 2586472751, 3432756715, - 1318634102, 3722487277, 3037304631, 433233786, 3750002877, 2504731459, 1111327015, - 3975736867, 2402230281, 4092718962, 3100052505, 3521430425, 2827154828, 1067483357, - 3495429909, 426635932, 2034644068, 725679489, 3705541400, 1308182381, 27549785, - 3001720496, 2982141597, 1090931027, 755020243, 3986354189, 307638580, 452574019, - 2384876926, 2147764179, 1360907484, 1701580099, 3034185952, 2765119653, 3279755577, - 3828140333, 2659043235, 4228353628, 701214306, 2460043371, 3943376509, 2084857792, - 2481277520, 859309333, 2928621220, 1933644685, 4152646669, 3310629548, 2361261213, - 790233558, 2517540072, 481283060, 2952921690, 3295251862, 1089451775, 2637751681, - 2915212660, 1343061717, 2355026672, 67684812, 4019593497, 3290479436, 1051433451, - 51111285, 15338687, 3779021741, 1430944862, 70037785, 2009147353, 4236701871, - 928261128, 2063919641, 2793993680, 2975111058, 3730415022, 3316612678, 2373806232, - 4153354125, 509071385, 2056228251, 4034784810, 1912268707, 169863045, 932848332, - 2282651407, 747279843, 3712980941, 2410099142, 3394315084, 3191572807, 4073182500, - 4262344652, 3712420663, 3000991259, 249137656, 2477445202, 3374467273, 392730170, - 4208559971, 24751401, 661761054, 1670592959, 2715927647, 985309803, 2570053358, - 619269634, 830547082, 1129119636, 3133886450, 328788870, 3449809720, 202644333, - 2719792059, 691527418, 3487733607, 3819095050, 1400269159, 709089170, 1057897966, - 1938975941, 4082466714, 3393893128, 3083496965, 1040123365, 295024253, 2788334176, - 1219456, 3641758945, 2029993123, 3231254260, 150555625, 3713963210, 2032382533, - 2088497043, 1392075576, 644811077, 3733090890, 3224165725, 1571908345, 2558692460, - 1493305706, 1678929187, 448105905, 699188129, 2017324335, 4286307548, 3368868963, - 3976741021, 3526784185, 2882973520, 3420335125, 3233347584, 1425242390, 982315917, - 2614735561, 2439972944, 4172908214, 3792239985, 3260669732, 2586472751, 3432756715, - 1926157640, 3722487277, 3037304631, 433233786, 3750002877, 625648993, 1111327015, - 3975736867, 2402230281, 4092718962, 1349560774, 3521430425, 2827154828, 1067483357, - 3495429909, 2808148912, 2034644068, 725679489, 3705541400, 1308182381, 212242504, - 3001720496, 2982141597, 1090931027, 755020243, 2510536004, 307638580, 452574019, - 2384876926, 2147764179, 3227931749, 1701580099, 3034185952, 2765119653, 3279755577, - 1054678914, 2659043235, 4228353628, 701214306, 2460043371, 381309305, 2084857792, - 2481277520, 859309333, 2928621220, 891630344, 4152646669, 3310629548, 2361261213, - 790233558, 1490030690, 481283060, 2952921690, 3295251862, 1089451775, 2025962691, - 2915212660, 1343061717, 2355026672, 67684812, 2217081575, 3290479436, 1051433451, - 51111285, 15338687, 3455020635, 1430944862, 70037785, 2009147353, 4236701871, - 1155691935, 2063919641, 2793993680, 2975111058, 3730415022, 403147571, 2373806232, - 4153354125, 509071385, 2056228251, 444685935, 1912268707, 169863045, 932848332, - 2282651407, 2077207745, 3712980941, 2410099142, 3394315084, 3191572807, 640536184, - 4262344652, 3712420663, 3000991259, 249137656, 368243227, 3374467273, 392730170, - 4208559971, 24751401, 495648080, 1670592959, 2715927647, 985309803, 2570053358, - 2181488546, 830547082, 1129119636, 3133886450, 328788870, 2497762979, 202644333, - 2719792059, 691527418, 3487733607, 1976943620, 1400269159, 709089170, 1057897966, - 1938975941, 2071351862, 3393893128, 3083496965, 1040123365, 295024253, 1440317859, - 1219456, 3641758945, 2029993123, 3231254260, 952956380, 3713963210, 2032382533, - 2088497043, 1392075576, 4180475645, 3733090890, 3224165725, 1571908345, 2558692460, - 3482549931, 1678929187, 448105905, 699188129, 2017324335, 2431113987, 3368868963, - 3976741021, 3526784185, 2882973520, 1900625235, 3233347584, 1425242390, 982315917, - 2614735561, 1128074864, 4172908214, 3792239985, 3260669732, 2586472751, 4095880420, - 1926157640, 3722487277, 3037304631, 433233786, 2927295412, 625648993, 1111327015, - 3975736867, 2402230281, 259216032, 1349560774, 3521430425, 2827154828, 1067483357, - 989690947, 2808148912, 2034644068, 725679489, 3705541400, 588787520, 212242504, - 3001720496, 2982141597, 1090931027, 1235811382, 2510536004, 307638580, 452574019, - 2384876926, 3536994565, 3227931749, 1701580099, 3034185952, 2765119653, 463890041, - 1054678914, 2659043235, 4228353628, 701214306, 3085494195, 381309305, 2084857792, - 2481277520, 859309333, 3760199179, 891630344, 4152646669, 3310629548, 2361261213, - 2550680915, 1490030690, 481283060, 2952921690, 3295251862, 4195487760, 2025962691, - 2915212660, 1343061717, 2355026672, 339445869, 2217081575, 3290479436, 1051433451, - 51111285, 1113202216, 3455020635, 1430944862, 70037785, 2009147353, 3982848623, - 1155691935, 2063919641, 2793993680, 2975111058, 1725337613, 403147571, 2373806232, - 4153354125, 509071385, 1474832043, 444685935, 1912268707, 169863045, 932848332, - 1500855137, 2077207745, 3712980941, 2410099142, 3394315084, 2800379966, 640536184, - 4262344652, 3712420663, 3000991259, 1028021485, 368243227, 3374467273, 392730170, - 4208559971, 108468246, 495648080, 1670592959, 2715927647, 985309803, 61959589, - 2181488546, 830547082, 1129119636, 3133886450, 3912020361, 2497762979, 202644333, - 2719792059, 691527418, 1984193076, 1976943620, 1400269159, 709089170, 1057897966, - 2381612490, 2071351862, 3393893128, 3083496965, 1040123365, 391784014, 1440317859, - 1219456, 3641758945, 2029993123, 2260373342, 952956380, 3713963210, 2032382533, - 2088497043, 135943164, 4180475645, 3733090890, 3224165725, 1571908345, 2660287325, - 3482549931, 1678929187, 448105905, 699188129, 4104693318, 2431113987, 3368868963, - 3976741021, 3526784185, 113762138, 1900625235, 3233347584, 1425242390, 982315917, - 599246177, 1128074864, 4172908214, 3792239985, 3260669732, 2309689974, 4095880420, - 1926157640, 3722487277, 3037304631, 3765223460, 866296319, 1169380319, 2919436659, - 3370646420, 1866719277, 3226685069, 4252262342, 1835269960, 1170376930, 1357078768, - 269175192, 3826888026, 3430363541, 1920758494, 51532769, 2919489927, 1568325914, - 3184357856, 43519013, 2108988015, 1398495041, 2844640139, 2317092036, 1774750014, - 2690907136, 1834465421, 1106469655, 2149810726, 4265420439, 2048218411, 1399986034, - 1361619115, 2504769226, 913700780, 2382994726, 4292849877, 1381838410, 250258264, - 1828569640, 1732718872, 1869949326, 835188347, 4180489913, 3049522050, 535168392, - 3972173823, 2763844722, 3401884753, 3750694101, 851518496, 1015521371, 1511969218, - 1597622074, 3810841601, 3326003776, 3141062630, 552856274, 4059179808, 175647012, - 3893497501, 1805118717, 1064213711, 2310866839, 1397146463, 1798096676, 279868399, - 1926726615, 2773068510, 347721208, 4099183723, 509136218, 2833615756, 3960499694, - 4236258712, 1765641675, 535748563, 354515646, 3307314159, 3160079941, 3252681800, - 2568363625, 3818514182, 3738662353, 899056999, 2531772068, 647726503, 2895823632, - 393777910, 1759531813, 2363148604, 2931477989, 3381169914, 3877595131, 2375539210, - 557544627, 273611522, 2717517554, 1935966767, 1738732887, 29153600, 20993454, - 3758163226, 1692844400, 2176938194, 378940221, 2888599759, 1173120554, 2732575460, - 3912766812, 522606644, 1925230852, 3887440328, 2111843275, 3549473366, 922916775, - 2889744544, 2970467682, 3039277863, 990580154, 55435595, 1665634070, 3043418336, - 2792050230, 2762503138, 1402344059, 2099263558, 3945248675, 3925566467, 2413979948, - 463637252, 3768636616, 3374572388, 2217956879, 791988933, 382210765, 1715859444, - 3462446413, 971427992, 3255404695, 2001750035, 2214129237, 320812374, 3688098101, - 920365480, 2819401059, 2932570681, 3749857130, 523943786, 1271514748, 4078439472, - 3501181265, 2475869985, 1797996951, 2300820710, 3994893924, 1739992082, 2475950326, - 3780826558, 1018851411, - }, -}; diff --git a/src/include/miopen/prelu.hpp b/src/include/miopen/prelu.hpp new file mode 100644 index 0000000000..35e5e7f611 --- /dev/null +++ b/src/include/miopen/prelu.hpp @@ -0,0 +1,53 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include + +namespace miopen { + +struct Handle; +struct TensorDescriptor; + +MIOPEN_INTERNALS_EXPORT size_t GetPReLUBackwardWorkspaceSize(Handle& handle, + const TensorDescriptor& inputDesc, + const TensorDescriptor& weightDesc); + +MIOPEN_INTERNALS_EXPORT miopenStatus_t PReLUBackward(Handle& handle, + Data_t workspace, + size_t workspaceSizeInBytes, + const TensorDescriptor& inputDesc, + ConstData_t input, + const TensorDescriptor& weightDesc, + ConstData_t weight, + const TensorDescriptor& doutputDesc, + ConstData_t doutput, + const TensorDescriptor& dinputDesc, + Data_t dinput, + const TensorDescriptor& dweightDesc, + Data_t dweight); + +} // namespace miopen diff --git a/src/include/miopen/prelu/invoke_params.hpp b/src/include/miopen/prelu/invoke_params.hpp new file mode 100644 index 0000000000..1acc2dc143 --- /dev/null +++ b/src/include/miopen/prelu/invoke_params.hpp @@ -0,0 +1,59 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include + +namespace miopen { + +namespace prelu { + +struct InvokeParams : public miopen::InvokeParams +{ + InvokeParams() = default; + + const TensorDescriptor* inputDesc = nullptr; + const TensorDescriptor* weightDesc = nullptr; + const TensorDescriptor* doutputDesc = nullptr; + const TensorDescriptor* dinputDesc = nullptr; + const TensorDescriptor* dweightDesc = nullptr; + + ConstData_t input = nullptr; + ConstData_t weight = nullptr; + ConstData_t doutput = nullptr; + Data_t dinput = nullptr; + Data_t dweight = nullptr; + Data_t workspace = nullptr; + std::size_t workspace_size = 0; + + std::size_t GetWorkspaceSize() const { return workspace_size; } + Data_t GetWorkspace() const { return workspace; } +}; + +} // namespace prelu + +} // namespace miopen diff --git a/src/include/miopen/prelu/problem_description.hpp b/src/include/miopen/prelu/problem_description.hpp new file mode 100644 index 0000000000..ce10d96046 --- /dev/null +++ b/src/include/miopen/prelu/problem_description.hpp @@ -0,0 +1,115 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include + +namespace miopen { + +struct NetworkConfig; + +namespace prelu { + +bool checkSameType(const TensorDescriptor& x, const TensorDescriptor& y); +bool checkSameLength(const TensorDescriptor& x, const TensorDescriptor& y); + +struct BackwardProblemDescription : ProblemDescriptionBase +{ + + BackwardProblemDescription(const TensorDescriptor& inputDesc_, + const TensorDescriptor& weightDesc_, + const TensorDescriptor& doutputDesc_, + const TensorDescriptor& dinputDesc_, + const TensorDescriptor& dweightDesc_) + : inputDesc(inputDesc_), + weightDesc(weightDesc_), + doutputDesc(doutputDesc_), + dinputDesc(dinputDesc_), + dweightDesc(dweightDesc_) + { + IsSameType(); + IsRightLength(); + } + + const TensorDescriptor& GetInputDesc() const { return inputDesc; } + const TensorDescriptor& GetWeightDesc() const { return weightDesc; } + const TensorDescriptor& GetdOuputDesc() const { return doutputDesc; } + const TensorDescriptor& GetdInputDesc() const { return dinputDesc; } + const TensorDescriptor& GetdWeightDesc() const { return dweightDesc; } + + bool IsSameType() const + { + if(!checkSameType(inputDesc, weightDesc)) + MIOPEN_THROW(miopenStatusBadParm, + "PReLU: Input and Weight tensor must have same type."); + if(!checkSameType(inputDesc, dinputDesc) || !checkSameType(weightDesc, dweightDesc)) + MIOPEN_THROW(miopenStatusBadParm, + "PReLU: Gradient tensors (excluding Output gradient) must share a same " + "type with Input and Weight tensor."); + return true; + } + + bool IsRightLength() const + { + if(!checkSameLength(inputDesc, doutputDesc) || !checkSameLength(inputDesc, dinputDesc)) + MIOPEN_THROW( + miopenStatusBadParm, + "PReLU: Input and Output Gradient tensors sizes must match with Input tensor."); + if(weightDesc.GetNumDims() != 1) + MIOPEN_THROW(miopenStatusBadParm, "PReLU: Weight tensor must have 1 dimension."); + if(weightDesc.GetElementSize() != 1 && + (inputDesc.GetNumDims() == 1 || + weightDesc.GetElementSize() != inputDesc.GetLengths()[1])) + MIOPEN_THROW( + miopenStatusBadParm, + "PReLU: Weight size must be 1 or equal to the second dim of Input tensor."); + if(!checkSameLength(weightDesc, dweightDesc)) + MIOPEN_THROW(miopenStatusBadParm, + "PReLU: Weight Gradient tensors sizes must match with Weight tensor."); + return true; + } + + bool IsSingleWeight() const + { + if(weightDesc.GetElementSize() > 1) + return false; + return true; + } + + NetworkConfig MakeNetworkConfig() const override; + +private: + TensorDescriptor inputDesc; + TensorDescriptor weightDesc; + TensorDescriptor doutputDesc; + TensorDescriptor dinputDesc; + TensorDescriptor dweightDesc; +}; + +} // namespace prelu + +} // namespace miopen diff --git a/src/include/miopen/prelu/solvers.hpp b/src/include/miopen/prelu/solvers.hpp new file mode 100644 index 0000000000..a2bec477f0 --- /dev/null +++ b/src/include/miopen/prelu/solvers.hpp @@ -0,0 +1,77 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include + +namespace miopen { + +namespace solver { + +namespace prelu { + +using BackwardSolverBase = + NonTunableSolverBase; + +struct Backward : BackwardSolverBase +{ + bool MayNeedWorkspace() const override { return true; } +}; + +struct SingleWeightBackward final : Backward +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext& context, + const miopen::prelu::BackwardProblemDescription& problem) const override; + ConvSolution + GetSolution(const ExecutionContext& context, + const miopen::prelu::BackwardProblemDescription& problem) const override; + std::size_t + GetWorkspaceSize(const ExecutionContext& context, + const miopen::prelu::BackwardProblemDescription& problem) const override; +}; + +struct MultiWeightsBackward final : Backward +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext& context, + const miopen::prelu::BackwardProblemDescription& problem) const override; + ConvSolution + GetSolution(const ExecutionContext& context, + const miopen::prelu::BackwardProblemDescription& problem) const override; + std::size_t + GetWorkspaceSize(const ExecutionContext& context, + const miopen::prelu::BackwardProblemDescription& problem) const override; +}; + +} // namespace prelu + +} // namespace solver + +} // namespace miopen diff --git a/src/include/miopen/prelu/utils.hpp b/src/include/miopen/prelu/utils.hpp new file mode 100644 index 0000000000..b7551a4135 --- /dev/null +++ b/src/include/miopen/prelu/utils.hpp @@ -0,0 +1,45 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include +#include +#include + +namespace miopen { +namespace solver { +namespace prelu { + +KernelInfo make_hip_kernel(std::vector localsize, + std::vector gridsize, + std::string kernel_file, + std::string kernel_name, + KernelBuildParameters build_params); + +} // namespace prelu +} // namespace solver +} // namespace miopen diff --git a/src/include/miopen/problem.hpp b/src/include/miopen/problem.hpp index 110e69299d..ebea04e1f6 100644 --- a/src/include/miopen/problem.hpp +++ b/src/include/miopen/problem.hpp @@ -186,7 +186,8 @@ struct Problem const FindOptions& options, std::size_t max_solutions, const Buffers& buffers, - const ConvolutionDescriptor& conv_desc) const; + const ConvolutionDescriptor& conv_desc, + const Problem& original) const; std::vector FindSolutionsImpl(Handle& handle, const FindOptions& options, diff --git a/src/include/miopen/problem_description_base.hpp b/src/include/miopen/problem_description_base.hpp index 3e19f85e8c..a7914a8b5a 100644 --- a/src/include/miopen/problem_description_base.hpp +++ b/src/include/miopen/problem_description_base.hpp @@ -28,7 +28,9 @@ #include #include +#include +#include #include namespace miopen { @@ -51,6 +53,36 @@ inline std::string GetDataTypeName(miopenDataType_t data_type) return "Unknown(" + std::to_string(data_type) + ")"; } +template +constexpr TElement GetN5(unsigned spatial_dims, const std::vector& data) +{ + return std::get<0>(GetNCDHW(spatial_dims, data)); +} + +template +constexpr TElement GetC5(unsigned spatial_dims, const std::vector& data) +{ + return std::get<1>(GetNCDHW(spatial_dims, data)); +} + +template +constexpr TElement GetD5(unsigned spatial_dims, const std::vector& data) +{ + return std::get<2>(GetNCDHW(spatial_dims, data)); +} + +template +constexpr TElement GetH5(unsigned spatial_dims, const std::vector& data) +{ + return std::get<3>(GetNCDHW(spatial_dims, data)); +} + +template +constexpr TElement GetW5(unsigned spatial_dims, const std::vector& data) +{ + return std::get<4>(GetNCDHW(spatial_dims, data)); +} + struct ProblemDescriptionBase { ProblemDescriptionBase() = default; @@ -60,6 +92,9 @@ struct ProblemDescriptionBase ProblemDescriptionBase& operator=(const ProblemDescriptionBase&) = default; [[nodiscard]] virtual NetworkConfig MakeNetworkConfig() const = 0; +#if MIOPEN_ENABLE_SQLITE + static std::string table_name() { return "config"; } +#endif }; } // namespace miopen diff --git a/src/include/miopen/problem_description_layout.hpp b/src/include/miopen/problem_description_layout.hpp new file mode 100644 index 0000000000..780086afe6 --- /dev/null +++ b/src/include/miopen/problem_description_layout.hpp @@ -0,0 +1,102 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2022 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include +#include + +#include + +namespace miopen { + +struct ProblemDescriptionLayoutBase : ProblemDescriptionBase +{ + ProblemDescriptionLayoutBase() = default; + ProblemDescriptionLayoutBase(const ProblemDescriptionLayoutBase&) = default; + ProblemDescriptionLayoutBase(const TensorDescriptor& in_, // x for Forward, y for Backward* + const TensorDescriptor& out_ // y for Forward, x for Backward* + ) + : ProblemDescriptionBase(), + in(in_), + out(out_), + in_layout(ComputeInLayout()), + out_layout(ComputeOutLayout()) + {} + virtual ~ProblemDescriptionLayoutBase() = default; + + ProblemDescriptionLayoutBase& operator=(const ProblemDescriptionLayoutBase&) = default; + + [[nodiscard]] virtual NetworkConfig MakeNetworkConfig() const = 0; + +protected: + TensorDescriptor in; + TensorDescriptor out; + std::string in_layout; + std::string out_layout; + + std::string ComputeInLayout() const + { + return in.GetLayout(in.GetLayout_str()); + } + + std::string ComputeOutLayout() const + { + return out.GetLayout(out.GetLayout_str()); + } +}; + +struct ProblemDescriptionWeightsBase : ProblemDescriptionLayoutBase +{ + ProblemDescriptionWeightsBase() = default; + ProblemDescriptionWeightsBase(const ProblemDescriptionWeightsBase&) = default; + ProblemDescriptionWeightsBase(const TensorDescriptor& in_, // x for Forward, y for Backward* + const TensorDescriptor& weights_, + const TensorDescriptor& out_ // y for Forward, x for Backward* + ) + : ProblemDescriptionLayoutBase(in_, out_), + weights(weights_), + weights_layout(ComputeWeightsLayout()) + {} + virtual ~ProblemDescriptionWeightsBase() = default; + + ProblemDescriptionWeightsBase& operator=(const ProblemDescriptionWeightsBase&) = default; + + [[nodiscard]] virtual NetworkConfig MakeNetworkConfig() const = 0; + +protected: + TensorDescriptor weights; + std::string weights_layout; + + std::string ComputeWeightsLayout() const + { + return weights.GetLayout(weights.GetLayout_str()); + } +}; + +} // namespace miopen diff --git a/src/include/miopen/reduce/invoke_params.hpp b/src/include/miopen/reduce/invoke_params.hpp index 11923fdd79..37fc64a734 100644 --- a/src/include/miopen/reduce/invoke_params.hpp +++ b/src/include/miopen/reduce/invoke_params.hpp @@ -32,21 +32,38 @@ namespace miopen { namespace reduce { -struct InvokeParams : public miopen::InvokeParams +struct ExtremeInvokeParams : public miopen::InvokeParams { - InvokeParams() = default; + ExtremeInvokeParams() = default; const TensorDescriptor* xDesc = nullptr; const TensorDescriptor* yDesc = nullptr; const TensorDescriptor* indiceDesc = nullptr; - ConstData_t x = nullptr; - Data_t y = nullptr; - Data_t indice = nullptr; - Data_t workspace = nullptr; - std::size_t workspace_size = 0; - int32_t dim = 0; - miopenSumNanPropagation_t nanPropagation = MIOPEN_SUM_NOT_PROPAGATE_NAN; + ConstData_t x = nullptr; + Data_t y = nullptr; + Data_t indice = nullptr; + std::size_t workspace_size = 0; + int32_t dim = 0; + + std::size_t GetWorkspaceSize() const { return 0; } + Data_t GetWorkspace() const { return nullptr; } +}; + +struct CalculationInvokeParams : public miopen::InvokeParams +{ + CalculationInvokeParams() = default; + + const TensorDescriptor* xDesc = nullptr; + const TensorDescriptor* yDesc = nullptr; + + ConstData_t x = nullptr; + Data_t y = nullptr; + Data_t workspace = nullptr; + std::size_t workspace_size = 0; + int32_t dim = 0; + miopenReduceCalculationNanPropagation_t nanPropagation = + MIOPEN_REDUCE_CALCULATION_NOT_PROPAGATE_NAN; std::size_t GetWorkspaceSize() const { return workspace_size; } Data_t GetWorkspace() const { return workspace; } diff --git a/src/include/miopen/reduce/problem_description.hpp b/src/include/miopen/reduce/problem_description.hpp index 03001a155b..294c79777e 100644 --- a/src/include/miopen/reduce/problem_description.hpp +++ b/src/include/miopen/reduce/problem_description.hpp @@ -37,21 +37,13 @@ struct NetworkConfig; namespace reduce { -struct ProblemDescription : ProblemDescriptionBase +struct ProblemDescriptionExtreme : ProblemDescriptionBase { - ProblemDescription(miopenSumNanPropagation_t nanPropagation_, - const TensorDescriptor& xDesc_, - const TensorDescriptor& yDesc_, - int32_t dim_) - : nanPropagation(nanPropagation_), xDesc(xDesc_), yDesc(yDesc_), dim(dim_) - { - } - - ProblemDescription(const TensorDescriptor& xDesc_, - const TensorDescriptor& yDesc_, - const TensorDescriptor& indiceDesc_, - int32_t dim_, - miopenReduceExtremeOp_t reduceExtremeOp_) + ProblemDescriptionExtreme(const TensorDescriptor& xDesc_, + const TensorDescriptor& yDesc_, + const TensorDescriptor& indiceDesc_, + int32_t dim_, + miopenReduceExtremeOp_t reduceExtremeOp_) : xDesc(xDesc_), yDesc(yDesc_), indiceDesc(indiceDesc_), @@ -60,15 +52,14 @@ struct ProblemDescription : ProblemDescriptionBase { } - ProblemDescription(const TensorDescriptor& xDesc_, - const TensorDescriptor& indiceDesc_, - int32_t dim_, - miopenReduceExtremeOp_t reduceExtremeOp_) + ProblemDescriptionExtreme(const TensorDescriptor& xDesc_, + const TensorDescriptor& indiceDesc_, + int32_t dim_, + miopenReduceExtremeOp_t reduceExtremeOp_) : xDesc(xDesc_), indiceDesc(indiceDesc_), dim(dim_), reduceExtremeOp(reduceExtremeOp_) { } - miopenSumNanPropagation_t GetNanPropagation_() const { return nanPropagation; } const TensorDescriptor& GetXDesc() const { return xDesc; } const TensorDescriptor& GetYDesc() const { return yDesc; } const TensorDescriptor& GetIndiceDesc() const { return indiceDesc; } @@ -147,9 +138,9 @@ struct ProblemDescription : ProblemDescriptionBase return true; } - bool IsAllPacked() const + bool IsAllContiguous() const { - if(!(xDesc.IsPacked() && yDesc.IsPacked())) + if(!(xDesc.IsContiguous() && yDesc.IsContiguous())) { return false; } @@ -157,9 +148,9 @@ struct ProblemDescription : ProblemDescriptionBase return true; } - bool IsAllPackedWithIndice() const + bool IsAllContiguousWithIndice() const { - if(!(xDesc.IsPacked() && yDesc.IsPacked() && indiceDesc.IsPacked())) + if(!(xDesc.IsContiguous() && yDesc.IsContiguous() && indiceDesc.IsContiguous())) { return false; } @@ -167,9 +158,9 @@ struct ProblemDescription : ProblemDescriptionBase return true; } - bool IsAllPackedIndice() const + bool IsAllContiguousIndice() const { - if(!(xDesc.IsPacked() && indiceDesc.IsPacked())) + if(!(xDesc.IsContiguous() && indiceDesc.IsContiguous())) { return false; } @@ -194,17 +185,126 @@ struct ProblemDescription : ProblemDescriptionBase NetworkConfig MakeNetworkConfig() const override; private: - miopenSumNanPropagation_t nanPropagation; TensorDescriptor xDesc; TensorDescriptor yDesc; TensorDescriptor indiceDesc; int32_t dim; + miopenReduceExtremeOp_t reduceExtremeOp; NetworkConfig MakeForwardNetworkConfig() const; }; +struct ProblemDescriptionCalculation : ProblemDescriptionBase +{ + ProblemDescriptionCalculation(miopenReduceCalculationNanPropagation_t nanPropagation_, + const TensorDescriptor& xDesc_, + const TensorDescriptor& yDesc_, + int32_t dim_, + miopenReduceCalculationOp_t reduceCalculationOp_) + : nanPropagation(nanPropagation_), + xDesc(xDesc_), + yDesc(yDesc_), + dim(dim_), + reduceCalculationOp(reduceCalculationOp_) + { + } + + miopenReduceCalculationNanPropagation_t GetNanPropagation_() const { return nanPropagation; } + const TensorDescriptor& GetXDesc() const { return xDesc; } + const TensorDescriptor& GetYDesc() const { return yDesc; } + int32_t GetDim() const { return dim; } + + bool IsValidLength() const + { + if(xDesc.GetLengths().size() == 1) + return true; + + int32_t posy = 0; + for(int32_t i = 0; i < xDesc.GetLengths().size(); ++i) + { + if(i == dim) + continue; + + if(xDesc.GetLengths()[i] != yDesc.GetLengths()[posy]) + { + MIOPEN_THROW(miopenStatusBadParm, "Reduce: Tensor dimension lengths do not match."); + } + + ++posy; + } + return true; + } + + bool IsValidDim() const + { + if((dim < 0) || (dim >= xDesc.GetLengths().size())) + { + MIOPEN_THROW( + miopenStatusBadParm, + "Reduce: is greater than 0 and less than or equal tensor dimension length."); + } + return true; + } + + bool IsValidInputNumel() const + { + auto xdims = xDesc.GetLengths(); + auto input_numel = + std::accumulate(xdims.begin(), xdims.end(), 1ULL, std::multiplies()); + if(input_numel > INT32_MAX) + MIOPEN_THROW(miopenStatusBadParm, "Reduce: input numel is bigger than INT_MAX."); + + return true; + } + + bool IsSameType() const + { + if(xDesc.GetType() != yDesc.GetType()) + { + return false; + } + return true; + } + + bool IsAllContiguous() const + { + if(!(xDesc.IsContiguous() && yDesc.IsContiguous())) + { + return false; + } + + return true; + } + + bool IsNotLastDim() const + { + if(dim == xDesc.GetLengths().size() - 1) + return false; + return true; + } + + bool IsLargeReduceSize() const + { + if(xDesc.GetLengths()[dim] > 64) + return false; + return true; + } + + NetworkConfig MakeNetworkConfig() const override; + +private: + miopenReduceCalculationNanPropagation_t nanPropagation; + TensorDescriptor xDesc; + TensorDescriptor yDesc; + + int32_t dim; + miopenReduceCalculationOp_t reduceCalculationOp; + + NetworkConfig MakeForwardNetworkConfig() const; +}; + } // namespace reduce } // namespace miopen diff --git a/src/include/miopen/reduce/solvers.hpp b/src/include/miopen/reduce/solvers.hpp index e17f93124d..7a8ea83ba8 100644 --- a/src/include/miopen/reduce/solvers.hpp +++ b/src/include/miopen/reduce/solvers.hpp @@ -35,71 +35,95 @@ namespace solver { namespace reduce { -using ReduceSolver = NonTunableSolverBase; +using ReduceExtremeSolver = + NonTunableSolverBase; +using ReduceCalculationSolver = + NonTunableSolverBase; -struct SumForward final : ReduceSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - bool IsApplicable(const ExecutionContext& context, - const miopen::reduce::ProblemDescription& problem) const override; - ConvSolution GetSolution(const ExecutionContext& context, - const miopen::reduce::ProblemDescription& problem) const override; - std::size_t GetWorkspaceSize(const ExecutionContext& context, - const miopen::reduce::ProblemDescription& problem) const override; - bool MayNeedWorkspace() const override { return true; } -}; - -struct ArgmaxForward final : ReduceSolver +struct ArgmaxForward final : ReduceExtremeSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } size_t XGridSize(std::vector indicedims) const; bool OverMaxGridSize(const ExecutionContext& context, - const miopen::reduce::ProblemDescription& problem) const; + const miopen::reduce::ProblemDescriptionExtreme& problem) const; bool IsApplicable(const ExecutionContext& context, - const miopen::reduce::ProblemDescription& problem) const override; - ConvSolution GetSolution(const ExecutionContext& context, - const miopen::reduce::ProblemDescription& problem) const override; + const miopen::reduce::ProblemDescriptionExtreme& problem) const override; + ConvSolution + GetSolution(const ExecutionContext& context, + const miopen::reduce::ProblemDescriptionExtreme& problem) const override; }; -struct ArgminForward final : ReduceSolver +struct ArgminForward final : ReduceExtremeSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } size_t XGridSize(std::vector indicedims) const; bool OverMaxGridSize(const ExecutionContext& context, - const miopen::reduce::ProblemDescription& problem) const; + const miopen::reduce::ProblemDescriptionExtreme& problem) const; bool IsApplicable(const ExecutionContext& context, - const miopen::reduce::ProblemDescription& problem) const override; - ConvSolution GetSolution(const ExecutionContext& context, - const miopen::reduce::ProblemDescription& problem) const override; + const miopen::reduce::ProblemDescriptionExtreme& problem) const override; + ConvSolution + GetSolution(const ExecutionContext& context, + const miopen::reduce::ProblemDescriptionExtreme& problem) const override; }; -struct MaxForward final : ReduceSolver +struct MaxForward final : ReduceExtremeSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } size_t XGridSize(std::vector ydims) const; bool OverMaxGridSize(const ExecutionContext& context, - const miopen::reduce::ProblemDescription& problem) const; + const miopen::reduce::ProblemDescriptionExtreme& problem) const; bool IsApplicable(const ExecutionContext& context, - const miopen::reduce::ProblemDescription& problem) const override; - ConvSolution GetSolution(const ExecutionContext& context, - const miopen::reduce::ProblemDescription& problem) const override; + const miopen::reduce::ProblemDescriptionExtreme& problem) const override; + ConvSolution + GetSolution(const ExecutionContext& context, + const miopen::reduce::ProblemDescriptionExtreme& problem) const override; }; -struct MinForward final : ReduceSolver +struct MinForward final : ReduceExtremeSolver { const std::string& SolverDbId() const override { return GetSolverDbId(); } size_t XGridSize(std::vector ydims) const; bool OverMaxGridSize(const ExecutionContext& context, - const miopen::reduce::ProblemDescription& problem) const; + const miopen::reduce::ProblemDescriptionExtreme& problem) const; + + bool IsApplicable(const ExecutionContext& context, + const miopen::reduce::ProblemDescriptionExtreme& problem) const override; + ConvSolution + GetSolution(const ExecutionContext& context, + const miopen::reduce::ProblemDescriptionExtreme& problem) const override; +}; + +struct ProdForward final : ReduceCalculationSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext& context, + const miopen::reduce::ProblemDescriptionCalculation& problem) const override; + ConvSolution + GetSolution(const ExecutionContext& context, + const miopen::reduce::ProblemDescriptionCalculation& problem) const override; + std::size_t + GetWorkspaceSize(const ExecutionContext& context, + const miopen::reduce::ProblemDescriptionCalculation& problem) const override; + bool MayNeedWorkspace() const override { return true; } +}; + +struct SumForward final : ReduceCalculationSolver +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } bool IsApplicable(const ExecutionContext& context, - const miopen::reduce::ProblemDescription& problem) const override; - ConvSolution GetSolution(const ExecutionContext& context, - const miopen::reduce::ProblemDescription& problem) const override; + const miopen::reduce::ProblemDescriptionCalculation& problem) const override; + ConvSolution + GetSolution(const ExecutionContext& context, + const miopen::reduce::ProblemDescriptionCalculation& problem) const override; + std::size_t + GetWorkspaceSize(const ExecutionContext& context, + const miopen::reduce::ProblemDescriptionCalculation& problem) const override; + bool MayNeedWorkspace() const override { return true; } }; } // namespace reduce diff --git a/src/include/miopen/reduce/utils.hpp b/src/include/miopen/reduce/utils.hpp index a3280c062c..1e3c3a489c 100644 --- a/src/include/miopen/reduce/utils.hpp +++ b/src/include/miopen/reduce/utils.hpp @@ -64,6 +64,34 @@ inline bool is_parallelism(size_t reqd_work_item_cnt, size_t output_numel, size_ (output_numel * reduce_size > reqd_work_item_cnt); } +inline bool IsImprovementOverROCm(const ExecutionContext& context, + const miopen::reduce::ProblemDescriptionCalculation& problem) +{ + auto xdims = problem.GetXDesc().GetLengths(); + auto ydims = problem.GetYDesc().GetLengths(); + auto dim = problem.GetDim(); + + auto reduce_size = xdims[dim]; + auto output_numel = + std::accumulate(ydims.begin(), ydims.end(), 1ULL, std::multiplies()); + + auto reqd_work_item_cnt = get_reqd_work_item_cnt(context); + + if(is_parallelism(reqd_work_item_cnt, output_numel, reduce_size)) + { + // It's large enough to parallelization, but calling the kernel twice is overhead. + // For cases smaller than this, ROCm pytorch performed better. + bool is_improvement_ROCm = (output_numel * reduce_size < reqd_work_item_cnt * 64); + // But the reduce size is small, MIOpen HIP performed better. + bool is_reduce_large = (reduce_size > 64); + + if(is_improvement_ROCm && is_reduce_large) + return false; + } + + return true; +} + } // namespace reduce } // namespace solver } // namespace miopen diff --git a/src/include/miopen/reducecalculation.hpp b/src/include/miopen/reducecalculation.hpp new file mode 100644 index 0000000000..c21e1863be --- /dev/null +++ b/src/include/miopen/reducecalculation.hpp @@ -0,0 +1,56 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#ifndef MIOPEN_REDUCE_CALCULATION_HPP_ +#define MIOPEN_REDUCE_CALCULATION_HPP_ + +#include + +namespace miopen { + +struct Handle; +struct TensorDescriptor; + +MIOPEN_INTERNALS_EXPORT std::size_t +GetReduceCalculationWorkspaceSize(Handle& handle, + const TensorDescriptor& xDesc, + const TensorDescriptor& yDesc, + int32_t dim, + miopenReduceCalculationOp_t reduceCalculationOp); + +MIOPEN_INTERNALS_EXPORT miopenStatus_t +ReduceCalculationForward(Handle& handle, + Data_t workspace, + size_t workspaceSizeInBytes, + const TensorDescriptor& xDesc, + ConstData_t x, + const TensorDescriptor& yDesc, + Data_t y, + miopenReduceCalculationNanPropagation_t nanPropagation, + int32_t dim, + miopenReduceCalculationOp_t reduceCalculationOp); + +} // namespace miopen +#endif // _MIOPEN_REDUCE_CALCULATION_HPP_ diff --git a/src/include/miopen/rnn.hpp b/src/include/miopen/rnn.hpp index 18d0b5028e..eba00e2a29 100644 --- a/src/include/miopen/rnn.hpp +++ b/src/include/miopen/rnn.hpp @@ -121,6 +121,11 @@ struct MIOPEN_INTERNALS_EXPORT RNNDescriptor : miopenRNNDescriptor const int* lensPerSeq, const void* padding_marker_ptr); + static SeqTensorDescriptor makeSeqTensorDescriptor( + c_array_view descs, + size_t seq_len, + miopenRNNBaseLayout_t layout = miopenRNNBaseLayout_t::miopenRNNDataSeqMajorNotPadded); + static void SeqTensorToTensorDescArray(const SeqTensorDescriptor& desc, std::vector& td, std::vector& ptd); @@ -364,6 +369,57 @@ struct MIOPEN_INTERNALS_EXPORT RNNDescriptor : miopenRNNDescriptor size_t RNNTransformerWorkspaceSize(const SeqTensorDescriptor& xDesc, miopenRNNFWDMode_t fwdMode) const; + // TODO rename + + void ModularForward(Handle& handle, + miopenRNNFWDMode_t fwdMode, + ConstData_t w, + const SeqTensorDescriptor& xDesc, + ConstData_t x, + const TensorDescriptor& hDesc, + ConstData_t hx, + Data_t hy, + const TensorDescriptor& cDesc, + ConstData_t cx, + Data_t cy, + const SeqTensorDescriptor& yDesc, + Data_t y, + Data_t workSpace, + size_t workSpaceSize, + Data_t reserveSpace, + size_t reserveSpaceSize) const; + + void ModularBackward(Handle& handle, + const SeqTensorDescriptor& yDesc, + ConstData_t dy, + const TensorDescriptor& hDesc, + ConstData_t hx, + ConstData_t dhy, + Data_t dhx, + const TensorDescriptor& cDesc, + ConstData_t cx, + ConstData_t dcy, + Data_t dcx, + const SeqTensorDescriptor& xDesc, + Data_t dx, + ConstData_t w, + Data_t workSpace, + size_t workSpaceSize, + Data_t reserveSpace, + size_t reserveSpaceSize) const; + + void ModularBackwardWeights(Handle& handle, + const SeqTensorDescriptor& xDesc, + ConstData_t x, + const TensorDescriptor& hDesc, + ConstData_t hx, + const SeqTensorDescriptor& yDesc, + Data_t w, + Data_t workSpace, + size_t workSpaceSize, + ConstData_t reserveSpace, + size_t /*reserveSpaceSize*/) const; + void RNNTransformerForward(Handle& handle, miopenRNNFWDMode_t fwdMode, ConstData_t w, diff --git a/src/include/miopen/rnn/base_ops.hpp b/src/include/miopen/rnn/base_ops.hpp new file mode 100644 index 0000000000..dbcacb0218 --- /dev/null +++ b/src/include/miopen/rnn/base_ops.hpp @@ -0,0 +1,321 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +namespace miopen { + +namespace rnn_base { + +template +std::array Concat(const std::array& a, const std::array& b) +{ + std::array result; + std::copy(a.cbegin(), a.cend(), result.begin()); + std::copy(b.cbegin(), b.cend(), result.begin() + N); + return result; +} + +inline miopen::GemmDescriptor GemmDescriptor64BitWraper(bool isColMajor_, + bool transA_, + bool transB_, + size_t m_, + size_t n_, + size_t k_, + size_t lda_, + size_t ldb_, + size_t ldc_, + size_t batch_count_, + long long int strideA_, + long long int strideB_, + long long int strideC_, + float alpha_, + float beta_, + miopenDataType_t dataType_, + bool deterministic_) +{ + return GemmDescriptor{isColMajor_, + transA_, + transB_, + static_cast(m_), + static_cast(n_), + static_cast(k_), + static_cast(lda_), + static_cast(ldb_), + static_cast(ldc_), + static_cast(batch_count_), + strideA_, + strideB_, + strideC_, + alpha_, + beta_, + dataType_, + deterministic_}; +} + +class RnnBaseFunctions +{ + RnnBaseFunctions() = default; + +public: + static miopenStatus_t BWD_GEMM_Hidden_Prop(const Handle& handle, + ConstData_t comb_gates_src_ptr, + size_t comb_gates_src_offset, + ConstData_t filter_src_ptr, + size_t filter_offset, + Data_t ht_dst_ptr, + size_t ht_dst_offset, + size_t gemm_batch_size, + size_t ht_dest_vec_size, + size_t comb_gates_size, + size_t tmp_gate_row_stride, + size_t filter_row_stride, + size_t ht_dest_row_stride, + miopenDataType_t data_type, + bool add_assign = true) + { +#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP + // no gemm work + if(gemm_batch_size == 0) + return miopenStatusSuccess; + + const miopen::GemmDescriptor gemm_desc = + GemmDescriptor64BitWraper(false, + false, + false, + gemm_batch_size, + ht_dest_vec_size, + comb_gates_size, + tmp_gate_row_stride, + filter_row_stride, + ht_dest_row_stride, + 1, // batch count + 0, // Stride A + 0, // Stride B + 0, // Stride C + 1, // alpha + add_assign ? 1 : 0, // beta + data_type, + false); + + return CallGemm(handle, + gemm_desc, + comb_gates_src_ptr, + comb_gates_src_offset, + filter_src_ptr, + filter_offset, + ht_dst_ptr, + ht_dst_offset, + GemmBackend_t::rocblas); +#else + return miopenStatusNotImplemented; +#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP + } + + static miopenStatus_t FWD_GEMM(const Handle& handle, + ConstData_t ht_ptr, + size_t ht_offset, + const miopen::TensorDescriptor& ht_dsc, + ConstData_t filter_ptr, + size_t filter_offset, + const miopen::TensorDescriptor& filter_dsc, + Data_t comb_gates_ptr, + size_t comb_gates_offset, + const miopen::TensorDescriptor& tmp_gates_dsc, + bool add_assign = true) + { + + assert(filter_dsc.GetNumDims() == 2 && tmp_gates_dsc.GetNumDims() == 2 && + ht_dsc.GetNumDims() == 2); + + const size_t batch_size = tmp_gates_dsc.GetLengths()[0]; + const size_t comb_gates_size = tmp_gates_dsc.GetLengths()[1]; + const size_t ht_vec_size = ht_dsc.GetLengths()[1]; + + assert(filter_dsc.GetLengths()[0] == comb_gates_size); + assert(filter_dsc.GetLengths()[1] == ht_vec_size); + assert(ht_dsc.GetLengths()[0] == batch_size); + + const size_t tmp_gates_ld_stride = tmp_gates_dsc.GetStrides()[0]; // {batch, comb_gates} + const size_t ht_dest_ld_stride = ht_dsc.GetStrides()[0]; // {batch, ht_vec} + const size_t filter_ld_stride = filter_dsc.GetStrides()[0]; // {comb_gates, ht_vec} + + // no gemm work + if(batch_size == 0) + return miopenStatusSuccess; + + [[maybe_unused]] const miopen::GemmDescriptor gemm_desc = + GemmDescriptor64BitWraper(false, + false, + true, + batch_size, + comb_gates_size, + ht_vec_size, + ht_dest_ld_stride, + filter_ld_stride, + tmp_gates_ld_stride, + 1, // batch count + 0, // Stride A + 0, // Stride B + 0, // Stride C + 1, // alpha + add_assign ? 1 : 0, // beta + ht_dsc.GetType(), + false); +#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP + return CallGemm(handle, + gemm_desc, + ht_ptr, + ht_offset, + filter_ptr, + filter_offset, + comb_gates_ptr, + comb_gates_offset, + GemmBackend_t::rocblas); +#else + + return miopenStatusNotImplemented; +#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP + } + + static miopenStatus_t BWD_GEMM_Hidden_Prop(const Handle& handle, + ConstData_t comb_gates_src_ptr, + size_t comb_gates_src_offset, + const miopen::TensorDescriptor& tmp_gates_src_dsc, + + ConstData_t filter_src_ptr, + size_t filter_src_offset, + const miopen::TensorDescriptor& filter_src_dsc, + + Data_t ht_dst_ptr, + size_t ht_dst_offset, + const miopen::TensorDescriptor& ht_dest_dsc, + bool add_assign = true) + { + assert(filter_src_dsc.GetNumDims() == 2 && tmp_gates_src_dsc.GetNumDims() == 2 && + ht_dest_dsc.GetNumDims() == 2); + + const size_t batch_size = tmp_gates_src_dsc.GetLengths()[0]; + const size_t comb_gates_size = tmp_gates_src_dsc.GetLengths()[1]; + const size_t ht_vec_size = ht_dest_dsc.GetLengths()[1]; + + assert(filter_src_dsc.GetLengths()[0] == comb_gates_size); + assert(filter_src_dsc.GetLengths()[1] == ht_vec_size); + assert(ht_dest_dsc.GetLengths()[0] == batch_size); + + const size_t tmp_gates_ld_stride = tmp_gates_src_dsc.GetStrides()[0]; // {batch, comb_gates} + const size_t filter_ld_stride = filter_src_dsc.GetStrides()[0]; // {comb_gates, ht_vec} + const size_t ht_dest_ld_stride = ht_dest_dsc.GetStrides()[0]; // {batch, ht_vec} + + return BWD_GEMM_Hidden_Prop(handle, + comb_gates_src_ptr, + comb_gates_src_offset, + filter_src_ptr, + filter_src_offset, + ht_dst_ptr, + ht_dst_offset, + batch_size, + ht_vec_size, + comb_gates_size, + tmp_gates_ld_stride, + filter_ld_stride, + ht_dest_ld_stride, + ht_dest_dsc.GetType(), + add_assign); + } + + static miopenStatus_t BWWei_GEMM(const Handle& handle, + ConstData_t comb_gates_ptr, + size_t comb_gates_offset, + const miopen::TensorDescriptor& tmp_gates_dsc, + ConstData_t ht_ptr, + size_t ht_offset, + const miopen::TensorDescriptor& ht_dsc, + Data_t filter_ptr, + size_t filter_offset, + const miopen::TensorDescriptor& filter_dsc, + bool add_assign = true) + { + + assert(filter_dsc.GetNumDims() == 2 && tmp_gates_dsc.GetNumDims() == 2 && + ht_dsc.GetNumDims() == 2); + + const size_t batch_size = tmp_gates_dsc.GetLengths()[0]; + const size_t comb_gates_size = tmp_gates_dsc.GetLengths()[1]; + const size_t ht_vec_size = ht_dsc.GetLengths()[1]; + + assert(filter_dsc.GetLengths()[0] == comb_gates_size); + assert(filter_dsc.GetLengths()[1] == ht_vec_size); + assert(ht_dsc.GetLengths()[0] == batch_size); + + const size_t tmp_gates_ld_stride = tmp_gates_dsc.GetStrides()[0]; // {batch, comb_gates} + const size_t ht_dest_ld_stride = ht_dsc.GetStrides()[0]; // {batch, ht_vec} + const size_t filter_ld_stride = filter_dsc.GetStrides()[0]; // {comb_gates, ht_vec} + + // no gemm work + if(batch_size == 0) + return miopenStatusSuccess; + + [[maybe_unused]] const miopen::GemmDescriptor gemm_desc = + GemmDescriptor64BitWraper(false, + true, + false, + comb_gates_size, + ht_vec_size, + batch_size, + tmp_gates_ld_stride, + ht_dest_ld_stride, + filter_ld_stride, + 1, // batch count + 0, // Stride A + 0, // Stride B + 0, // Stride C + 1, // alpha + add_assign ? 1 : 0, // beta + ht_dsc.GetType(), + false); +#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP + return CallGemm(handle, + gemm_desc, + comb_gates_ptr, + comb_gates_offset, + ht_ptr, + ht_offset, + filter_ptr, + filter_offset, + GemmBackend_t::rocblas); +#else + + return miopenStatusNotImplemented; +#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP + } +}; + +} // namespace rnn_base +} // namespace miopen diff --git a/src/include/miopen/rnn/multi_stream_utils.hpp b/src/include/miopen/rnn/multi_stream_utils.hpp new file mode 100644 index 0000000000..5866c10b34 --- /dev/null +++ b/src/include/miopen/rnn/multi_stream_utils.hpp @@ -0,0 +1,135 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +MIOPEN_DECLARE_ENV_VAR_BOOL(MIOPEN_MS_WA_FIX) + +namespace miopen { + +class MultiStreamController +{ +public: + static constexpr int rootStreamId = 0; + + MultiStreamController(const Handle& handle, int extra_stream_cnt) + : streamPoolIdsMapping{init_stream_pool_ids(handle, extra_stream_cnt)}, + streamPoolCache{init_stream_pool(handle, streamPoolIdsMapping)}, + activeHandle{handle} + { + } + + hipStream_t getStream(int stream_id) const { return streamPoolCache[stream_id]; } + + void ChangeActiveStream(int stream_id) const + { + activeHandle.SetStreamFromPool(streamPoolIdsMapping[stream_id]); + } + + hipError_t RecordEvent(const hipEvent_t event, int stream_id) const + { + return hipEventRecord(event, getStream(stream_id)); + } + + hipError_t SetWaitEvent(const hipEvent_t event, int stream_id) const + { + return hipStreamWaitEvent(getStream(stream_id), event, 0); + } + + void RootWaitToAllStreams() const + { + for(size_t i = 0, stream_cnt = size(); i < stream_cnt; i++) + { + if(i != rootStreamId) + { + const miopen::HipEventPtr sync_event = make_hip_fast_event(); + RecordEvent(sync_event.get(), i); + SetWaitEvent(sync_event.get(), rootStreamId); + } + } + } + + void AllStreamsWaitRoot() const + { + const miopen::HipEventPtr main_event = make_hip_fast_event(); + + RecordEvent(main_event.get(), rootStreamId); + + for(size_t i = 0, stream_cnt = size(); i < stream_cnt; i++) + { + if(i != rootStreamId) + SetWaitEvent(main_event.get(), i); + } + }; + + size_t size() const { return streamPoolIdsMapping.size(); } + + ~MultiStreamController() { ChangeActiveStream(rootStreamId); } + +private: + static std::vector init_stream_pool_ids(const Handle& handle, int extra_stream_cnt) + { + std::vector ids; + ids.reserve(extra_stream_cnt + 1); + + bool ms_wa_fix_active = extra_stream_cnt > 2 && !env::disabled(MIOPEN_MS_WA_FIX); + handle.SetStreamFromPool(0); + int wa_steams = ms_wa_fix_active ? (handle.GetStream() == nullptr ? 3 : 2) : 0; + + handle.ReserveExtraStreamsInPool(extra_stream_cnt + wa_steams); + + for(int i = 0; i <= extra_stream_cnt + wa_steams; i++) + if(!(ms_wa_fix_active && (i > 0 && i <= wa_steams))) + ids.push_back(i); + + return ids; + } + + static std::vector init_stream_pool(const Handle& handle, + const std::vector& pool_ids) + { + std::vector pool; + pool.reserve(pool_ids.size()); + + for(auto id : pool_ids) + { + handle.SetStreamFromPool(id); + pool.push_back(handle.GetStream()); + } + handle.SetStreamFromPool(0); + + return pool; + } + + const std::vector streamPoolIdsMapping; + const std::vector streamPoolCache; + const Handle& activeHandle; +}; + +} // namespace miopen diff --git a/src/include/miopen/rnn/solvers.hpp b/src/include/miopen/rnn/solvers.hpp new file mode 100644 index 0000000000..429bcee752 --- /dev/null +++ b/src/include/miopen/rnn/solvers.hpp @@ -0,0 +1,1097 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include +#include "miopen/rnn/tmp_buffer_utils.hpp" + +namespace miopen { + +namespace rnn_base { + +struct runtimeArgsFwd +{ + const ConstData_t x; + const ConstData_t hx; + const ConstData_t cx; + const Data_t y; + const Data_t hy; + const Data_t cy; + const ConstData_t w; + const Data_t workSpace; + const Data_t reserveSpace; +}; + +class RNNModuleAlgoBase +{ + +public: + static RNNModuleAlgoBase create(const RNNDescriptor& rnnDesc, + const SeqTensorDescriptor& xDesc, + const SeqTensorDescriptor& yDesc, + const TensorDescriptor& hDesc, + miopenRNNFWDMode_t mode) + { + auto [max_layers_hid, max_batch_hid, hidden_vec_sz] = miopen::tien<3>(hDesc.GetLengths()); + auto [max_batch_in, max_seq, input_vec_sz] = miopen::tien<3>(xDesc.GetLengths()); + + assert(max_batch_in <= max_batch_hid); + + auto layers_cnt = static_cast(rnnDesc.nLayers); + const bool is_seq_bidir = rnnDesc.dirMode == miopenRNNbidirection; + + assert(static_cast(layers_cnt) * (is_seq_bidir ? 2 : 1) <= max_layers_hid); + + auto gates_cnt = static_cast(rnnDesc.nHiddenTensorsPerLayer); + + // class update req + assert(!is_seq_bidir); + const size_t seq_directions = is_seq_bidir ? 2 : 1; + + // TODO all size_t + GeneralLstmRedBuffer rb_layout = GeneralLstmRedBuffer::build( + layers_cnt, xDesc.GetTotalSequenceLen(), seq_directions, hidden_vec_sz); + + GeneralLstmTempBuffer workspace_info = GeneralLstmTempBuffer::build( + layers_cnt, xDesc.GetTotalSequenceLen(), seq_directions, hidden_vec_sz); + + WeightsBufferDescriptor weights_layout = + WeightsBufferDescriptor::create(static_cast(input_vec_sz), + static_cast(hidden_vec_sz), + layers_cnt, + rnnDesc.biasMode, + rnnDesc.inputMode, + gates_cnt, + is_seq_bidir); + + BatchController batch_controller = BatchController::Create(xDesc); + + HiddenBuffersDescriptor hidden_hxcx_info{hDesc}; + + IOBufferDescriptor x_info{IOBufferDescriptor::build(xDesc)}; + IOBufferDescriptor y_info{IOBufferDescriptor::build(yDesc)}; + + return {std::move(rb_layout), + workspace_info, + weights_layout, + hidden_hxcx_info, + x_info, + y_info, + rnnDesc, + batch_controller, + mode}; + } + + RNNModuleAlgoBase(RNNModuleAlgoBase&&) = default; + // RNNModuleAlgoBase(RNNModuleAlgoBase const&) = default; + + RNNModuleAlgoBase(GeneralLstmRedBuffer rb_layout, + GeneralLstmTempBuffer workspace_info, + WeightsBufferDescriptor weights_layout, + HiddenBuffersDescriptor hidden_hxcx_info, + IOBufferDescriptor x_info, + IOBufferDescriptor y_info, + const RNNDescriptor& rnn_desc, + BatchController batch_controller, + miopenRNNFWDMode_t fwd_mode) + : reservLayout(std::move(rb_layout)), + workspaceInfo(std::move(workspace_info)), + weightsLayout(std::move(weights_layout)), + hiddenHxCxInfo(std::move(hidden_hxcx_info)), + xInfo(std::move(x_info)), + yInfo(std::move(y_info)), + rnnDesc(rnn_desc), + tanhDesc{miopenActivationTANH, 1, 1, 1}, + sigDesc{miopenActivationLOGISTIC, 1, 0, 1}, + reluDesc{miopenActivationRELU, 1, 0, 1}, + batchController(std::move(batch_controller)), + fwdMode(fwd_mode), + isBidirectSeq(false) + { + } + + const GeneralLstmRedBuffer reservLayout; + // const WorkspaceBufferDescriptor workspaceInfo; + const GeneralLstmTempBuffer workspaceInfo; + + const WeightsBufferDescriptor weightsLayout; + const HiddenBuffersDescriptor hiddenHxCxInfo; + + const IOBufferDescriptor xInfo; + const IOBufferDescriptor yInfo; + + const RNNDescriptor& rnnDesc; + + const ActivationDescriptor tanhDesc; + const ActivationDescriptor sigDesc; + const ActivationDescriptor reluDesc; + + const BatchController batchController; + + const miopenRNNFWDMode_t fwdMode; + + const bool isBidirectSeq; + + inline size_t getVirtualLayer(const size_t layer_id, SequenceDirection direction) const + { + return layer_id * (isBidirectSeq ? 2 : 1) + + (direction == SequenceDirection::Forward ? 0 : 1); + } +}; + +class RNNForwardDataModularAlgo : RNNModuleAlgoBase +{ +public: + // Compute API + // base API + void PrepareWriteBuffers(const Handle& handle, const runtimeArgsFwd& runtimeArgs) const; + + void PropX(const Handle& handle, const runtimeArgsFwd& runtimeArgs) const; + + void AddBias(const Handle& handle, const runtimeArgsFwd& runtimeArgs) const; + void PropHxCx(const Handle& handle, + const runtimeArgsFwd& runtimeArgs, + unsigned int layer, + const SequenceIterator& currentSeq, + SequenceDirection direction) const; + + void PropHiddenHt(const Handle& handle, + const runtimeArgsFwd& runtimeArgs, + int layer, + const SequenceIterator& currentSeq, + SequenceDirection direction) const; + + void UpdateHStatePerTimeSeq(const Handle& handle, + const runtimeArgsFwd& runtimeArgs, + int layer, + const SequenceIterator& seq, + SequenceDirection direction) const; + + void PropHyCy(const Handle& handle, + const runtimeArgsFwd& runtimeArgs, + size_t layer, + const SequenceIterator& currentSeq, + SequenceDirection direction) const; + + void PropHiddenY(const Handle& handle, + const runtimeArgsFwd& runtimeArgs, + size_t layer, + SequenceDirection direction) const; + + void PropY(const Handle& handle, const runtimeArgsFwd& runtimeArgs) const; + + // ext API + void PropX(const Handle& handle, + const runtimeArgsFwd& runtimeArgs, + size_t gemm_batch_offset, + size_t gemm_batch_size) const; + + void PropHiddenY(const Handle& handle, + const runtimeArgsFwd& runtimeArgs, + size_t layer, + SequenceDirection direction, + const SequenceIterator& firstSeq, + const SequenceIterator& lastSeq) const; + + void PropHiddenY(const Handle& handle, + const runtimeArgsFwd& runtimeArgs, + size_t layer, + SequenceDirection direction, + size_t gemm_batch_size, + size_t gemm_batch_offset) const; + + void PropX(const Handle& handle, + const runtimeArgsFwd& runtimeArgs, + SequenceDirection direction, + const SequenceIterator& firstSeq, + const SequenceIterator& lastSeq) const; + + void PropX(const Handle& handle, + const runtimeArgsFwd& runtimeArgs, + SequenceDirection direction) const; + + void PropX(const Handle& handle, + const runtimeArgsFwd& runtimeArgs, + SequenceDirection direction, + size_t gemm_batch_offset, + size_t gemm_batch_size) const; + + /// end compute API + + static bool IsApplicable() + { +#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP + return true; +#else + return false; +#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP + } + + RNNForwardDataModularAlgo(RNNModuleAlgoBase base) : RNNModuleAlgoBase(std::move(base)) {} + +private: + template + inline miopen::TensorDescriptor BuildLstmTmpBlockDesc2D(const BufType& buf_info, + const size_t batch_size) const + { + const std::array& tmp_block_stride = buf_info.getGateBlockStride(); + const std::array& tmp_block_size = buf_info.getGateBlockSize(); + + // batch, gateBlock_elements + return miopen::TensorDescriptor{rnnDesc.dataType, + {batch_size, tmp_block_size[3]}, + {tmp_block_stride[1], tmp_block_stride[3]}}; + } + + inline miopen::TensorDescriptor BuildLstmFilterXDesc2D(int layer_id) const + { + assert(rnnDesc.inputMode == 0 || layer_id != 0); + // TODO replace by stride + auto x_vec = layer_id != 0 ? weightsLayout.xInVec : weightsLayout.inVec; + + // gateBlock_elements, ht_vec + return miopen::TensorDescriptor{ + rnnDesc.dataType, {weightsLayout.gatesCnt * weightsLayout.hVec, x_vec}, {x_vec, 1}}; + } + + inline miopen::TensorDescriptor BuildLstmFilterHidDesc2D() const + { + // TODO replace by stride + auto h_vec = weightsLayout.hVec; + + // gateBlock_elements, ht_vec + return miopen::TensorDescriptor{ + rnnDesc.dataType, {weightsLayout.gatesCnt * weightsLayout.hVec, h_vec}, {h_vec, 1}}; + } + + inline miopen::TensorDescriptor BuildWsHtDesc2D(size_t batch_size) const + { + auto& ht_stride = workspaceInfo.getHiddenStateStride(); + auto& ht_size = workspaceInfo.hStateSizes; + + // batch, gateBlock_elements + return miopen::TensorDescriptor{ + rnnDesc.dataType, {batch_size, ht_size[3]}, {ht_stride[1], ht_stride[3]}}; + } + + // 2 dims batch, vec + inline miopen::TensorDescriptor BuildHxCxDesc2D(size_t batch_size) const + { + const std::vector hx_size{batch_size, hiddenHxCxInfo.getHiddenSize()}; + const std::vector hx_stride{hiddenHxCxInfo.getStrides()[1], + hiddenHxCxInfo.getStrides()[2]}; + + return miopen::TensorDescriptor{rnnDesc.dataType, hx_size, hx_stride}; + } + + // 3 dims layer, batch, vec + inline miopen::TensorDescriptor BuildHxCxDesc3D(size_t layer_size, size_t batch_size) const + { + const std::vector hx_accum_size{ + layer_size, batch_size, hiddenHxCxInfo.getHiddenSize()}; + + return miopen::TensorDescriptor{ + rnnDesc.dataType, hx_accum_size, hiddenHxCxInfo.getStrides()}; + } + + // 3 dims layer, batch, vec + inline miopen::TensorDescriptor BuildTempDhtDesc3D(size_t layer_size, size_t batch_size) const + { + const std::vector dy_dhy_accum_size{ + layer_size, batch_size, hiddenHxCxInfo.getHiddenSize()}; + + const auto ws_dy_stride = [](const auto& ws_4dim_strides) -> std::vector { + // convert 4dim stride to 3 dim without direction + // TODO change hiddenBufferDesc + return std::vector{ws_4dim_strides[0], ws_4dim_strides[1], ws_4dim_strides[3]}; + }(workspaceInfo.getHiddenStateStride()); + + return miopen::TensorDescriptor{rnnDesc.dataType, dy_dhy_accum_size, ws_dy_stride}; + } +}; + +class RNNBackwardDataModularAlgo : RNNModuleAlgoBase +{ +public: + void PrepareWriteBuffers(const Handle& handle, Data_t dhx, Data_t dcx, Data_t workSpace) const; + + void PropDhy(const Handle& handle, + ConstData_t dhy, + Data_t workSpace, + unsigned int layer, + const SequenceIterator& currentSeq, + SequenceDirection direction) const; + + void PropHiddenDht(const Handle& handle, + ConstData_t w, + Data_t workSpace, + int layer, + const SequenceIterator& currentSeq, + SequenceDirection direction) const; + + void UpdateHStatePerTimeSeq(const Handle& handle, + ConstData_t dcy, + ConstData_t cx, + Data_t, + Data_t workSpace, + Data_t reserveSpace, + int layer, + const SequenceIterator& seq, + SequenceDirection direction) const; + + void PropDhxDcx(const Handle& handle, + ConstData_t w, + Data_t dhx, + Data_t dcx, + Data_t workSpace, + Data_t reserveSpace, + size_t layer, + const SequenceIterator& currentSeq, + SequenceDirection direction) const; + + void PropDy(const Handle& handle, ConstData_t dy, Data_t workSpace) const; + + void PropHiddenDy(const Handle& handle, + ConstData_t w, + Data_t workSpace, + Data_t reserveSpace, + size_t layer, + SequenceDirection direction) const; + + void PropHiddenDy(const Handle& handle, + ConstData_t w, + Data_t workSpace, + Data_t reserveSpace, + size_t layer, + SequenceDirection direction, + const SequenceIterator& firstSeq, + const SequenceIterator& lastSeq) const; + + void PropHiddenDy(const Handle& handle, + ConstData_t w, + Data_t workSpace, + Data_t reserveSpace, + size_t layer, + SequenceDirection direction, + size_t gemm_batch_size, + size_t gemm_batch_offset) const; + + void PropDx(const Handle& handle, + ConstData_t w, + ConstData_t workSpace, + Data_t dx, + SequenceDirection direction, + const SequenceIterator& firstSeq, + const SequenceIterator& lastSeq) const; + + void PropDx(const Handle& handle, + ConstData_t w, + ConstData_t workSpace, + Data_t dx, + SequenceDirection direction) const; + + void PropDx(const Handle& handle, + ConstData_t w, + ConstData_t workSpace, + Data_t dx, + SequenceDirection direction, + size_t gemm_batch_offset, + size_t gemm_batch_size) const; + static bool IsApplicable() + { +#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP + return true; +#else + return false; +#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP + } + + RNNBackwardDataModularAlgo(RNNModuleAlgoBase&& base) : RNNModuleAlgoBase(std::move(base)) {} + +private: + template + inline miopen::TensorDescriptor BuildLstmTmpBlockDesc2D(const BufType& buf_info, + const size_t batch_size) const + { + const std::array& tmp_block_stride = buf_info.getGateBlockStride(); + const std::array& tmp_block_size = buf_info.getGateBlockSize(); + + // batch, gateBlock_elements + return miopen::TensorDescriptor{rnnDesc.dataType, + {batch_size, tmp_block_size[3]}, + {tmp_block_stride[1], tmp_block_stride[3]}}; + } + + inline miopen::TensorDescriptor BuildLstmFilterXDesc2D(int layer_id) const + { + assert(rnnDesc.inputMode == 0 || layer_id != 0); + // TODO replace by stride + auto x_vec = layer_id != 0 ? weightsLayout.xInVec : weightsLayout.inVec; + + // gateBlock_elements, ht_vec + return miopen::TensorDescriptor{ + rnnDesc.dataType, {weightsLayout.gatesCnt * weightsLayout.hVec, x_vec}, {x_vec, 1}}; + } + + inline miopen::TensorDescriptor BuildLstmFilterHidDesc2D() const + { + // TODO replace by stride + auto h_vec = weightsLayout.hVec; + + // gateBlock_elements, ht_vec + return miopen::TensorDescriptor{ + rnnDesc.dataType, {weightsLayout.gatesCnt * weightsLayout.hVec, h_vec}, {h_vec, 1}}; + } + + inline miopen::TensorDescriptor BuildWsHtDesc2D(size_t batch_size) const + { + auto& ht_stride = workspaceInfo.getHiddenStateStride(); + auto& ht_size = workspaceInfo.hStateSizes; + + // batch, gateBlock_elements + return miopen::TensorDescriptor{ + rnnDesc.dataType, {batch_size, ht_size[3]}, {ht_stride[1], ht_stride[3]}}; + } + + // 2 dims batch, vec + inline miopen::TensorDescriptor BuildHxCxDesc2D(size_t batch_size) const + { + const std::vector hx_size{batch_size, hiddenHxCxInfo.getHiddenSize()}; + const std::vector hx_stride{hiddenHxCxInfo.getStrides()[1], + hiddenHxCxInfo.getStrides()[2]}; + + return miopen::TensorDescriptor{rnnDesc.dataType, hx_size, hx_stride}; + } + + // 3 dims layer, batch, vec + inline miopen::TensorDescriptor BuildHxCxDesc3D(size_t layer_size, size_t batch_size) const + { + const std::vector hx_accum_size{ + layer_size, batch_size, hiddenHxCxInfo.getHiddenSize()}; + + return miopen::TensorDescriptor{ + rnnDesc.dataType, hx_accum_size, hiddenHxCxInfo.getStrides()}; + } + + // 3 dims layer, batch, vec + inline miopen::TensorDescriptor BuildTempDhtDesc3D(size_t layer_size, size_t batch_size) const + { + const std::vector dy_dhy_accum_size{ + layer_size, batch_size, hiddenHxCxInfo.getHiddenSize()}; + + const auto ws_dy_stride = [](const auto& ws_4dim_strides) -> std::vector { + // convert 4dim stride to 3 dim without direction + // TODO change hiddenBufferDesc + return std::vector{ws_4dim_strides[0], ws_4dim_strides[1], ws_4dim_strides[3]}; + }(workspaceInfo.getHiddenStateStride()); + + return miopen::TensorDescriptor{rnnDesc.dataType, dy_dhy_accum_size, ws_dy_stride}; + } + + inline size_t getVirtualLayer(const size_t layer_id, SequenceDirection direction) const + { + return layer_id * (isBidirectSeq ? 2 : 1) + + (direction == SequenceDirection::Forward ? 0 : 1); + } +}; + +class RNNModularSingleStreamFWD +{ +public: + RNNModularSingleStreamFWD(const RNNDescriptor& rnn, + const SeqTensorDescriptor& xDesc, + const SeqTensorDescriptor& yDesc, + const TensorDescriptor& hDesc, + miopenRNNFWDMode_t mode) + : rnnAlgoModules(RNNModuleAlgoBase::create(rnn, xDesc, yDesc, hDesc, mode)), + rnnDesc(rnn), + max_seq_len(xDesc.GetMaxSequenceLength()) + { + } + + static bool IsApplicable() + { +#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP + return true; +#else + return false; +#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP + } + + // TODO + static size_t GetWsSize() { return 0; }; + + void ComputeFWD(Handle& handle, const runtimeArgsFwd& runtimeArgs) const; + + const rnn_base::RNNForwardDataModularAlgo rnnAlgoModules; + + const RNNDescriptor& rnnDesc; + const size_t max_seq_len; +}; + +class RNNModularSingleStreamBWD +{ +public: + RNNModularSingleStreamBWD(const RNNDescriptor& rnn, + const SeqTensorDescriptor& xDesc, + const SeqTensorDescriptor& yDesc, + const TensorDescriptor& hDesc, + miopenRNNFWDMode_t mode) + : rnnAlgoModules(RNNModuleAlgoBase::create(rnn, xDesc, yDesc, hDesc, mode)), + rnnDesc(rnn), + max_seq_len(xDesc.GetMaxSequenceLength()) + { + } + + static bool IsApplicable() + { +#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP + return true; +#else + return false; +#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP + } + + // TODO + static size_t GetWsSize() { return 0; }; + + void ComputeBWD(Handle& handle, + ConstData_t dy, + ConstData_t dhy, + Data_t dhx, + ConstData_t cx, + ConstData_t dcy, + Data_t dcx, + Data_t dx, + ConstData_t w, + Data_t workSpace, + Data_t reserveSpace) const; + + const rnn_base::RNNBackwardDataModularAlgo rnnAlgoModules; + const RNNDescriptor& rnnDesc; + const size_t max_seq_len; +}; + +class RNNModularMultiStreamBWD +{ +public: + RNNModularMultiStreamBWD(const RNNDescriptor& rnn, + const SeqTensorDescriptor& xDesc, + const SeqTensorDescriptor& yDesc, + const TensorDescriptor& hDesc, + miopenRNNFWDMode_t mode) + : rnnAlgoModules(RNNModuleAlgoBase::create(rnn, xDesc, yDesc, hDesc, mode)), + rnnDesc(rnn), + max_seq_len(xDesc.GetMaxSequenceLength()) + { + } + + static bool IsApplicable() + { +#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP + return true; +#else + return false; +#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP + } + + // TODO + static size_t GetWsSize() { return 0; }; + + struct runtimeArgsBwd + { + const Handle* handle; + ConstData_t dy; + ConstData_t dhy; + Data_t dhx; + ConstData_t cx; + ConstData_t dcy; + Data_t dcx; + Data_t dx; + ConstData_t w; + Data_t workSpace; + Data_t reserveSpace; + }; + + void ComputeBWD(Handle& handle, + ConstData_t dy, + ConstData_t dhy, + Data_t dhx, + ConstData_t cx, + ConstData_t dcy, + Data_t dcx, + Data_t dx, + ConstData_t w, + Data_t workSpace, + Data_t reserveSpace) const; + + bool ChunkDispatch(const runtimeArgsBwd& args, + size_t chunk_size, + size_t chunk_time_offset, + size_t chunk_layer_offset) const; + +private: + void PrologueDispatch(const runtimeArgsBwd& args) const; + + const rnn_base::RNNBackwardDataModularAlgo rnnAlgoModules; + const RNNDescriptor& rnnDesc; + const size_t max_seq_len; +}; + +class RNNBackwardWeightsModularAlgo +{ +public: + static RNNBackwardWeightsModularAlgo create(const RNNDescriptor& rnnDesc, + const SeqTensorDescriptor& xDesc, + const SeqTensorDescriptor& yDesc, + const TensorDescriptor& hDesc) + { + auto [max_layers_hid, max_batch_hid, hidden_vec_sz] = miopen::tien<3>(hDesc.GetLengths()); + auto [max_batch_in, max_seq, input_vec_sz] = miopen::tien<3>(xDesc.GetLengths()); + + assert(max_batch_in <= max_batch_hid); + + size_t layers_cnt = rnnDesc.nLayers; + const bool is_seq_bidir = rnnDesc.dirMode == miopenRNNbidirection; + + assert(layers_cnt * (is_seq_bidir ? 2 : 1) <= max_layers_hid); + + auto gates_cnt = static_cast(rnnDesc.nHiddenTensorsPerLayer); + + // class update req + assert(!is_seq_bidir); + const size_t seq_directions = is_seq_bidir ? 2 : 1; + + GeneralLstmRedBuffer rb_layout = GeneralLstmRedBuffer::build( + layers_cnt, xDesc.GetTotalSequenceLen(), seq_directions, hidden_vec_sz); + + GeneralLstmTempBuffer workspace_info = GeneralLstmTempBuffer::build( + layers_cnt, xDesc.GetTotalSequenceLen(), seq_directions, hidden_vec_sz); + + WeightsBufferDescriptor weights_layout = WeightsBufferDescriptor::create(input_vec_sz, + hidden_vec_sz, + layers_cnt, + rnnDesc.biasMode, + rnnDesc.inputMode, + gates_cnt, + is_seq_bidir); + + BatchController batch_controller = BatchController::Create(xDesc); + + HiddenBuffersDescriptor hidden_hxcx_info{hDesc}; + + IOBufferDescriptor x_info{IOBufferDescriptor::build(xDesc)}; + IOBufferDescriptor y_info{IOBufferDescriptor::build(yDesc)}; + + return {std::move(rb_layout), + workspace_info, + weights_layout, + hidden_hxcx_info, + x_info, + y_info, + rnnDesc, + batch_controller}; + } + + void PrepareWriteBuffers(const Handle& handle, Data_t w) const; + + void PhisXInputWeights(const Handle& handle, Data_t dw, Data_t workSpace, ConstData_t x) const; + + void HiddenXInputWeights(const Handle& handle, + Data_t dw, + ConstData_t workSpace, + ConstData_t reserveSpace, + size_t layer) const; + + void BiasUpdate(const Handle& handle, + Data_t dw, + Data_t workSpace, + size_t layer, + size_t workSpaceSize) const; + + void HiddenHStateWeights(const Handle& handle, + Data_t dw, + ConstData_t workSpace, + ConstData_t reserveSpace, + const SequenceIterator& seq, + size_t layer, + SequenceDirection direction) const + { + const size_t gemm_batch_size = [&]() -> size_t { + if(seq.isFirst()) + return 0; + + if(direction == SequenceDirection::Reverse) + return batchController.getBatchSize(seq.getPhisVal()); + else + return batchController.getBatchSize(seq.getPrev().getPhisVal()); + }(); + + if(gemm_batch_size != 0) + return HiddenHStateWeights_Unchecked( + handle, dw, workSpace, reserveSpace, seq, layer, direction, gemm_batch_size); + } + + void HiddenHStateWeights(const Handle& handle, + Data_t dw, + ConstData_t workSpace, + ConstData_t reserveSpace, + size_t layer, + size_t max_seq_len, + const SequenceDirection direction) const + { + size_t start_seq_id = 0; + const size_t last_seq = max_seq_len - 1; + for(auto i = start_seq_id + 1; i <= last_seq; i++) + { + + if(batchController.getBatchSize(i) != batchController.getBatchSize(start_seq_id) || + i == last_seq) + { + const size_t gemm_batch_size = (batchController.getBatchSum(i - 1) - + batchController.getBatchSum(start_seq_id)) + + batchController.getBatchSize(i); + + if(gemm_batch_size != 0) + { + const auto first_logical_val = direction == SequenceDirection::Forward + ? start_seq_id + : (max_seq_len - 1) - start_seq_id - 1; + const auto seq = + SequenceIterator(first_logical_val, direction, max_seq_len, false); + + HiddenHStateWeights_Unchecked(handle, + dw, + workSpace, + reserveSpace, + seq, + layer, + direction, + gemm_batch_size); + } + start_seq_id = i; + } + } + } + + void PhisHStateWeights(const Handle& handle, + Data_t dw, + ConstData_t workSpace, + ConstData_t hx, + size_t layer, + size_t max_seq_len, + SequenceDirection direction) const + { + if(hx == nullptr) + return; + + for(auto i = max_seq_len; i > 0; i--) + { + const auto seq = SequenceIterator(i - 1, direction, max_seq_len, false); + + PhisHStateWeights(handle, dw, workSpace, hx, seq, layer, direction); + } + } + + static bool IsApplicable() + { +#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP + return true; +#else + return false; +#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP + } + +private: + RNNBackwardWeightsModularAlgo(GeneralLstmRedBuffer rb_layout, + GeneralLstmTempBuffer workspace_info, + WeightsBufferDescriptor weights_layout, + HiddenBuffersDescriptor hidden_hxcx_info, + IOBufferDescriptor x_info, + IOBufferDescriptor y_info, + const RNNDescriptor& rnn_desc, + BatchController batch_controller) + : reservLayout(std::move(rb_layout)), + workspaceInfo(std::move(workspace_info)), + weightsLayout(std::move(weights_layout)), + hiddenHxCxInfo(std::move(hidden_hxcx_info)), + xInfo(std::move(x_info)), + yInfo(std::move(y_info)), + rnnDesc(rnn_desc), + batchController(std::move(batch_controller)) + { + } + + void HiddenHStateWeights_Unchecked(const Handle& handle, + Data_t dw, + ConstData_t workSpace, + ConstData_t reserveSpace, + const SequenceIterator& seq, + size_t layer, + SequenceDirection direction, + size_t gemm_batch_size) const; + + void PhisHStateWeights(const Handle& handle, + Data_t dw, + ConstData_t workSpace, + ConstData_t hx, + const SequenceIterator& seq, + size_t layer, + SequenceDirection direction) const; + + template + inline miopen::TensorDescriptor BuildLstmTmpBlockDesc2D(const BufType& buf_info, + const size_t batch_size) const + { + const std::array& tmp_block_stride = buf_info.getGateBlockStride(); + const std::array& tmp_block_size = buf_info.getGateBlockSize(); + + // batch, gateBlock_elements + return miopen::TensorDescriptor{rnnDesc.dataType, + {batch_size, tmp_block_size[3]}, + {tmp_block_stride[1], tmp_block_stride[3]}}; + } + + inline miopen::TensorDescriptor BuildLstmFilterXDesc2D(int layer_id) const + { + assert(rnnDesc.inputMode == 0 || layer_id != 0); + // TODO replace by stride + auto x_vec = layer_id != 0 ? weightsLayout.xInVec : weightsLayout.inVec; + + // gateBlock_elements, ht_vec + return miopen::TensorDescriptor{ + rnnDesc.dataType, {weightsLayout.gatesCnt * weightsLayout.hVec, x_vec}, {x_vec, 1}}; + } + + inline miopen::TensorDescriptor BuildLstmFilterHidDesc2D() const + { + // TODO replace by stride + auto h_vec = weightsLayout.hVec; + + // gateBlock_elements, ht_vec + return miopen::TensorDescriptor{ + rnnDesc.dataType, {weightsLayout.gatesCnt * weightsLayout.hVec, h_vec}, {h_vec, 1}}; + } + + template + inline miopen::TensorDescriptor BuildTmpHtDesc2D(const BufType& buf_info, + size_t batch_size) const + { + auto& ht_stride = buf_info.getHiddenStateStride(); + auto& ht_size = buf_info.hStateSizes; + + // batch, gateBlock_elements + return miopen::TensorDescriptor{ + rnnDesc.dataType, {batch_size, ht_size[3]}, {ht_stride[1], ht_stride[3]}}; + } + + // 2 dims batch, vec + inline miopen::TensorDescriptor BuildHxCxDesc2D(size_t batch_size) const + { + const std::vector hx_size{batch_size, hiddenHxCxInfo.getHiddenSize()}; + const std::vector hx_stride{hiddenHxCxInfo.getStrides()[1], + hiddenHxCxInfo.getStrides()[2]}; + + return miopen::TensorDescriptor{rnnDesc.dataType, hx_size, hx_stride}; + } + + // 3 dims layer, batch, vec + inline miopen::TensorDescriptor BuildHxCxDesc3D(size_t layer_size, size_t batch_size) const + { + const std::vector hx_accum_size{ + layer_size, batch_size, hiddenHxCxInfo.getHiddenSize()}; + + return miopen::TensorDescriptor{ + rnnDesc.dataType, hx_accum_size, hiddenHxCxInfo.getStrides()}; + } + + // 3 dims layer, batch, vec + inline miopen::TensorDescriptor BuildTempDhtDesc3D(size_t layer_size, size_t batch_size) const + { + const std::vector dy_dhy_accum_size{ + layer_size, batch_size, hiddenHxCxInfo.getHiddenSize()}; + + const auto ws_dy_stride = [](const auto& ws_4dim_strides) -> std::vector { + // convert 4dim stride to 3 dim without direction + // TODO change hiddenBufferDesc + return std::vector{ws_4dim_strides[0], ws_4dim_strides[1], ws_4dim_strides[3]}; + }(workspaceInfo.getHiddenStateStride()); + + return miopen::TensorDescriptor{rnnDesc.dataType, dy_dhy_accum_size, ws_dy_stride}; + } + + // 3 dims layer, batch, vec + inline miopen::TensorDescriptor BuildWeiBiasDesc2D() const + { + const std::vector bias_size = [](const auto& wei_4dim_size) -> std::vector { + // wei_4dim_size{layer, dir, gate, vec} + return {1, wei_4dim_size[1] * wei_4dim_size[2] * wei_4dim_size[3]}; + }(weightsLayout.getBiasSize()); + + const auto bias_stride = [](const auto& wei_4dim_strides) -> std::vector { + // convert 4dim stride to 2 dim without direction + return std::vector{wei_4dim_strides[0], wei_4dim_strides[3]}; + }(weightsLayout.getBiasStride()); + + return miopen::TensorDescriptor{rnnDesc.dataType, bias_size, bias_stride}; + } + + inline size_t getVirtualLayer(const size_t layer_id, SequenceDirection direction) const + { + return layer_id * (isBidirectSeq ? 2 : 1) + + (direction == SequenceDirection::Forward ? 0 : 1); + } + + inline size_t getHxBatchSizeReadAtTime(const SequenceIterator& seq, + SequenceDirection direction) const + { + if(seq.isLast()) + return batchController.getBatchSize(seq.getPhisVal()); + + if(direction == SequenceDirection::Reverse) + { + return batchController.getBatchSize(seq.getPhisVal()) - + batchController.getBatchSize(seq.getPrev().getPhisVal()); + } + return 0; + } + + const GeneralLstmRedBuffer reservLayout; + // const WorkspaceBufferDescriptor workspaceInfo; + const GeneralLstmTempBuffer workspaceInfo; + + const WeightsBufferDescriptor weightsLayout; + const HiddenBuffersDescriptor hiddenHxCxInfo; + + const IOBufferDescriptor xInfo; + const IOBufferDescriptor yInfo; + + const RNNDescriptor& rnnDesc; + + const ActivationDescriptor tanhDesc = {miopenActivationTANH, 1, 1, 1}; + const ActivationDescriptor sigDesc = {miopenActivationLOGISTIC, 1, 0, 1}; + const ActivationDescriptor reluDesc = {miopenActivationRELU, 1, 0, 1}; + + const BatchController batchController; + + const bool isBidirectSeq = false; +}; + +class RNNModularSingleStreamBWWeights +{ +public: + RNNModularSingleStreamBWWeights(const RNNDescriptor& rnn, + const SeqTensorDescriptor& xDesc, + const SeqTensorDescriptor& yDesc, + const TensorDescriptor& hDesc) + : rnnAlgoModules(RNNBackwardWeightsModularAlgo::create(rnn, xDesc, yDesc, hDesc)), + rnnDesc(rnn), + max_seq_len(xDesc.GetMaxSequenceLength()) + { + } + + static bool IsApplicable() + { +#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP + return true; +#else + return false; +#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP + } + + // TODO + static size_t GetWsSize() { return 0; }; + + void Compute(const Handle& handle, + ConstData_t x, + ConstData_t hx, + Data_t dw, + Data_t workSpace, + size_t /*workSpaceSize*/, + ConstData_t reserveSpace, + size_t /*reserveSpaceSize*/) const; + + const rnn_base::RNNBackwardWeightsModularAlgo rnnAlgoModules; + const RNNDescriptor& rnnDesc; + const size_t max_seq_len; +}; + +class RNNModularMultiStreamBWWeights +{ +public: + RNNModularMultiStreamBWWeights(const RNNDescriptor& rnn, + const SeqTensorDescriptor& xDesc, + const SeqTensorDescriptor& yDesc, + const TensorDescriptor& hDesc) + : rnnAlgoModules(RNNBackwardWeightsModularAlgo::create(rnn, xDesc, yDesc, hDesc)), + rnnDesc(rnn), + max_seq_len(xDesc.GetMaxSequenceLength()) + { + } + + static bool IsApplicable() + { +#if MIOPEN_USE_GEMM && MIOPEN_BACKEND_HIP + return true; +#else + return false; +#endif // MIOPEN_USE_GEMM&& MIOPEN_BACKEND_HIP + } + + // TODO + static size_t GetWsSize() { return 0; }; + + struct runtimeArgsBww + { + const Handle* handle; + ConstData_t x; + ConstData_t hx; + Data_t dw; + Data_t workSpace; + ConstData_t reserveSpace; + }; + + void Compute(const Handle& handle, + ConstData_t x, + ConstData_t hx, + Data_t dw, + Data_t workSpace, + size_t /*workSpaceSize*/, + ConstData_t reserveSpace, + size_t /*reserveSpaceSize*/) const; + +private: + void PrologueDispatch(const runtimeArgsBww& args) const; + + const rnn_base::RNNBackwardWeightsModularAlgo rnnAlgoModules; + const RNNDescriptor& rnnDesc; + const size_t max_seq_len; +}; + +} // namespace rnn_base +} // namespace miopen diff --git a/src/include/miopen/rnn/tmp_buffer_utils.hpp b/src/include/miopen/rnn/tmp_buffer_utils.hpp new file mode 100644 index 0000000000..39806aa73d --- /dev/null +++ b/src/include/miopen/rnn/tmp_buffer_utils.hpp @@ -0,0 +1,930 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +#include +#include + +#include +#include +#include +#include + +#include +#include + +namespace miopen { + +// https://github.com/ROCm/MIOpen/issues/3048 +namespace WA_RHEL { + +template +OutputIt exclusive_scan_wa(InputIt first, InputIt last, OutputIt d_first, T init, BinaryOp op) +{ + if(first == last) + return d_first; + + auto acc = init; + + for(; first != last; ++first) + { + *d_first++ = acc; + acc = op(std::move(acc), *first); // std::move since C++11 + } + + return d_first; +} + +} // namespace WA_RHEL + +namespace rnn_base { + +enum class SequenceDirection +{ + Forward = 0, + Reverse = 1 +}; + +enum class LstmGateAndState +{ + I = 0, + F = 1, + O = 2, + G = 3, + St = 4, + Ht = 5 +}; + +/** + * \section Extensions for layouts. + */ + +// GRU +// TODO + +// LSTM +template +class GeneralLstmWsExt +{ +public: + size_t getGasOffset(const size_t layer_id, + const size_t vector_id, + const SequenceDirection direction, + LstmGateAndState gas) const + { + return static_cast(this)->getGasOffsetImpl( + layer_id, vector_id, direction, gas); + } + + const std::array& getGateAndStateStride(LstmGateAndState gas) const + { + return static_cast(this)->getGateAndStateStrideImpl(gas); + } +}; + +template +class LstmWsGateBlockExt +{ +public: + size_t getGateBlockOffset(const size_t layer_id, + const size_t vector_id, + const SequenceDirection direction) const + { + return static_cast(this)->getGasOffset( + layer_id, vector_id, direction, LstmGateAndState::I); + } + + const std::array& getGateBlockStride() const + { + return static_cast(this)->getGateAndStateStride(LstmGateAndState::I); + } + + const std::array& getGateBlockSize() const + { + return static_cast(this)->getGateBlockSizeImpl(); + } +}; + +template +class LstmActiveCellExt +{ +public: + size_t getActiveCellOffset(const size_t layer_id, + const size_t vector_id, + const SequenceDirection direction) const + { + return static_cast(this)->getActiveCellOffsetImpl( + layer_id, vector_id, direction); + } + + const std::array& getActiveCellStride() const + { + return static_cast(this)->getActiveCellStrideImpl(); + } +}; + +// pure RNN +template +class BaseRnnWsBufferPacked +{ +public: + BaseRnnWsBufferPacked() = default; + size_t getHiddenStateOffset(const size_t layer_id, + const size_t vector_id, + const SequenceDirection direction) const + { + return static_cast(this)->getHiddenStateOffsetImpl( + layer_id, vector_id, direction); + } + + // layer, minor dim(seq or sample), directions, element + const std::array& getHiddenStateStride() const + { + return static_cast(this)->getHiddenStateStrideImpl(); + } + + size_t getBufferSize() const { return static_cast(this)->getBufferSizeImpl(); } +}; + +/** + * \section Standard layouts for temporary buffers + */ +////// + +class GeneralRNNTempBuffer : public BaseRnnWsBufferPacked +{ +protected: + GeneralRNNTempBuffer(const std::array& hstate_strides, + const std::array& hstate_sizes, + size_t total_element_cnt) + : hStateStrides{hstate_strides}, + hStateSizes{hstate_sizes}, + totalElementCnt{total_element_cnt} + { + } + +public: + static GeneralRNNTempBuffer + build(size_t layers_cnt, size_t vectors_per_layer, size_t directions, size_t hidden_vec_sz) + { + const std::array h_state_sizes{ + layers_cnt, vectors_per_layer, directions, hidden_vec_sz}; + std::array h_state_strides = {0, 0, 0, 1}; + std::partial_sum(h_state_sizes.crbegin(), + std::prev(h_state_sizes.crend()), + std::next(h_state_strides.rbegin()), + std::multiplies{}); + + auto total_element_cnt = h_state_strides[0] * h_state_sizes[0]; + return GeneralRNNTempBuffer{h_state_strides, h_state_sizes, total_element_cnt}; + } + + size_t getHiddenStateOffsetImpl(const size_t layer_id, + const size_t vector_id, + const SequenceDirection direction) const + { + const std::array pos{layer_id, vector_id, static_cast(direction)}; + + return std::inner_product( + pos.cbegin(), pos.cend(), hStateStrides.cbegin(), static_cast(0)); + } + + size_t getBufferSizeImpl() const { return totalElementCnt; } + + const std::array& getHiddenStateStrideImpl() const { return hStateStrides; } + + // layer, comb dim(seq, sample), direction, vector element + const std::array hStateStrides; + // layers, comb dims(seq, sample), directions, elements + const std::array hStateSizes; + const size_t totalElementCnt; +}; + +/* + *struct Workspace_LSTM_BWD{ //packed + * struct layer{ + * struct all_sequences_all_samples{ + * struct mat_mul_block{ + * struct gate_vector{ + * float[hidden_size]; HidUpdate-> write + * } I, F, O, G; + * } gemm_block[directions]; + * + * struct dcx_vec{ + * float[hidden_size]; HidUpdate-> write + * } dy[directions]; + * + * struct dy_vec{ + * float[hidden_size]; HidUpdate <-read + * } dy[directions]; + * + * } combi_dim[seq_len * batch_size=totalBatch]; [seq][batch] + * } layers[nLayer]; + *} + */ + +class GeneralLstmTempBuffer : public GeneralRNNTempBuffer, + public GeneralLstmWsExt, + public LstmWsGateBlockExt +{ +protected: + GeneralLstmTempBuffer(const std::array& h_state_strides, + const std::array& h_state_sizes, + const std::array& lstm_gate_sizes, + const std::array& lstm_gate_strides, + const std::array& lstm_gates_block_sizes, + size_t total_element_cnt) + : GeneralRNNTempBuffer{h_state_strides, h_state_sizes, total_element_cnt}, + gateSizes{lstm_gate_sizes}, + gateStride{lstm_gate_strides}, + gateBlockSizes{lstm_gates_block_sizes} + { + } + +public: + static GeneralLstmTempBuffer + build(size_t layers_cnt, size_t comp_dim_per_layer, size_t directions, size_t hidden_vec_sz) + { + + constexpr size_t MS_len = 2; + constexpr size_t LS_size = 2; + // Most significant sizes + //{layer, comp_dim_per_layer} + std::array block_MS_size = {layers_cnt, comp_dim_per_layer}; + + //{layer, comp_dim_per_layer, directions,vec} + const auto h_state_sizes = + Concat(block_MS_size, std::array{directions, hidden_vec_sz}); + + const auto lstm_gate_sizes = + Concat(block_MS_size, std::array{directions, hidden_vec_sz}); + + const auto lstm_gates_block_sizes = + Concat(block_MS_size, std::array{directions, 4 * hidden_vec_sz}); + + // Less significant stride + // LSStride{directions,vec} + std::array h_state_LS_strides{}; + std::array lstm_gate_LS_strides{}; + + WA_RHEL::exclusive_scan_wa(h_state_sizes.crbegin(), + std::next(h_state_sizes.crbegin(), h_state_LS_strides.size()), + h_state_LS_strides.rbegin(), + 1LL, + std::multiplies{}); + + WA_RHEL::exclusive_scan_wa( + lstm_gates_block_sizes.crbegin(), + std::next(lstm_gates_block_sizes.crbegin(), lstm_gate_LS_strides.size()), + lstm_gate_LS_strides.rbegin(), + 1LL, + std::multiplies{}); + + const auto gas_block_stride = [&lstm_gates_block_sizes, + &h_state_sizes, + &lstm_gate_LS_strides, + &h_state_LS_strides](const auto last_dim) { + auto gate_size_in_block = lstm_gates_block_sizes[last_dim] * lstm_gate_LS_strides[0]; + auto h_state_size_in_block = h_state_sizes[last_dim] * h_state_LS_strides[0]; + auto c_state_size_in_block = h_state_size_in_block; + return gate_size_in_block + c_state_size_in_block + h_state_size_in_block; + }(MS_len); + + // Most significant stride + // MSStride{layer, comp_dim_per_layer} + std::array block_MS_strides{}; + + WA_RHEL::exclusive_scan_wa(block_MS_size.crbegin(), + std::next(block_MS_size.crbegin(), block_MS_strides.size()), + block_MS_strides.rbegin(), + gas_block_stride, + std::multiplies{}); + + const std::array h_state_strides = Concat(block_MS_strides, h_state_LS_strides); + const std::array lstm_gate_strides = + Concat(block_MS_strides, lstm_gate_LS_strides); + + auto total_element_cnt = h_state_strides[0] * h_state_sizes[0]; + + return {h_state_strides, + h_state_sizes, + lstm_gate_sizes, + lstm_gate_strides, + lstm_gates_block_sizes, + total_element_cnt}; + } + + size_t getHiddenStateOffset(const size_t layer_id, + const size_t vector_id, + const SequenceDirection direction) const + { + return getGasOffset(layer_id, vector_id, direction, LstmGateAndState::Ht); + } + + size_t getGasOffsetImpl(const size_t layer_id, + const size_t vector_id, + const SequenceDirection direction, + LstmGateAndState gas) const + { + auto start_ident = getGasIndent(gas); + + if(gas == LstmGateAndState::Ht || gas == LstmGateAndState::St) + return start_ident + + GeneralRNNTempBuffer::getHiddenStateOffset(layer_id, vector_id, direction); + + const std::array pos{layer_id, vector_id, static_cast(direction)}; + + return start_ident + + std::inner_product( + pos.cbegin(), pos.cend(), hStateStrides.cbegin(), static_cast(0)); + } + + // layer, minor dim(seq or sample), directions, element + const std::array& getGateAndStateStrideImpl(LstmGateAndState gas) const + { + if(gas == LstmGateAndState::Ht || gas == LstmGateAndState::St) + return getHiddenStateStride(); + return gateStride; + } + + // layer, minor dim(seq or sample), directions, element + const std::array& getGateBlockSizeImpl() const { return gateBlockSizes; } + + const std::array gateSizes; + const std::array gateStride; + const std::array gateBlockSizes; + +private: + size_t getGasIndent(LstmGateAndState gas) const + { + switch(gas) + { + case LstmGateAndState::I: + case LstmGateAndState::G: + case LstmGateAndState::F: + case LstmGateAndState::O: return static_cast(gas) * gateStride[3] * gateSizes[3]; + case LstmGateAndState::St: return gateStride[2] * gateSizes[2]; // direction DIM + case LstmGateAndState::Ht: + return (gateStride[2] + getHiddenStateStride()[2]) * gateSizes[2]; + } + return 0; + } +}; + +/* + *struct ReserveSpace_LSTM{ //packed + * struct layer{ + * struct all_sequences_all_samples{ + * + * struct mat_mul_block{ + * struct gate_vector{ + * float[hidden_size]; HidUpdate-> write + * } I, F, O, G; + * } gemm_block[directions]; + * + * struct dcx_vec{ + * float[hidden_size]; HidUpdate-> write + * } dy[directions]; + * + * struct dy_vec{ + * float[hidden_size]; HidUpdate <-read + * } dy[directions]; + * + * } combi_dim[seq_len * batch_size=totalBatch]; [seq][batch] + * } layers[nLayer]; + * + * struct extra_activation_vec{ + * float[hidden_size]; + * } active_cell[nLayer*bidirect*totalBatch]; + *} + */ + +class GeneralLstmRedBuffer : public GeneralLstmTempBuffer, + public LstmActiveCellExt +{ +protected: + GeneralLstmRedBuffer(const GeneralLstmTempBuffer& base, + const std::array& active_cells_sizes, + const std::array& active_cells_strides, + size_t active_cells_ident, + size_t active_cell_elements) + : GeneralLstmTempBuffer{base}, + activeCellSize{active_cells_sizes}, + activeCellStride{active_cells_strides}, + activeCellsIdent{active_cells_ident}, + activeCellElements{active_cell_elements} + { + } + +public: + static GeneralLstmRedBuffer + build(size_t layers_cnt, size_t comp_dim_per_layer, size_t directions, size_t hidden_vec_sz) + { + + auto base = + GeneralLstmTempBuffer::build(layers_cnt, comp_dim_per_layer, directions, hidden_vec_sz); + + auto active_cells_ident = base.gateStride[0] * base.gateSizes[0]; + + std::array active_cells_sizes{ + layers_cnt, comp_dim_per_layer, directions, hidden_vec_sz}; + + std::array active_cells_strides = {}; + + WA_RHEL::exclusive_scan_wa(active_cells_sizes.crbegin(), + active_cells_sizes.crend(), + active_cells_strides.rbegin(), + 1LL, + std::multiplies{}); + + auto active_cell_elements = active_cells_strides[0] * active_cells_sizes[0]; + + return {base, + active_cells_sizes, + active_cells_strides, + active_cells_ident, + active_cell_elements}; + } + + size_t getBufferSizeImpl() const { return activeCellsIdent + activeCellElements; } + + size_t getActiveCellOffsetImpl(const size_t layer_id, + const size_t vector_id, + const SequenceDirection direction) const + { + const std::array pos{layer_id, vector_id, static_cast(direction)}; + + return activeCellsIdent + + std::inner_product( + pos.cbegin(), pos.cend(), activeCellStride.cbegin(), static_cast(0)); + } + + const std::array& getActiveCellStrideImpl() const { return activeCellStride; } + + const std::array activeCellSize; + const std::array activeCellStride; + const size_t activeCellsIdent; + const size_t activeCellElements; +}; + +/** + * \section Сlasses for easier navigation. + */ + +class BatchController +{ +public: + template + static BatchController Create(VectorT&& batchs_at_time) + { + std::vector batch{std::forward(batchs_at_time)}; + std::vector b_prefix_sums(batch.size()); + b_prefix_sums[0] = 0; + + std::partial_sum(batch.cbegin(), std::prev(batch.cend()), std::next(b_prefix_sums.begin())); + + return BatchController{std::move(batch), std::move(b_prefix_sums)}; + } + + static BatchController Create(const SeqTensorDescriptor& desc) + { + return Create(desc.GetBatchesPerSequence()); + } + + size_t getTotalBatchSum() const + { + return *batchAtTime.crbegin() + *batchPrefSumAtTime.crbegin(); + } + + template + size_t getBatchSize(TimeIndexT time_id) const + { + return batchAtTime[time_id]; + } + + template + size_t getBatchSum(TimeIndexT time_id) const + { + return batchPrefSumAtTime[time_id]; + } + +private: + template >::value, bool> = true> + explicit BatchController(T&& batch_at_time, T&& batch_prefix_sums) + : batchAtTime(batch_at_time), batchPrefSumAtTime{std::forward(batch_prefix_sums)} + { + } + + const std::vector batchAtTime; + const std::vector batchPrefSumAtTime; +}; + +class SequenceIterator +{ +public: + SequenceIterator(size_t logical_value, SequenceDirection dir, size_t seq_size, bool is_fwd_pass) + : value(logical_value), + startVal(is_fwd_pass ? 0 : seq_size - 1), + endVal(is_fwd_pass ? seq_size - 1 : 0), + maxVal(seq_size - 1), + isReverseSeq(dir == SequenceDirection::Reverse), + isFwdPass(is_fwd_pass) + { + assert(logical_value < seq_size); + } + + size_t getLogVal() const { return value; } + size_t getPhisVal() const { return !isReverseSeq ? value : maxVal - value; } + + SequenceIterator getNext() const + { + assert(!isLast()); + return PartClone(value + (isFwdPass ? 1 : -1)); + } + SequenceIterator getPrev() const + { + assert(!isFirst()); + return PartClone(value + (isFwdPass ? -1 : 1)); + } + + bool isFirst() const { return value == startVal; } + + bool isLast() const { return value == endVal; } + +private: + SequenceIterator PartClone(const size_t newValue) const + { + return {newValue, startVal, endVal, maxVal, isReverseSeq, isFwdPass}; + } + + SequenceIterator(size_t value_, + size_t start_val, + size_t end_val, + size_t max_val, + bool is_reverse_seq, + bool is_fwd_pass) + : value(value_), + startVal(start_val), + endVal(end_val), + maxVal(max_val), + isReverseSeq(is_reverse_seq), + isFwdPass(is_fwd_pass) + { + } + + const size_t value; + const size_t startVal; + const size_t endVal; + const size_t maxVal; + const bool isReverseSeq; + const bool isFwdPass; +}; + +/** + * \section User defined buffers. + */ + +/* + *struct Weights{ + * struct filter{ + * + * struct gateInputMatrix{ + * float[hidden_size][input_size] + * } gateFilters[gate_cnt]; + * + * struct gateHiddenMatrix{ + * float[hidden_size][hidden_size] + * } gateFiltersHiddden[gate_cnt]; + * } filters[n_layers][bidirect]; + * + * struct bias{ + * struct { + * float[hidden_size] + * }gateBias[gate_cnt] + * }biases[n_layers][bias_cnt][bidirect] + *} + */ +struct WeightsBufferDescriptor +{ +private: + static size_t hidden_xinput_size(size_t hidden_sz, bool bidirect_mode) + { + if(!bidirect_mode) + return hidden_sz; + MIOPEN_THROW("execution failure: bidirect is not supported by this solver"); + } + + static auto filter_size(size_t input_vector_sz, size_t hidden_vec_sz, size_t gates) + { + return (input_vector_sz + hidden_vec_sz) * hidden_vec_sz * gates; + } + + static size_t bias_start_offset(size_t phis_layer_filter_sz, + size_t hidden_layer_filter_sz, + size_t layers_cnt, + bool bidirect_mode) + { + if(!bidirect_mode) + { + return phis_layer_filter_sz + hidden_layer_filter_sz * (layers_cnt - 1); + } + + MIOPEN_THROW("execution failure: bidirect is not supported by this solver"); + } + + WeightsBufferDescriptor(size_t input_vector_sz, + size_t hidden_vec_sz, + size_t hidden_xinput_sz, + size_t layers_cnt, + size_t gates, + size_t bias_mode, + + size_t lin_filter_sz, + size_t hidden_filter_sz, + const std::array& bias_strides, + const std::array& bias_lens, + size_t bias_off) + : inVec(input_vector_sz), + hVec(hidden_vec_sz), + xInVec(hidden_xinput_sz), + layers(layers_cnt), + gatesCnt(gates), + biasCnt(bias_mode), + biasStrides{bias_strides}, + biasLens{bias_lens}, + biasStartOff(bias_off), + linLayerFilterSize(lin_filter_sz), + hiddenLayerFilterSize(hidden_filter_sz) + { + } + +public: + static WeightsBufferDescriptor create(size_t input_vector_sz, + size_t hidden_vec_sz, + size_t layers_cnt, + size_t bias_mode, + size_t input_mode, + size_t gates, + bool bidirect_mode) + { + const size_t directions_num = bidirect_mode ? 2 : 1; + const size_t x_in_vec = hidden_xinput_size(hidden_vec_sz, bidirect_mode); + + size_t input_vector_filter_sz = input_mode == 0 ? input_vector_sz : 0; + + const size_t linLayerFilterSize = filter_size(input_vector_filter_sz, hidden_vec_sz, gates); + const size_t hiddenLayerFilterSize = filter_size(x_in_vec, hidden_vec_sz, gates); + + const size_t bias_start_off = + bias_start_offset(linLayerFilterSize, hiddenLayerFilterSize, layers_cnt, bidirect_mode); + + //[layer][dir][param_id][vector] + const std::array bias_lens{layers_cnt, directions_num, gates, hidden_vec_sz}; + const std::array bias_strides = [](const auto& lens, size_t bias_cnt) { + std::array strides = {0, 0, 0, 1}; + std::partial_sum(lens.crbegin(), + std::prev(lens.crend()), + std::next(strides.rbegin()), + std::multiplies{}); + strides[0] *= bias_cnt; + return strides; + }(bias_lens, bias_mode * 2); + + return {input_vector_filter_sz, + hidden_vec_sz, + x_in_vec, + layers_cnt, + gates, + bias_mode, + linLayerFilterSize, + hiddenLayerFilterSize, + bias_strides, + bias_lens, + bias_start_off}; + } + + const size_t inVec, hVec; + const size_t xInVec; // for bidirect TODO + + const size_t layers; + const size_t gatesCnt; + const size_t + biasCnt; // 0 - no bisa; 1 - one bias; 2 - separate bias for x_vec and for hidden_vec +private: + //[layer][dir][param_id][vector] + const std::array biasStrides; + const std::array biasLens; + + const size_t biasStartOff; + + const size_t linLayerFilterSize; + const size_t hiddenLayerFilterSize; + +public: + size_t getParamRelativeOff([[maybe_unused]] size_t layer_id, int /*dir_id*/, int param_id) const + { + assert(layer_id > 0); + return param_id * hVec * xInVec; + } + + size_t getPhisParamRelativeOff(int /*dir_id*/, int param_id) const + { + return hVec * ((static_cast(param_id) >= gatesCnt) + ? inVec * gatesCnt + xInVec * (param_id - gatesCnt) + : inVec * param_id); + } + + size_t getMatrixOff(size_t layer_id, int dir_id, int param_id) const + { + size_t offset = 0; + if(layer_id > 0) + { + offset += linLayerFilterSize; + } + else + { + return getPhisParamRelativeOff(dir_id, param_id); + } + if(layer_id > 1) + { + offset += hiddenLayerFilterSize * (layer_id - 1); + } + return offset + getParamRelativeOff(layer_id, dir_id, param_id); + } + + size_t getMatrixXinOff(size_t layer_id, int dir_id) const + { + return getMatrixOff(layer_id, dir_id, 0); + } + + size_t getMatrixHidOff(size_t layer_id, int dir_id) const + { + return getMatrixOff(layer_id, dir_id, gatesCnt); + } + + size_t getBiasOff(size_t layer_id, int dir_id, int param_id) const + { + return biasStartOff + biasStrides[0] * layer_id + biasStrides[1] * dir_id + + biasStrides[2] * param_id; + } + + size_t getBiasXinOff(size_t layer_id, int dir_id, int param_id) const + { + assert(param_id < gatesCnt); + return getBiasOff(layer_id, dir_id, param_id); + } + + size_t getBiasHidOff(size_t layer_id, int dir_id, int param_id) const + { + assert(param_id < gatesCnt); + return getBiasOff(layer_id, dir_id, param_id + gatesCnt); + } + + //[layers][dirs][params][vec_size] + const std::array& getBiasSize() const { return biasLens; } + + //[layers][dirs][params][vec_size] + const std::array& getBiasStride() const { return biasStrides; } +}; + +class HiddenBuffersDescriptor +{ +public: + explicit HiddenBuffersDescriptor(const TensorDescriptor& hx_desc) + : lens(hx_desc.GetLengths().begin(), hx_desc.GetLengths().begin() + 3), + strides(hx_desc.GetStrides().begin(), hx_desc.GetStrides().begin() + 3) + { + } + + inline size_t getOffset(const size_t virtual_layer_id) const + { + return getOffset(virtual_layer_id, 0); + } + + inline size_t getOffset(const size_t virtual_layer_id, const size_t batch_id) const + { + return strides[0] * virtual_layer_id + strides[1] * batch_id; + } + + inline const std::vector& getStrides() const { return strides; } + + inline const std::vector& GetLengths() const { return lens; } + + inline size_t getVirtualLayersCnt() const { return lens[0]; } + inline size_t getMiniBatchSize() const { return lens[1]; } + inline size_t getHiddenSize() const { return lens[2]; } + + static std::vector MakeStrides(const std::vector& lengths) + { + return {lengths[1] * lengths[2], lengths[2], 1}; + } + +private: + // local caching + + const std::vector lens; + const std::vector strides; +}; + +class IOBufferDescriptor +{ + IOBufferDescriptor() = default; + IOBufferDescriptor(std::vector&& buffer_lens, + std::vector&& buffer_strides, + std::vector&& packed_lens, + std::vector&& packed_strides, + std::vector&& seq_lens_per_sample) + : lens{std::move(buffer_lens)}, + strides{std::move(buffer_strides)}, + packedLens{std::move(packed_lens)}, + packedStrides{std::move(packed_strides)}, + seqLensPerSample{std::move(seq_lens_per_sample)} + { + } + +public: + static IOBufferDescriptor build(const SeqTensorDescriptor& xyDesc) + { + //{batch, seq_cnt, vector} + auto lens = xyDesc.GetLengths(); + auto strides = xyDesc.GetPaddedStrides(); + + //{ combine(batch, seq_cnt), vector} + std::vector packed_lens{xyDesc.GetTotalSequenceLen(), lens[2]}; + std::vector packed_strides(2); + + WA_RHEL::exclusive_scan_wa(packed_lens.crbegin(), + std::next(packed_lens.crbegin(), packed_strides.size()), + packed_strides.rbegin(), + 1LL, + std::multiplies{}); + + std::vector seq_lens_per_sample = xyDesc.GetSequenceLengthsVector(); + return {(std::move(lens)), + (std::move(strides)), + (std::move(packed_lens)), + (std::move(packed_strides)), + (std::move(seq_lens_per_sample))}; + } + + inline size_t getPackedOffset(const size_t batch_id) const + { + return packedStrides[0] * batch_id; + } + + inline std::vector getFullSeqMajorStrides() const { return packedStrides; } + inline std::vector getFullSeqMajorSize() const { return packedLens; } + + inline size_t getMiniBatchSize() const { return lens[0]; } + inline size_t getMaxSeqSize() const { return lens[1]; } + inline size_t getHiddenSize() const { return lens[2]; } + + inline size_t getSeqSize(size_t sample_id) const { return seqLensPerSample[sample_id]; } + inline size_t getTotalSeqCnt() const { return packedLens[0]; } + + static std::vector MakeStrides(const std::vector& lengths) + { + return {lengths[1] * lengths[2], lengths[2], 1}; + } + + // private: + // local caching + + const std::vector lens; + const std::vector strides; + + const std::vector packedLens; + const std::vector packedStrides; + + const std::vector seqLensPerSample; +}; + +} // namespace rnn_base +} // namespace miopen diff --git a/src/include/miopen/rope.hpp b/src/include/miopen/rope.hpp new file mode 100644 index 0000000000..5995354c1a --- /dev/null +++ b/src/include/miopen/rope.hpp @@ -0,0 +1,57 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#ifndef MIOPEN_ROPE_HPP_ +#define MIOPEN_ROPE_HPP_ + +#include + +namespace miopen { + +struct Handle; +struct TensorDescriptor; + +MIOPEN_INTERNALS_EXPORT miopenStatus_t RoPEForward(Handle& handle, + const TensorDescriptor& xDesc, + ConstData_t x, + const TensorDescriptor& cosDesc, + ConstData_t cos, + const TensorDescriptor& sinDesc, + ConstData_t sin, + const TensorDescriptor& yDesc, + Data_t y); + +MIOPEN_INTERNALS_EXPORT miopenStatus_t RoPEBackward(Handle& handle, + const TensorDescriptor& dyDesc, + ConstData_t dy, + const TensorDescriptor& cosDesc, + ConstData_t cos, + const TensorDescriptor& sinDesc, + ConstData_t sin, + const TensorDescriptor& dxDesc, + Data_t dx); + +} // namespace miopen +#endif // MIOPEN_ROPE_HPP_ diff --git a/src/include/miopen/rope/invoke_params.hpp b/src/include/miopen/rope/invoke_params.hpp new file mode 100644 index 0000000000..8db5923204 --- /dev/null +++ b/src/include/miopen/rope/invoke_params.hpp @@ -0,0 +1,73 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +namespace miopen { +namespace rope { + +struct FwdInvokeParams : public miopen::InvokeParams +{ + FwdInvokeParams() = default; + + const TensorDescriptor* xDesc = nullptr; + const TensorDescriptor* cosDesc = nullptr; + const TensorDescriptor* sinDesc = nullptr; + const TensorDescriptor* yDesc = nullptr; + + ConstData_t x = nullptr; + ConstData_t cos = nullptr; + ConstData_t sin = nullptr; + Data_t y = nullptr; + + std::size_t GetWorkspaceSize() const { return 0; } + Data_t GetWorkspace() const { return nullptr; } +}; + +struct BwdInvokeParams : public miopen::InvokeParams +{ + BwdInvokeParams() = default; + + const TensorDescriptor* dyDesc = nullptr; + const TensorDescriptor* cosDesc = nullptr; + const TensorDescriptor* sinDesc = nullptr; + const TensorDescriptor* dxDesc = nullptr; + + ConstData_t dy = nullptr; + ConstData_t cos = nullptr; + ConstData_t sin = nullptr; + Data_t dx = nullptr; + + std::size_t GetWorkspaceSize() const { return 0; } + Data_t GetWorkspace() const { return nullptr; } +}; + +} // namespace rope + +} // namespace miopen diff --git a/src/include/miopen/rope/problem_description.hpp b/src/include/miopen/rope/problem_description.hpp new file mode 100644 index 0000000000..0338d7e2b1 --- /dev/null +++ b/src/include/miopen/rope/problem_description.hpp @@ -0,0 +1,168 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include +#include +#include +#include + +namespace miopen { + +struct NetworkConfig; + +namespace rope { + +struct ProblemDescriptionFwd : ProblemDescriptionBase +{ + ProblemDescriptionFwd(const TensorDescriptor& xDesc_, + const TensorDescriptor& cosDesc_, + const TensorDescriptor& sinDesc_, + const TensorDescriptor& yDesc_) + : xDesc(xDesc_), cosDesc(cosDesc_), sinDesc(sinDesc_), yDesc(yDesc_) + { + IsValidLength(); + } + + const TensorDescriptor& GetXDesc() const { return xDesc; } + const TensorDescriptor& GetCosDesc() const { return cosDesc; } + const TensorDescriptor& GetSinDesc() const { return sinDesc; } + const TensorDescriptor& GetYDesc() const { return yDesc; } + + bool IsValidLength() const + { + if(xDesc.GetLengths() != yDesc.GetLengths()) + { + MIOPEN_THROW(miopenStatusBadParm, + "RoPEForward: Tensor x and y dimension lengths do not match."); + } + + if(cosDesc.GetLengths() != sinDesc.GetLengths()) + { + MIOPEN_THROW(miopenStatusBadParm, + "RoPEForward: Tensor cos and sin dimension lengths do not match."); + } + return true; + } + + bool IsSameType() const + { + if(xDesc.GetType() != yDesc.GetType()) + { + return false; + } + return true; + } + + bool IsAllContiguous() const + { + if(!(xDesc.IsContiguous() && cosDesc.IsContiguous() && sinDesc.IsContiguous() && + yDesc.IsContiguous())) + { + return false; + } + + return true; + } + + NetworkConfig MakeNetworkConfig() const override; + +private: + TensorDescriptor xDesc; + TensorDescriptor cosDesc; + TensorDescriptor sinDesc; + TensorDescriptor yDesc; + + NetworkConfig MakeForwardNetworkConfig() const; +}; + +struct ProblemDescriptionBwd : ProblemDescriptionBase +{ + ProblemDescriptionBwd(const TensorDescriptor& dyDesc_, + const TensorDescriptor& cosDesc_, + const TensorDescriptor& sinDesc_, + const TensorDescriptor& dxDesc_) + : dyDesc(dyDesc_), cosDesc(cosDesc_), sinDesc(sinDesc_), dxDesc(dxDesc_) + { + IsValidLength(); + } + + const TensorDescriptor& GetDYDesc() const { return dyDesc; } + const TensorDescriptor& GetCosDesc() const { return cosDesc; } + const TensorDescriptor& GetSinDesc() const { return sinDesc; } + const TensorDescriptor& GetDXDesc() const { return dxDesc; } + + bool IsValidLength() const + { + if(dyDesc.GetLengths() != dxDesc.GetLengths()) + { + MIOPEN_THROW(miopenStatusBadParm, + "RoPEForward: Tensor dy and dx dimension lengths do not match."); + } + + if(cosDesc.GetLengths() != sinDesc.GetLengths()) + { + MIOPEN_THROW(miopenStatusBadParm, + "RoPEForward: Tensor cos and sin dimension lengths do not match."); + } + return true; + } + + bool IsSameType() const + { + if(dyDesc.GetType() != dxDesc.GetType()) + { + return false; + } + return true; + } + + bool IsAllContiguous() const + { + if(!(dyDesc.IsContiguous() && cosDesc.IsContiguous() && sinDesc.IsContiguous() && + dxDesc.IsContiguous())) + { + return false; + } + + return true; + } + + NetworkConfig MakeNetworkConfig() const override; + +private: + TensorDescriptor dyDesc; + TensorDescriptor cosDesc; + TensorDescriptor sinDesc; + TensorDescriptor dxDesc; + + NetworkConfig MakeForwardNetworkConfig() const; +}; + +} // namespace rope + +} // namespace miopen diff --git a/src/include/miopen/rope/solvers.hpp b/src/include/miopen/rope/solvers.hpp new file mode 100644 index 0000000000..859aff2965 --- /dev/null +++ b/src/include/miopen/rope/solvers.hpp @@ -0,0 +1,65 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include +#include + +namespace miopen { + +namespace solver { + +namespace rope { + +using RoPESolverFwd = NonTunableSolverBase; +using RoPESolverBwd = NonTunableSolverBase; + +struct RoPEForward final : RoPESolverFwd +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext& context, + const miopen::rope::ProblemDescriptionFwd& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::rope::ProblemDescriptionFwd& problem) const override; +}; + +struct RoPEBackward final : RoPESolverBwd +{ + const std::string& SolverDbId() const override { return GetSolverDbId(); } + + bool IsApplicable(const ExecutionContext& context, + const miopen::rope::ProblemDescriptionBwd& problem) const override; + ConvSolution GetSolution(const ExecutionContext& context, + const miopen::rope::ProblemDescriptionBwd& problem) const override; +}; + +} // namespace rope + +} // namespace solver + +} // namespace miopen diff --git a/src/include/miopen/softmarginloss.hpp b/src/include/miopen/softmarginloss.hpp new file mode 100644 index 0000000000..58d1b2c0ce --- /dev/null +++ b/src/include/miopen/softmarginloss.hpp @@ -0,0 +1,67 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#ifndef MIOPEN_SOFTMARGINLOSS_HPP_ +#define MIOPEN_SOFTMARGINLOSS_HPP_ + +#include "miopen/miopen.h" +#include + +namespace miopen { + +struct Handle; +struct TensorDescriptor; + +MIOPEN_INTERNALS_EXPORT std::size_t +GetSoftMarginLossForwardWorkspaceSize(Handle& handle, + const TensorDescriptor& iDesc, + const TensorDescriptor& tDesc, + const TensorDescriptor& oDesc, + miopenLossReductionMode_t reduction); + +MIOPEN_INTERNALS_EXPORT miopenStatus_t SoftMarginLossForward(Handle& handle, + Data_t workspace, + size_t workspaceSizeInBytes, + const TensorDescriptor& iDesc, + ConstData_t i, + const TensorDescriptor& tDesc, + ConstData_t t, + const TensorDescriptor& oDesc, + Data_t o, + miopenLossReductionMode_t reduction); + +MIOPEN_INTERNALS_EXPORT miopenStatus_t SoftMarginLossBackward(Handle& handle, + const TensorDescriptor& iDesc, + ConstData_t i, + const TensorDescriptor& tDesc, + ConstData_t t, + const TensorDescriptor& dODesc, + ConstData_t dO, + const TensorDescriptor& dIDesc, + Data_t dI, + miopenLossReductionMode_t reduction); + +} // namespace miopen +#endif // _MIOPEN_SOFTMARGINLOSS_HPP_ diff --git a/src/include/miopen/softmarginloss/invoke_params.hpp b/src/include/miopen/softmarginloss/invoke_params.hpp new file mode 100644 index 0000000000..2184cdd33c --- /dev/null +++ b/src/include/miopen/softmarginloss/invoke_params.hpp @@ -0,0 +1,59 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#pragma once + +#include +#include + +namespace miopen { +namespace softmarginloss { + +struct InvokeParams : public miopen::InvokeParams +{ + InvokeParams() = default; + + const TensorDescriptor* iDesc = nullptr; + const TensorDescriptor* tDesc = nullptr; + const TensorDescriptor* oDesc = nullptr; + const TensorDescriptor* dODesc = nullptr; + const TensorDescriptor* dIDesc = nullptr; + + ConstData_t i = nullptr; + ConstData_t t = nullptr; + Data_t o = nullptr; + ConstData_t dO = nullptr; + Data_t dI = nullptr; + + Data_t workspace = nullptr; + std::size_t workspace_size = 0; + std::size_t GetWorkspaceSize() const { return workspace_size; } + Data_t GetWorkspace() const { return workspace; } +}; + +} // namespace softmarginloss + +} // namespace miopen diff --git a/src/include/miopen/softmarginloss/problem_description.hpp b/src/include/miopen/softmarginloss/problem_description.hpp new file mode 100644 index 0000000000..d9f0ed0271 --- /dev/null +++ b/src/include/miopen/softmarginloss/problem_description.hpp @@ -0,0 +1,125 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include "miopen/miopen.h" +#include +#include +#include + +namespace miopen { + +struct NetworkConfig; + +namespace softmarginloss { + +struct ForwardProblemDescription : ProblemDescriptionBase +{ + ForwardProblemDescription(const TensorDescriptor& iDesc_, + const TensorDescriptor& tDesc_, + const TensorDescriptor& oDesc_, + const miopenLossReductionMode_t reduction_) + : iDesc(iDesc_), tDesc(tDesc_), oDesc(oDesc_), reduction(reduction_) + { + if(iDesc.GetType() != tDesc.GetType() || iDesc.GetType() != oDesc.GetType()) + { + MIOPEN_THROW(miopenStatusBadParm, "SoftMarginLoss: Tensor types do not match."); + } + if(iDesc.GetLengths() != tDesc.GetLengths()) + { + MIOPEN_THROW( + miopenStatusBadParm, + "SoftMarginLoss: Input tensor and target tensor dimension lengths do not match."); + } + if((reduction == MIOPEN_LOSS_REDUCTION_NONE) && (iDesc.GetLengths() != oDesc.GetLengths())) + { + MIOPEN_THROW(miopenStatusBadParm, + "SoftMarginLoss: When doing forward with no reduction, output tensor and " + "input tensor dimension lengths should be equal."); + } + if((reduction != MIOPEN_LOSS_REDUCTION_NONE) && (oDesc.GetElementSize() != 1)) + { + MIOPEN_THROW(miopenStatusBadParm, + "SoftMarginLoss: When doing forward reduction, output tensor size " + "need to be 1."); + } + } + + const TensorDescriptor& GetiDesc() const { return iDesc; } + const TensorDescriptor& GettDesc() const { return tDesc; } + const TensorDescriptor& GetoDesc() const { return oDesc; } + miopenLossReductionMode_t Getreduction() const { return reduction; } + + NetworkConfig MakeNetworkConfig() const override; + +private: + TensorDescriptor iDesc; + TensorDescriptor tDesc; + TensorDescriptor oDesc; + miopenLossReductionMode_t reduction; +}; + +struct BackwardProblemDescription : ProblemDescriptionBase +{ + BackwardProblemDescription(const TensorDescriptor& iDesc_, + const TensorDescriptor& tDesc_, + const TensorDescriptor& dODesc_, + const TensorDescriptor& dIDesc_, + const miopenLossReductionMode_t reduction_) + : iDesc(iDesc_), tDesc(tDesc_), dODesc(dODesc_), dIDesc(dIDesc_), reduction(reduction_) + { + if(iDesc.GetType() != tDesc.GetType() || iDesc.GetType() != dODesc.GetType() || + iDesc.GetType() != dIDesc.GetType()) + { + MIOPEN_THROW(miopenStatusBadParm, "SoftMarginLoss: Tensor types do not match."); + } + if(iDesc.GetLengths() != tDesc.GetLengths() || iDesc.GetLengths() != dODesc.GetLengths() || + iDesc.GetLengths() != dIDesc.GetLengths()) + { + MIOPEN_THROW(miopenStatusBadParm, + "SoftMarginLoss: Tensor dimension lengths do not match."); + } + } + + const TensorDescriptor& GetiDesc() const { return iDesc; } + const TensorDescriptor& GettDesc() const { return tDesc; } + const TensorDescriptor& GetdODesc() const { return dODesc; } + const TensorDescriptor& GetdIDesc() const { return dIDesc; } + miopenLossReductionMode_t Getreduction() const { return reduction; } + + NetworkConfig MakeNetworkConfig() const override; + +private: + TensorDescriptor iDesc; + TensorDescriptor tDesc; + TensorDescriptor dODesc; + TensorDescriptor dIDesc; + miopenLossReductionMode_t reduction; +}; + +} // namespace softmarginloss + +} // namespace miopen diff --git a/src/include/miopen/softmarginloss/solvers.hpp b/src/include/miopen/softmarginloss/solvers.hpp new file mode 100644 index 0000000000..6b82141e82 --- /dev/null +++ b/src/include/miopen/softmarginloss/solvers.hpp @@ -0,0 +1,79 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ +#pragma once + +#include +#include + +namespace miopen { + +namespace solver { + +namespace softmarginloss { + +using ForwardSoftMarginLossSolver = + NonTunableSolverBase; + +using BackwardSoftMarginLossSolver = + NonTunableSolverBase; + +struct SoftMarginLossForward final : ForwardSoftMarginLossSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + bool + IsApplicable(const ExecutionContext& context, + const miopen::softmarginloss::ForwardProblemDescription& problem) const override; + ConvSolution + GetSolution(const ExecutionContext& context, + const miopen::softmarginloss::ForwardProblemDescription& problem) const override; + std::size_t GetWorkspaceSize( + const ExecutionContext& context, + const miopen::softmarginloss::ForwardProblemDescription& problem) const override; + bool MayNeedWorkspace() const override { return true; } +}; + +struct SoftMarginLossBackward final : BackwardSoftMarginLossSolver +{ + const std::string& SolverDbId() const override + { + return GetSolverDbId(); + } + bool + IsApplicable(const ExecutionContext& context, + const miopen::softmarginloss::BackwardProblemDescription& problem) const override; + ConvSolution + GetSolution(const ExecutionContext& context, + const miopen::softmarginloss::BackwardProblemDescription& problem) const override; +}; + +} // namespace softmarginloss + +} // namespace solver + +} // namespace miopen diff --git a/src/include/miopen/solver.hpp b/src/include/miopen/solver.hpp index 891b3d0d5e..14f8776bc7 100644 --- a/src/include/miopen/solver.hpp +++ b/src/include/miopen/solver.hpp @@ -29,44 +29,30 @@ #include -#include -#include #include #include -#include -#include -#include -#include +#include #include #include -#include - -#include #include #include -#include -#include #include -#include namespace miopen { namespace debug { -/// If set to true, then always enable ConvDirectNaive* solver, regardless of environment value -/// MIOPEN_DEBUG_CONV_DIRECT_NAIVE_CONV_* that control enable/disable of these solvers. -/// Currently used during driver using naive kernel as gpu reference. -MIOPEN_EXPORT extern bool - AlwaysEnableConvDirectNaive; // NOLINT (cppcoreguidelines-avoid-non-const-global-variables) +/// Enables deprecated solvers. +/// This variable is intended for use in unit tests. +// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables) +MIOPEN_INTERNALS_EXPORT extern bool enable_deprecated_solvers; } // namespace debug struct AnyInvokeParams; namespace solver { -/// \todo Move wave_size into abstraction wich represent GPU information -const int wave_size = 64; /// Base class for problem solvers. /// @@ -93,14 +79,8 @@ struct SolverBase return null_id; } - /// Returns true if solution can work on given SW/HW platform (runtime/device) - /// and provides correct result for the problem config. - /// - /// Every SolverBase which IsApplicable() for some problem config must be able to - /// GetDefaultPerformanceConfig() so that GetSolution() would return valid - /// solution for a problem (i.e. convolution). In other words, if a Solution - /// says "I'm suitable" for a problem, it agrees to solve that problem correctly. - virtual bool IsApplicable(const ExecutionContext& ctx, const boost::any& problem) const = 0; + /// Returns true for tunable solvers + virtual bool IsTunable() const = 0; /// [Informative as of Sep 2020] The minimum requirement for Dynamic Solvers: /// Batch size and input picture size (N, W, H) must NOT be compiled into the @@ -108,81 +88,104 @@ struct SolverBase /// run-time parameters. virtual bool IsDynamic() const { return false; } - /// [Informative as of Sep 2020] Returns an approximated value of the expected - /// WTI or -2.0 when this value can't be computed. Tips: - /// * Value 1.0 corresponds to the 100% utilization of HW capabilities as - /// if Direct computational algorithm is used. - /// * [Notice] WTI may exceed 1.0 for highly optimized algorithms like Winograd. - /// * @see https://github.com/ROCm/MIOpen/issues/410 - virtual float GetWti(const ExecutionContext& ctx, const boost::any& problem) const = 0; - - // Returns the workspace size required by the solver for a given ExecutionContext - virtual size_t GetWorkspaceSize(const ExecutionContext& ctx, - const boost::any& problem) const = 0; - - // Must return true if a Solver has its own implementation of GetWorkspaceSize(). + /// Must return true if a Solver has its own implementation of GetWorkspaceSize(). virtual bool MayNeedWorkspace() const { return false; } protected: template static const std::string& GetSolverDbId() { - static const auto result = ComputeSolverDbId(get_type_name()); +#if BUILD_SHARED_LIBS && MIOPEN_ENABLE_FIN_INTERFACE + /// When using this function outside of the shared library, the static local variable is + /// duplicated, both the library and the program using it have their own copy, but only one + /// of them is initialized, depending on which entity calls the function first—the library + /// or the program. + /// \todo This needs to be removed when the interface matures, and internal class/function + /// templates are no longer used by the fin. + static std::string result; + if(result.empty()) + { + // The "new" operator is used here to avoid segmentation fault (since the variable is + // not initialized). + new(&result) std::string(ComputeSolverDbId(type_name_bare())); + } +#else // !BUILD_SHARED_LIBS || !MIOPEN_ENABLE_FIN_INTERFACE + static const auto result = ComputeSolverDbId(type_name_bare()); +#endif // !BUILD_SHARED_LIBS || !MIOPEN_ENABLE_FIN_INTERFACE return result; } SolverBase() = default; SolverBase(const SolverBase&) = default; private: - static std::string ComputeSolverDbId(const std::string& type_name) + static std::string ComputeSolverDbId(std::string_view type_name) { - auto idx = type_name.find_last_of(':'); - auto name = type_name.substr(idx + 1); - std::replace(name.begin(), name.end(), ',', '-'); - name.erase(std::remove(name.begin(), name.end(), ' '), name.end()); + auto name = std::string(type_name); + if(name.back() == '>') + { + std::replace(name.begin(), name.end(), ',', '-'); + name.erase(std::remove(name.begin(), name.end(), ' '), name.end()); + } return name; } }; +/// Common interface for tunable and non-tunable solvers template -struct SolverMixin : SolverBase +struct SolverInterface : SolverBase { static_assert(std::is_base_of{}, "Context must be derived of ExecutionContext"); - virtual bool IsApplicable(const Context&, const Problem&) const = 0; - virtual float GetWti(const Context&, const Problem&) const { return -2.0f; }; - virtual size_t GetWorkspaceSize(const Context&, const Problem&) const { return 0; }; + /// Returns true if a Solver can work on given SW/HW platform (runtime/device) + /// and provides correct result for the problem config. + virtual bool IsApplicable(const Context& ctx, const Problem& problem) const = 0; - bool IsApplicable(const ExecutionContext& ctx, const boost::any& problem) const final - { - return IsApplicable(dynamic_cast(ctx), - boost::any_cast(problem)); - } + static constexpr float wti_approximate_worst = -2; - float GetWti(const ExecutionContext& ctx, const boost::any& problem) const final - { - return GetWti(dynamic_cast(ctx), boost::any_cast(problem)); - } + /// [Informative as of Sep 2020] Returns an approximated value of the expected + /// WTI or wti_approximate_worst when this value can't be computed. Tips: + /// * Value 1.0 corresponds to the 100% utilization of HW capabilities as + /// if Direct computational algorithm is used. + /// * [Notice] WTI may exceed 1.0 for highly optimized algorithms like Winograd. + /// * @see https://github.com/ROCm/MIOpen/issues/410 + virtual float GetWti(const Context&, const Problem&) const { return wti_approximate_worst; }; - size_t GetWorkspaceSize(const ExecutionContext& ctx, const boost::any& problem) const final - { - return GetWorkspaceSize(dynamic_cast(ctx), - boost::any_cast(problem)); - } + /// Returns the workspace size required by the solver for the given Problem + virtual size_t GetWorkspaceSize(const Context&, const Problem&) const { return 0; }; }; -/// Base class for non tunable solvers +/// Common interface for non-tunable solvers template -struct NonTunableSolverBase : SolverMixin +struct SolverInterfaceNonTunable : SolverInterface { /// Takes problem config, optimization parameters and other info /// and computes information required to build and run the kernel(s). - virtual ConvSolution GetSolution(const Context&, const Problem&) const = 0; - virtual InvokerFactory GetInvokerFactory(const Context& ctx, const Problem& problem) const + virtual ConvSolution GetSolution(const Context& ctx, const Problem& problem) const = 0; +}; + +/// Common interface for tunable solvers +template +struct SolverInterfaceTunable : SolverInterface +{ + /// This function is a simplified version of FindSolution(), it does not obey search parameters + /// from the Context and does not use the database. Intended to be used in unit tests. + virtual ConvSolution FindSolutionSimple(const Context& ctx, + const Problem& problem, + const AnyInvokeParams& invoke_ctx) const = 0; +}; + +/// Base class for non-tunable solvers +template +struct SolverBaseNonTunable : SolverInterfaceNonTunable +{ + bool IsTunable() const final { return false; }; + + InvokerFactory GetInvokerFactory(const Context& ctx, const Problem& problem) const { - return *GetSolution(ctx, problem).invoker_factory; + const auto solution = this->GetSolution(ctx, problem); + return *solution.invoker_factory; } }; @@ -191,89 +194,57 @@ struct TunableSolverTrait }; /// Base class for tunable solvers -template -struct TunableSolverBase : SolverMixin, TunableSolverTrait +template +struct SolverBaseTunable : SolverInterfaceTunable, TunableSolverTrait { + bool IsTunable() const final { return true; }; + /// Initializes performance config to the default values. /// The function may involve some heuristic to guess the best solution /// configuration. It is assumed that the function takes constant time /// to finish and does not run kernels to measure performance etc. /// The function shall always return valid config. - /// - /// The int parameter is needed only to not change the name of the - /// function in the derived class. Function declarations that differ - /// only by its return type cannot be overloaded. - virtual boost::any - GetDefaultPerformanceConfig(const Context& ctx, const Problem& problem, int) const = 0; + virtual PerformanceConfig GetDefaultPerformanceConfig(const Context& ctx, + const Problem& problem) const = 0; /// Should return false if performance config is wrong for a problem. /// Main use is validation of values read from the perf db. virtual bool IsValidPerformanceConfig(const Context& ctx, const Problem& problem, - const PerfConfig& config) const = 0; + const PerformanceConfig& config) const = 0; /// Search - /// - /// The int parameter is needed only to not change the name of the - /// function in the derived class. Function declarations that differ - /// only by its return type cannot be overloaded. - virtual boost::any Search(const Context& ctx, - const Problem& problem, - const AnyInvokeParams& invoke_ctx, - int) const = 0; + virtual PerformanceConfig + Search(const Context& ctx, const Problem& problem, const AnyInvokeParams& invoke_ctx) const = 0; /// Tunable solvers provide a GetSolution that takes a Context and PerformanceConfig - virtual ConvSolution - GetSolution(const Context& ctx, const Problem& problem, const PerfConfig& config) const = 0; - virtual InvokerFactory - GetInvokerFactory(const Context& ctx, const Problem& problem, const PerfConfig& config) const - { - return *GetSolution(ctx, problem, config).invoker_factory; - } -}; - -template -struct TunableSolverMixin : TunableSolverBase -{ - static_assert(std::is_base_of{}, - "PerformanceConfig must be derived of PerfConfig"); - - virtual PerformanceConfig GetDefaultPerformanceConfig(const Context&, const Problem&) const = 0; - virtual bool - IsValidPerformanceConfig(const Context&, const Problem&, const PerformanceConfig&) const = 0; - virtual PerformanceConfig - Search(const Context&, const Problem&, const AnyInvokeParams&) const = 0; - virtual ConvSolution - GetSolution(const Context&, const Problem&, const PerformanceConfig&) const = 0; + virtual ConvSolution GetSolution(const Context& ctx, + const Problem& problem, + const PerformanceConfig& config) const = 0; - boost::any - GetDefaultPerformanceConfig(const Context& ctx, const Problem& problem, int) const final + ConvSolution FindSolutionSimple(const Context& ctx, + const Problem& problem, + const AnyInvokeParams& invoke_ctx) const final { - return GetDefaultPerformanceConfig(ctx, problem); + const PerformanceConfig config = Search(ctx, problem, invoke_ctx); + return GetSolution(ctx, problem, config); } - bool IsValidPerformanceConfig(const Context& ctx, - const Problem& problem, - const PerfConfig& config) const final + InvokerFactory GetInvokerFactory(const Context& ctx, + const Problem& problem, + const PerformanceConfig& config) const { - return IsValidPerformanceConfig( - ctx, problem, dynamic_cast(config)); + return *GetSolution(ctx, problem, config).invoker_factory; } +}; - boost::any Search(const Context& ctx, - const Problem& problem, - const AnyInvokeParams& invoke_ctx, - int) const final - { - return Search(ctx, problem, invoke_ctx); - } +// \todo Should be removed +template +using NonTunableSolverBase = SolverBaseNonTunable; - ConvSolution - GetSolution(const Context& ctx, const Problem& problem, const PerfConfig& config) const final - { - return GetSolution(ctx, problem, dynamic_cast(config)); - } -}; +// \todo Should be removed +template +using TunableSolverMixin = SolverBaseTunable; template struct IsTunable : std::is_base_of @@ -282,5075 +253,6 @@ struct IsTunable : std::is_base_of "Raw trait shouldn't be passed, explicit type is needed"); }; -namespace conv { - -/// Typedef for convolution non-tunable solvers -using ConvSolver = NonTunableSolverBase; - -/// Typedef for convolution tunable solvers -template -using ConvTunableSolver = - TunableSolverMixin; - -struct PerformanceConfigConvAsm3x3U : PerfConfigBase -{ - int limit_wave_cnt; // [0..9] - int filters_per_wave; // [1..8] - int output_lines_per_wave; // [1..8] - - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm3x3U(int lwc, int fpw, int olpw); - PerformanceConfigConvAsm3x3U() : PerformanceConfigConvAsm3x3U(-1, -1, -1) {} - PerformanceConfigConvAsm3x3U(bool) : PerformanceConfigConvAsm3x3U(0, 1, 1) {} - - template - static void Visit(Self&& self, F f) - { - f(self.limit_wave_cnt, "limit_wave_cnt"); - f(self.filters_per_wave, "filters_per_wave"); - f(self.output_lines_per_wave, "output_lines_per_wave"); - } - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvAsm3x3U& other) const; -}; - -struct ConvAsm3x3U final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm3x3U GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsm3x3U&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm3x3U - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsm3x3U&) const override; -}; - -struct PerformanceConfigConvAsm1x1U : PerfConfigBase -{ - // ----------------- // Full set Optimized Spare - // ---------------------------------------------------------------------------- - int read_size; // [1..4] - int k_mult; // 1,[4,8,12..32] 2^n[8..32] 1,4 - int chunks_per_wave; // [1..16] [1..8] - int chunk_size; // 2^n[1..64] 2^n[16..64] 1,4 - int n_mult; // [1..8] [1..4] - int c_mult; // 2^n[1..32] 2^n[1..4] - int waves_c_in_group; // [1..8] [1..4] - int waves_k_in_group; // 1,[2,4,8] 1,[2,4,8] - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT - PerformanceConfigConvAsm1x1U(int, int, int, int, int, int, int, int, bool); - PerformanceConfigConvAsm1x1U() - : PerformanceConfigConvAsm1x1U(-1, -1, -1, -1, -1, -1, -1, -1, false) - { - } - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1U(bool spare); - - template - static void Visit(Self&& self, F f) - { - f(self.read_size, "read_size"); - f(self.k_mult, "k_mult"); - f(self.chunks_per_wave, "chunks_per_wave"); - f(self.chunk_size, "chunk_size"); - f(self.n_mult, "n_mult"); - f(self.c_mult, "c_mult"); - f(self.waves_c_in_group, "waves_c_in_group"); - f(self.waves_k_in_group, "waves_k_in_group"); - } - - // clang-format off - int GetReadSize() const { return read_size; } - int GetKMult() const { return k_mult; } - int GetChunksPerWave() const { return chunks_per_wave; } - int GetChunkSize() const { return chunk_size; } - int GetNMult() const { return n_mult; } - int GetCMult() const { return c_mult; } - int GetWavesCInGroup() const { return waves_c_in_group; } - int GetWavesKInGroup() const { return waves_k_in_group; } - int GetNPerGpr() const { assert(chunk_size); return 64 / chunk_size; } - // clang-format on - - MIOPEN_INTERNALS_EXPORT void StaticHeuristic(const miopen::conv::ProblemDescription& problem); - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool - IsModelApplicable(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const; - bool IsValidValue() const { return IsValidValueImpl(8); } - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - bool IsValid(const miopen::conv::ProblemDescription& problem) const - { - return IsValidImpl(problem, 8); - } - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvAsm1x1U& other) const; - -private: -#if MIOPEN_ENABLE_AI_KERNEL_TUNING - bool IsPartiallyValid(const miopen::conv::ProblemDescription& problem, - int sequence_length) const - { - return IsValidImpl(problem, sequence_length); - } - bool IsPartiallyValidValue(int sequence_length) const - { - return IsValidValueImpl(sequence_length); - } - bool RunParameterPredictionModel(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - bool ModelApplyToken(int index, std::string value, const miopen::conv::ProblemDescription&); -#endif - bool IsValidImpl(const miopen::conv::ProblemDescription& problem, int sequence_length) const; - bool IsValidValueImpl(int sequence_length) const; -}; - -struct ConvAsm1x1U final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1U GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsm1x1U&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1U - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsm1x1U&) const override; -}; - -struct PerformanceConfigConvAsm1x1UV2 : PerfConfigBase -{ - // ----------------- // Full set Optimized Spare - // ---------------------------------------------------------------------------- - int chunk_size; // 2^n[1..64] 2^n[16..64] - int dwords_per_ld; // [1..4] 1,2,3 - int k_mult; // [1..32] 8,16 1,2,3,4 - int c_mult; // [1..32] 2^n[1..4] - int n_mult; // [1..32] 1,2 - int w_mult; // [1..32] 1,2 - int h_mult; // [1..32] 1,2 - int h_per_chunk; // 2^n[1..64] [2,4,8] - int waves_k_in_group; // [1..8] 2,4 - int waves_c_in_group; // [1..8] 1,2 - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT - PerformanceConfigConvAsm1x1UV2(int, int, int, int, int, int, int, int, int, int, bool); - PerformanceConfigConvAsm1x1UV2() - : PerformanceConfigConvAsm1x1UV2(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, false) - { - } - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1UV2(bool spare); - - template - static void Visit(Self&& self, F f) - { - f(self.chunk_size, "chunk_size"); - f(self.dwords_per_ld, "dwords_per_ld"); - f(self.k_mult, "k_mult"); - f(self.c_mult, "c_mult"); - f(self.n_mult, "n_mult"); - f(self.w_mult, "w_mult"); - f(self.h_mult, "h_mult"); - f(self.h_per_chunk, "h_per_chunk"); - f(self.waves_k_in_group, "waves_k_in_group"); - f(self.waves_c_in_group, "waves_c_in_group"); - } - - // clang-format off - int GetChunkSize() const { return chunk_size; } - int GetDwordsPerLd() const { return dwords_per_ld; } - int GetCMult() const { return c_mult; } - int GetKMult() const { return k_mult; } - int GetNMult() const { return n_mult; } - int GetWMult() const { return w_mult; } - int GetHMult() const { return h_mult; } - int GetHPerChunk() const { return h_per_chunk; } - int GetWavesCInGroup() const { return waves_c_in_group; } - int GetWavesKInGroup() const { return waves_k_in_group; } - int GetNPerGpr() const { assert(chunk_size); return 64 / chunk_size; } - // clang-format on - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvAsm1x1UV2& other) const; -}; - -struct ConvAsm1x1UV2 final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1UV2 GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsm1x1UV2&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsm1x1UV2 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsm1x1UV2&) const override; -}; - -struct ConvAsm5x10u2v2f1 final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsm5x10u2v2b1 final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsm7x7c3h224w224k64u2v2p3q3f1 final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvOclDirectFwd11x11 final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvOclDirectFwdGen final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct PerformanceImplicitGemm : PerfConfigBase -{ - int BPerBlock; // 2^n[8..16] - int KPerBlock; // 2^n[32..128] - int EPerBlock; // 2^n[4..16] - - int GemmNRepeat; // == 2 - - int GemmMPerThreadSubC; // 2^n[2..4] - int GemmNPerThreadSubC; // 2^n[2..4] - - int GemmMLevel0Cluster; // 2^n[1..4] - int GemmNLevel0Cluster; // 2^n[1..4] - int GemmMLevel1Cluster; // 2^n[1..4] - int GemmNLevel1Cluster; // 2^n[1..4] - - int InBlockCopyClusterLengths_E; // 2^n[4..16] - int InBlockCopyClusterLengths_B; // 2^n[8..16] - int InBlockCopyClusterLengths_N1; // 2^n[1..2] - int InBlockCopyClusterLengths_N2; // 2^n[1..4] - - int WeiBlockCopyClusterLengths_E; // 2^n[1..4] - int WeiBlockCopyClusterLengths_K; // 2^n[16..128] - - bool use_spare_set; - - PerformanceImplicitGemm( - int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, int, bool); - - PerformanceImplicitGemm() - : PerformanceImplicitGemm( - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, false) - { - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemm(bool spare); - - template - static void Visit(Self&& self, F f) - { - f(self.BPerBlock, "BPerBlock"); - f(self.KPerBlock, "KPerBlock"); - f(self.EPerBlock, "EPerBlock"); - f(self.GemmNRepeat, "GemmNRepeat"); - f(self.GemmMPerThreadSubC, "GemmMPerThreadSubC"); - f(self.GemmNPerThreadSubC, "GemmNPerThreadSubC"); - f(self.GemmMLevel0Cluster, "GemmMLevel0Cluster"); - f(self.GemmNLevel0Cluster, "GemmNLevel0Cluster"); - f(self.GemmMLevel1Cluster, "GemmMLevel1Cluster"); - f(self.GemmNLevel1Cluster, "GemmNLevel1Cluster"); - f(self.InBlockCopyClusterLengths_E, "InBlockCopyClusterLengths_E"); - f(self.InBlockCopyClusterLengths_N1, "InBlockCopyClusterLengths_N1"); - f(self.InBlockCopyClusterLengths_B, "InBlockCopyClusterLengths_B"); - f(self.InBlockCopyClusterLengths_N2, "InBlockCopyClusterLengths_N2"); - f(self.WeiBlockCopyClusterLengths_E, "WeiBlockCopyClusterLengths_E"); - f(self.WeiBlockCopyClusterLengths_K, "WeiBlockCopyClusterLengths_K"); - } - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemm& other) const; -}; - -struct PerformanceImplicitGemmV4R1 : public PerformanceImplicitGemm -{ - PerformanceImplicitGemmV4R1(int a, - int b, - int c, - int d, - int e, - int f, - int g, - int h, - int i, - int j, - int k, - int l, - int m, - int n, - int o, - int p, - bool q) - : PerformanceImplicitGemm(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) - { - } - - PerformanceImplicitGemmV4R1() - : PerformanceImplicitGemmV4R1( - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, false) - { - } - - PerformanceImplicitGemmV4R1(bool spare) : PerformanceImplicitGemm(spare) {} - - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceImplicitGemmV4R4Fwd : PerfConfigBase -{ - int BlockSize; - - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - - int GemmMPerThread; - int GemmNPerThread; - - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4Fwd(int, int, int, int, int, int, bool); - - PerformanceImplicitGemmV4R4Fwd(int a, int b, int c, int d, int e, int f) - : PerformanceImplicitGemmV4R4Fwd(a, b, c, d, e, f, false) - { - } - - PerformanceImplicitGemmV4R4Fwd() : PerformanceImplicitGemmV4R4Fwd(-1, -1, -1, -1, -1, -1, false) - { - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4Fwd(bool spare); - - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemmV4R4Fwd& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.BlockSize, "BlockSize"); - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerThread, "GemmMPerThread"); - f(self.GemmNPerThread, "GemmNPerThread"); - } - - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateBlockGemmPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); -}; - -struct PerformanceImplicitGemmV4R4WrW : PerfConfigBase -{ - int BlockSize; - - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - - int GemmMPerThread; - int GemmNPerThread; - - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4WrW(int, int, int, int, int, int, bool); - - PerformanceImplicitGemmV4R4WrW(int a, int b, int c, int d, int e, int f) - : PerformanceImplicitGemmV4R4WrW(a, b, c, d, e, f, false) - { - } - - PerformanceImplicitGemmV4R4WrW() : PerformanceImplicitGemmV4R4WrW(-1, -1, -1, -1, -1, -1, false) - { - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4WrW(bool spare); - - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemmV4R4WrW& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.BlockSize, "BlockSize"); - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerThread, "GemmMPerThread"); - f(self.GemmNPerThread, "GemmNPerThread"); - } - - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateBlockGemmPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); -}; - -struct PerformanceImplicitGemmBwdDataV1R1 : PerfConfigBase -{ - int BlockSize; - - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - - int GemmMPerThread; - int GemmNPerThread; - - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV1R1(int, int, int, int, int, int, bool); - - PerformanceImplicitGemmBwdDataV1R1() - : PerformanceImplicitGemmBwdDataV1R1(-1, -1, -1, -1, -1, -1, false) - { - } - - PerformanceImplicitGemmBwdDataV1R1(int a, int b, int c, int d, int e, int f) - : PerformanceImplicitGemmBwdDataV1R1(a, b, c, d, e, f, false) - { - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV1R1(bool spare); - - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemmBwdDataV1R1& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.BlockSize, "BlockSize"); - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerThread, "GemmMPerThread"); - f(self.GemmNPerThread, "GemmNPerThread"); - } - - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateBlockGemmPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); -}; - -struct PerformanceImplicitGemmBwdDataV4R1 : PerfConfigBase -{ - int BlockSize; - - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - - int GemmMPerThread; - int GemmNPerThread; - - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1(int, int, int, int, int, int, bool); - - PerformanceImplicitGemmBwdDataV4R1() - : PerformanceImplicitGemmBwdDataV4R1(-1, -1, -1, -1, -1, -1, false) - { - } - - PerformanceImplicitGemmBwdDataV4R1(int a, int b, int c, int d, int e, int f) - : PerformanceImplicitGemmBwdDataV4R1(a, b, c, d, e, f, false) - { - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1(bool spare); - - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceImplicitGemmBwdDataV4R1& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.BlockSize, "BlockSize"); - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerThread, "GemmMPerThread"); - f(self.GemmNPerThread, "GemmNPerThread"); - } - - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateBlockGemmPerformanceParameters() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmCThreadCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple MIOPEN_INTERNALS_EXPORT - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); -}; - -struct PerformanceImplicitGemmBwdDataV4R1Xdlops - : PerfConfigBase -{ - int GemmNPerBlock; // 2^n[8..16] - int GemmMPerBlock; // 2^n[32..128] - int GemmKPerBlock; // 2^n[4..16] - - int GemmKPACKSize; // 2^[1..4] - - int GemmMPerWave; - int GemmNPerWave; - - // GemmAThreadCopyMoreGemmK is currently a fix value, is untunable - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmKPack; - - bool use_spare_set; - MIOPEN_INTERNALS_EXPORT - PerformanceImplicitGemmBwdDataV4R1Xdlops(int, int, int, int, int, int, bool, bool, bool); - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops(); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops(bool spare); - PerformanceImplicitGemmBwdDataV4R1Xdlops( - int a, int b, int c, int d, int e, int f, bool g, bool h) - : PerformanceImplicitGemmBwdDataV4R1Xdlops(a, b, c, d, e, f, g, h, false) - { - } - - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmBwdDataV4R1Xdlops& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmKPACKSize, "GemmKPACKSize"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); - } - - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); -}; - -struct ConvHipImplicitGemmV4R1Fwd final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R1&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R1&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; -}; - -struct ConvHipImplicitGemmV4R4Fwd final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4Fwd GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R4Fwd&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4Fwd - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R4Fwd&) const override; - -private: - static std::tuple CalculateGemmSize(const miopen::conv::ProblemDescription&); - - friend struct PerformanceImplicitGemmV4R4Fwd; -}; - -struct PerformanceConvMlirIgemm : PerfConfigBase -{ - int BlockSize; - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - int GemmMPerThread; - int GemmNPerThread; - bool use_spare_set; - - /// \ref https://github.com/ROCm/MIOpen/issues/1154 - static PerformanceConvMlirIgemm& MlirHeuristicInitRequest() - { - static PerformanceConvMlirIgemm heur; - heur.SetMlirHeuristicInitRequest(); - return heur; - } - - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm(int, int, int, int, int, int, bool); - - PerformanceConvMlirIgemm(int a, int b, int c, int d, int e, int f) - : PerformanceConvMlirIgemm(a, b, c, d, e, f, false) - { - } - - PerformanceConvMlirIgemm() : PerformanceConvMlirIgemm(-1, -1, -1, -1, -1, -1, false) {} - - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm(bool spare); - - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConvMlirIgemm& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.BlockSize, "BlockSize"); - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerThread, "GemmMPerThread"); - f(self.GemmNPerThread, "GemmNPerThread"); - } - - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - -private: - void SetMlirHeuristicInitRequest(); -}; - -struct ConvMlirIgemmFwd final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemm&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemm&) const override; -}; - -struct PerformanceConvMlirIgemmXdlops : PerfConfigBase -{ - int GemmMPerBlock; // 2^n[32..128] - int GemmNPerBlock; // 2^n[8..16] - int GemmKPerBlock; // 2^n[4..16] - int GemmMPerWave; - int GemmNPerWave; - int GemmKPACKSize; // 2^[1..4] - - // GemmAThreadCopyMoreGemmK is currently a fix value, is untunable - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmKPack; - - bool use_spare_set; - - /// \ref https://github.com/ROCm/MIOpen/issues/1154 - static PerformanceConvMlirIgemmXdlops& MlirHeuristicInitRequest() - { - static PerformanceConvMlirIgemmXdlops heur; - heur.SetMlirHeuristicInitRequest(); - return heur; - } - - MIOPEN_INTERNALS_EXPORT - PerformanceConvMlirIgemmXdlops(int, int, int, int, int, int, bool, bool, bool); - - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops(); - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops(bool spare); - PerformanceConvMlirIgemmXdlops(int a, int b, int c, int d, int e, int f, bool g, bool h) - : PerformanceConvMlirIgemmXdlops(a, b, c, d, e, f, g, h, false) - { - } - - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConvMlirIgemmXdlops& other) const; - - template - static void Visit(Self&& self, F f) - { - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmKPACKSize, "GemmKPACKSize"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); - } - - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - -private: - void SetMlirHeuristicInitRequest(); -}; - -struct ConvMlirIgemmFwdXdlops final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; -}; - -struct ConvHipImplicitGemmV4R4WrW final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4WrW GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R4WrW&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R4WrW - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R4WrW&) const override; - -private: - static std::tuple CalculateGemmSize(const miopen::conv::ProblemDescription&); - - friend struct PerformanceImplicitGemmV4R4WrW; -}; - -struct ConvMlirIgemmWrW final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemm&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemm&) const override; -}; - -struct ConvMlirIgemmWrWXdlops final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; -}; - -struct PerformanceImplicitGemmForwardV4R4Xdlops - : PerfConfigBase -{ - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - int GemmMPerWave; - int GemmNPerWave; - int GemmKPack; - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmKPack; - int GemmBThreadDataPerRead_GemmN; - - MIOPEN_INTERNALS_EXPORT - PerformanceImplicitGemmForwardV4R4Xdlops(int, int, int, int, int, int, bool, bool, int); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops(); - PerformanceImplicitGemmForwardV4R4Xdlops(bool) : PerformanceImplicitGemmForwardV4R4Xdlops() {} - - template - static void Visit(Self&& self, F f) - { - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmKPack, "GemmKPack"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); - f(self.GemmBThreadDataPerRead_GemmN, "GemmBThreadDataPerRead_GemmN"); - } - - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmForwardV4R4Xdlops& other) const; - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceImplicitGemmForwardV4R5Xdlops - : PerfConfigBase -{ - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - int GemmMPerWave; - int GemmNPerWave; - int GemmKPack; - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmKPack; - int GemmBThreadDataPerRead_GemmN; - - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT - PerformanceImplicitGemmForwardV4R5Xdlops(int, int, int, int, int, int, bool, bool, int, bool); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops(); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops(bool spare); - - PerformanceImplicitGemmForwardV4R5Xdlops( - int a, int b, int c, int d, int e, int f, bool g, bool h, int i) - : PerformanceImplicitGemmForwardV4R5Xdlops(a, b, c, d, e, f, g, h, i, false) - { - } - - template - static void Visit(Self&& self, F f) - { - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmKPack, "GemmKPack"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); - f(self.GemmBThreadDataPerRead_GemmN, "GemmBThreadDataPerRead_GemmN"); - } - - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmForwardV4R5Xdlops& other) const; - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm - : PerfConfigBase -{ - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - int GemmMPerWave; - int GemmNPerWave; - int GemmKPack; - int GemmMFactor; - int GemmNFactor; - int GemmKFactor; - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmKPack; - int GemmBThreadDataPerRead_GemmN; - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm( - int, int, int, int, int, int, int, int, int, bool, bool, int); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm(); - PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm(bool) - : PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm() - { - } - - template - static void Visit(Self&& self, F f) - { - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmKPack, "GemmKPack"); - f(self.GemmMFactor, "GemmMFactor"); - f(self.GemmNFactor, "GemmNFactor"); - f(self.GemmKFactor, "GemmKFactor"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); - f(self.GemmBThreadDataPerRead_GemmN, "GemmBThreadDataPerRead_GemmN"); - } - - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm& other) const; - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceImplicitGemmBwdV1R1Xdlops : PerfConfigBase -{ - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - int GemmMPerWave; - int GemmNPerWave; - int GemmKPack; - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmKPack; - - MIOPEN_INTERNALS_EXPORT - PerformanceImplicitGemmBwdV1R1Xdlops(int, int, int, int, int, int, bool, bool); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdV1R1Xdlops(); - PerformanceImplicitGemmBwdV1R1Xdlops(bool) : PerformanceImplicitGemmBwdV1R1Xdlops() {} - - template - static void Visit(Self&& self, F f) - { - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmKPack, "GemmKPack"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmKPack, "GemmBThreadCopyMoreGemmKPack"); - } - - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmBwdV1R1Xdlops& other) const; - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmForwardV4R4Xdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R4Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R4Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - -private: - static std::tuple - CalculateGemmSize(const miopen::conv::ProblemDescription&); - - friend struct PerformanceImplicitGemmForwardV4R4Xdlops; -}; - -struct ConvHipImplicitGemmForwardV4R4Xdlops_Padded_Gemm final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - -private: - static std::tuple CalculateGemmSize( - const miopen::conv::ProblemDescription&, int GemmMFactor, int GemmNFactor, int GemmKFactor); - - friend struct PerformanceImplicitGemmForwardV4R4Xdlops_Padded_Gemm; -}; - -struct ConvHipImplicitGemmForwardV4R5Xdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R5Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R5Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R5Xdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; -}; - -struct ConvHipImplicitGemmV4R1WrW final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R1&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmV4R1&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmV4R1 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; -}; - -struct ConvHipImplicitGemmBwdDataV1R1 final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV1R1 GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdDataV1R1&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV1R1 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdDataV1R1&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool MayNeedWorkspace() const override { return true; } - -private: - static std::tuple CalculateGemmSize(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - - friend struct PerformanceImplicitGemmBwdDataV1R1; -}; - -struct ConvMlirIgemmBwd final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemm&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemm - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemm&) const override; -}; - -struct ConvMlirIgemmBwdXdlops final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvMlirIgemmXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvMlirIgemmXdlops&) const override; -}; - -struct ConvHipImplicitGemmBwdDataV4R1 final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1 GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdDataV4R1&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdDataV4R1&) const override; - -private: - static int CalculateNumberOfGemm(const miopen::conv::ProblemDescription&); - static std::tuple CalculateGemmSize(const miopen::conv::ProblemDescription&, - int gemm_id); - - friend struct PerformanceImplicitGemmBwdDataV4R1; -}; - -struct ConvHipImplicitGemmBwdDataV4R1Xdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdDataV4R1Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdDataV4R1Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdDataV4R1Xdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - -private: - static int CalculateNumberOfGemm(const miopen::conv::ProblemDescription&); - static std::tuple CalculateGemmSize(const miopen::conv::ProblemDescription&, - int gemm_id); - - friend struct PerformanceImplicitGemmBwdDataV4R1Xdlops; -}; - -struct ConvHipImplicitGemmBwdDataV1R1Xdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdV1R1Xdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdV1R1Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmBwdV1R1Xdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmBwdV1R1Xdlops&) const override; - -private: - static std::tuple - CalculateGemmSize(const miopen::conv::ProblemDescription&); - - friend struct PerformanceImplicitGemmBwdV1R1Xdlops; -}; - -struct ConvAsmImplicitGemmV4R1DynamicFwd final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsmImplicitGemmV4R1DynamicFwd_1x1 final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsmImplicitGemmV4R1DynamicWrw final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsmImplicitGemmGTCDynamicWrwXdlops final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsmImplicitGemmV4R1DynamicBwd final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsmImplicitGemmGTCDynamicFwdXdlops final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvAsmImplicitGemmGTCDynamicBwdXdlops final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -/// Holds common member functions for the Solvers which share the same -/// "legacy exhaustive search" machinery. -struct ConvOclDirectFwdLegacyExhaustiveSearch : ConvTunableSolver -{ - MIOPEN_INTERNALS_EXPORT LegacyPerformanceConfig GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT LegacyPerformanceConfig - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - -private: - template - LegacyPerformanceConfig SearchImpl(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const; -}; - -struct ConvOclDirectFwd : ConvOclDirectFwdLegacyExhaustiveSearch -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT static ConvSolution - BaseGetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const LegacyPerformanceConfig&); - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const LegacyPerformanceConfig&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const LegacyPerformanceConfig&) const override; -}; - -struct ConvOclDirectFwd1x1 final : ConvOclDirectFwdLegacyExhaustiveSearch -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const LegacyPerformanceConfig&) const override; - - bool IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const LegacyPerformanceConfig&) const override - { - return true; - } -}; - -struct ConvBinWinograd3x3U final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvBinWinogradRxS final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct PerformanceConfigConvBinWinogradRxS : PerfConfigBase -{ - int n_groups; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvBinWinogradRxS(int n_groups_); - PerformanceConfigConvBinWinogradRxS() : PerformanceConfigConvBinWinogradRxS(-1) {} - PerformanceConfigConvBinWinogradRxS(bool) : PerformanceConfigConvBinWinogradRxS(1) {} - - template - static void Visit(Self&& self, F f) - { - f(self.n_groups, "n_groups"); - } - int GetNGroups() const { return n_groups; } - - template - void HeuristicInit(const ExecutionContext&, const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - bool IsValid(const ExecutionContext& ctx, const miopen::conv::ProblemDescription&) const - { - return IsValid(ctx); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&) const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvBinWinogradRxS& other) const; -}; - -template -struct ConvBinWinoRxS final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - static const std::string& GetSolverDbId() - { - static const std::string dbId = std::string("ConvBinWinogradRxSf") - .append(std::to_string(Winodata)) - .append("x") - .append(std::to_string(Winofilter)); - return dbId; - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvBinWinogradRxS GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvBinWinogradRxS&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvBinWinogradRxS - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvBinWinogradRxS&) const override; - -private: - static size_t GetNGroups(const size_t group_conv, const size_t grid_group_size) - { - assert(group_conv != 0); - return grid_group_size / group_conv; - } -}; - -// Suppress misleading clang warnings -#if defined(__clang__) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wweak-template-vtables" -#endif - -extern template struct ConvBinWinoRxS<2, 3>; -extern template struct ConvBinWinoRxS<3, 2>; - -#if defined(__clang__) -#pragma clang diagnostic pop -#endif - -struct ConvBinWinogradRxSf2x3g1 final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT float GetWti(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -template -struct ConvMPBidirectWinograd final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId< - ConvMPBidirectWinograd>(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - // kernel_file_name for solver identification - static fs::path GetSolverFileNames(int id) - { - static const fs::path names[3] = {"xform_bidirect_winograd_data.s", - "xform_bidirect_winograd_filter.s", - "xform_bidirect_winograd_out.s"}; - return names[id]; - } - - static std::string GetSolverKernelNames(int id) - { - static const std::string name_suffix = - '_' + std::to_string(WinoDataH) + '_' + std::to_string(WinoDataW) + '_' + - std::to_string(WinoFilterH) + '_' + std::to_string(WinoFilterW); - static const std::string names[3] = { - "miopenGcnAsmMPBidirectWinogradXformData" + name_suffix, - "miopenGcnAsmMPBidirectWinogradXformFilter" + name_suffix, - "miopenGcnAsmMPBidirectWinogradXformOut" + name_suffix}; - return names[id]; - } - - static int GetSolverWinoXformHWSize() { return WinoDataH + WinoFilterH - 1; } -}; - -// To suppress misleading clang warnings -#if defined(__clang__) && defined(CONV_MP_BIDIRECTIONAL_WINOGRAD_CPP) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wweak-template-vtables" -#endif - -extern template struct ConvMPBidirectWinograd<2, 3>; -extern template struct ConvMPBidirectWinograd<3, 3>; -extern template struct ConvMPBidirectWinograd<4, 3>; -extern template struct ConvMPBidirectWinograd<5, 3>; -extern template struct ConvMPBidirectWinograd<6, 3>; - -#if defined(__clang__) && defined(CONV_MP_BIDIRECTIONAL_WINOGRAD_CPP) -#pragma clang diagnostic pop -#endif - -template -struct ConvMPBidirectWinograd_xdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId< - ConvMPBidirectWinograd_xdlops>(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override - { - return ConvHipImplicitGemmForwardV4R4Xdlops{}.IsDynamic() && - ConvMPBidirectWinograd{} - .IsDynamic() && - IsThisSolverDynamic(); - } - - PerformanceImplicitGemmForwardV4R4Xdlops - GetDefaultPerformanceConfig(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const override - { - const auto xdlops_problem = GetTransformedProblem(problem); - const auto xdlops_ctx = GetTransformedConvContext(ctx, xdlops_problem); - - return ConvHipImplicitGemmForwardV4R4Xdlops{}.GetDefaultPerformanceConfig(xdlops_ctx, - xdlops_problem); - } - - bool - IsValidPerformanceConfig(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem, - const PerformanceImplicitGemmForwardV4R4Xdlops& config) const override - { - const auto xdlops_problem = GetTransformedProblem(problem); - const auto xdlops_ctx = GetTransformedConvContext(ctx, xdlops_problem); - - return ConvHipImplicitGemmForwardV4R4Xdlops{}.IsValidPerformanceConfig( - xdlops_ctx, xdlops_problem, config); - } - - size_t GetWorkspaceSize(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const override - { - const auto xdlops_problem = GetTransformedProblem(problem); - const auto xdlops_ctx = GetTransformedConvContext(ctx, xdlops_problem); - - return ConvMPBidirectWinograd() - .GetWorkspaceSize(ctx, problem) + - ConvHipImplicitGemmForwardV4R4Xdlops{}.GetWorkspaceSize(xdlops_ctx, xdlops_problem); - } - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmForwardV4R4Xdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmForwardV4R4Xdlops&) const override; - -private: - ExecutionContext - GetTransformedConvContext(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& transformed_problem) const; - miopen::conv::ProblemDescription - GetTransformedProblem(const miopen::conv::ProblemDescription& problem) const; - - // kernel_file_name for solver identification - static fs::path GetSolverFileNames(int id) - { - return ConvMPBidirectWinograd:: - GetSolverFileNames(id); - } - - static std::string GetSolverKernelNames(int id) - { - return ConvMPBidirectWinograd:: - GetSolverKernelNames(id); - } - - static int GetSolverWinoXformHWSize() - { - return ConvMPBidirectWinograd:: - GetSolverWinoXformHWSize(); - } - - bool IsThisSolverDynamic() const { return true; } -}; - -// To suppress misleading clang warnings -#if defined(__clang__) && defined(CONV_MP_BIDIRECTIONAL_WINOGRAD_CPP) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wweak-template-vtables" -#endif - -extern template struct ConvMPBidirectWinograd_xdlops<2, 3>; -extern template struct ConvMPBidirectWinograd_xdlops<3, 3>; -extern template struct ConvMPBidirectWinograd_xdlops<4, 3>; -extern template struct ConvMPBidirectWinograd_xdlops<5, 3>; -extern template struct ConvMPBidirectWinograd_xdlops<6, 3>; - -#if defined(__clang__) && defined(CONV_MP_BIDIRECTIONAL_WINOGRAD_CPP) -#pragma clang diagnostic pop -#endif - -template -struct ConvWinograd3x3MultipassWrW final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId< - ConvWinograd3x3MultipassWrW>(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool IsDynamic() const override { return true; } - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - // kernel_file_name for solver identification - static fs::path GetSolverFileNames(int id) - { - static const fs::path names[3] = {"xform_data.s", "xform_filter.s", "xform_out.s"}; - return names[id]; - } - - static std::string GetSolverKernelNames(int id) - { - static const std::string name_suffix = - '_' + std::to_string(WinoDataH) + '_' + std::to_string(WinoDataW) + '_' + - std::to_string(WinoFilterH) + '_' + std::to_string(WinoFilterW); - static const std::string names[3] = {"miopenGcnAsmWinogradXformData" + name_suffix, - "miopenGcnAsmWinogradXformFilter" + name_suffix, - "miopenGcnAsmWinogradXformOut" + name_suffix}; - - return names[id]; - } - - static int GetGroupCountMult() { return 4; } - - static int GetSolverWinoXformHWSize(const miopen::conv::ProblemDescription& problem, int id) - { - if(id == 0) - { - return WinoDataH + - (WinoFilterH - 1) * (WinoDataH == 7 ? 2 : problem.GetKernelStrideH()); - } - else - { - return WinoDataW + - (WinoFilterW - 1) * (WinoDataW == 7 ? 2 : problem.GetKernelStrideW()); - } - } - -private: - InvokerFactory PrepareInvokerFactory(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - std::size_t ws_sz) const; -}; - -// To suppress misleading clang warnings -#if defined(__clang__) && defined(CONV_MULTIPASS_WINO3X3WRW_CPP) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wweak-template-vtables" -#endif - -extern template struct ConvWinograd3x3MultipassWrW<3, 2>; -extern template struct ConvWinograd3x3MultipassWrW<3, 3>; -extern template struct ConvWinograd3x3MultipassWrW<3, 4>; -extern template struct ConvWinograd3x3MultipassWrW<3, 5>; -extern template struct ConvWinograd3x3MultipassWrW<3, 6>; -extern template struct ConvWinograd3x3MultipassWrW<7, 2>; -extern template struct ConvWinograd3x3MultipassWrW<7, 3>; -extern template struct ConvWinograd3x3MultipassWrW<1, 1, 7, 2>; -extern template struct ConvWinograd3x3MultipassWrW<1, 1, 7, 3>; -extern template struct ConvWinograd3x3MultipassWrW<7, 2, 1, 1>; -extern template struct ConvWinograd3x3MultipassWrW<7, 3, 1, 1>; -extern template struct ConvWinograd3x3MultipassWrW<5, 3>; -extern template struct ConvWinograd3x3MultipassWrW<5, 4>; - -#if defined(__clang__) && defined(CONV_MULTIPASS_WINO3X3WRW_CPP) -#pragma clang diagnostic pop -#endif - -struct PerformanceConfigAsmDirect3x3WrW : PerfConfigBase -{ - int limit_wave_cnt; // [0..9] - int reverse_inout; // [0..1], 1 is allowed for stride=1x1 only. - int chunk_size; // {16,8}, Smaller values increase register pressure. - int k_per_wave; // {1,2,4,8} && ((chunk_size * k_per_wave) <= 64). - // Higher values increase register pressure. - int pipe_lines_depth; // [1..16] && (pipe_lines_depth <= img_h). - // Higher values increase register pressure. - int n_per_group; // [1..8] && (n_per_group <= batch_size). - - PerformanceConfigAsmDirect3x3WrW(int lwc, int rio, int csz, int kpw, int pld, int npg); - PerformanceConfigAsmDirect3x3WrW() : PerformanceConfigAsmDirect3x3WrW(-1, -1, -1, -1, -1, -1) {} - PerformanceConfigAsmDirect3x3WrW(bool) : PerformanceConfigAsmDirect3x3WrW(0, 0, 8, 1, 1, 1) {} - - template - static void Visit(Self&& self, F f) - { - f(self.limit_wave_cnt, "limit_wave_cnt"); - f(self.reverse_inout, "reverse_inout"); - f(self.chunk_size, "chunk_size"); - f(self.k_per_wave, "k_per_wave"); - f(self.pipe_lines_depth, "pipe_lines_depth"); - f(self.n_per_group, "n_per_group"); - } - - // clang-format off - int GetLimitWaveCnt() const { return limit_wave_cnt; } - int GetReverseInout() const { return reverse_inout; } - int GetChunkSize() const { return chunk_size; } - int GetKPerWave() const { return k_per_wave; } - int GetPipeLinesDepth() const { return pipe_lines_depth; } - int GetNPerGroup() const { return n_per_group; } - int GetCPerWave() const { assert(chunk_size); return 64 / chunk_size; } // clang-format on - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigAsmDirect3x3WrW& other) const; -}; - -struct ConvAsmBwdWrW3x3 final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmDirect3x3WrW GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmDirect3x3WrW&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmDirect3x3WrW - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmDirect3x3WrW& config) const override; -}; - -template -struct ConvWinoFuryRxS final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId>(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT float GetWti(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -// Suppress misleading clang warnings -#if defined(__clang__) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wweak-template-vtables" -#endif - -extern template struct ConvWinoFuryRxS<2, 3>; -// extern template struct ConvWinoFuryRxS<3, 2>; - -#if defined(__clang__) -#pragma clang diagnostic pop -#endif - -struct PerformanceConfigConvAsmBwdWrW1x1 : PerfConfigBase -{ - - int chunk_size; // {1,2,4,8,16} - int c_per_gpr; // {1,2,4,8,16} - int c_mult; // {1,2,4,8,16} - int k_per_gpr; // {1,2,4,8,16} - int k_mult; // {1,2,4,8,16} - int n_per_gpr; // {1,2,4} - int n_part_cnt; // [1..8] - int read_size; // [1..4] - int short_store; // {0,1} - int data_prefetch; // [0..4] - bool use_spare_set; - - /// The following conditions must be met. - /// - /// Shader design-related constraints: - /// - (A) (chunk_size * c_per_gpr) == 16 - /// - (B) k_per_gpr <= c_per_gpr - /// - (C) (c_mult > 1 || k_mult > 1) - /// ? ((fwd_C % (c_per_gpr * c_mult) == 0) && (fwd_K % (k_per_gpr * k_mult) == 0)) - /// : (true) - /// - /// Resource-related constraints: - /// - (D) c_mult * k_mult * k_per_gpr + 9 + (c_mult + k_mult) * read_size * pipe_depth <= 256 - /// - /// Where: - /// - fwd_C := Num input channels for forward convolution (-c). - /// For backward, this is actually n_outputs. - /// - fwd_K := Num output channels for forward convolution (-k). - /// For backward, this is actually n_inputs. - - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsmBwdWrW1x1(int chunk_size_, - int c_per_gpr_, - int c_mult_, - int k_per_gpr_, - int k_mult_, - int n_per_gpr_, - int n_part_cnt_, - int read_size_, - int short_store_, - int data_prefetch_, - bool); - PerformanceConfigConvAsmBwdWrW1x1() - : PerformanceConfigConvAsmBwdWrW1x1(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, false) - { - } - PerformanceConfigConvAsmBwdWrW1x1(bool spare) - : PerformanceConfigConvAsmBwdWrW1x1(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, spare) - { - } - - template - static void Visit(Self&& self, F f) - { - f(self.chunk_size, "chunk_size"); - f(self.c_per_gpr, "c_per_gpr"); - f(self.c_mult, "c_mult"); - f(self.k_per_gpr, "k_per_gpr"); - f(self.k_mult, "k_mult"); - f(self.n_per_gpr, "n_per_gpr"); - f(self.n_part_cnt, "n_part_cnt"); - f(self.read_size, "read_size"); - f(self.short_store, "short_store"); - f(self.data_prefetch, "data_prefetch"); - } - - // clang-format off - int GetChunkSize() const { return chunk_size; } - int GetCPerGpr() const { return c_per_gpr; } - int GetCMult() const { return c_mult; } - int GetKPerGpr() const { return k_per_gpr; } - int GetKMult() const { return k_mult; } - int GetNPerGpr() const { return n_per_gpr; } - int GetNPartCnt() const { return n_part_cnt; } - int GetHWPerGpr() const { assert(c_per_gpr); assert(n_per_gpr); assert(chunk_size); - return wave_size / (c_per_gpr * n_per_gpr * chunk_size); } // "hw" stands for "height-and-width". - int GetReadSize() const { return read_size; } - int GetShortStore() const {return short_store; } - int GetDataPrefetch() const { return data_prefetch; } - // clang-format on - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigConvAsmBwdWrW1x1& other) const; -}; - -struct ConvAsmBwdWrW1x1 final : ConvTunableSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsmBwdWrW1x1 MIOPEN_INTERNALS_EXPORT - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsmBwdWrW1x1&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvAsmBwdWrW1x1 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvAsmBwdWrW1x1&) const override; -}; - -/// N_BATCH_LOOPS - {1,2,4,8,16} Num batches processed in single workitem. -/// Required workspace size depends on it. However there is a restriction in the internal -/// Solver API that this shouldn't be so. Therefore the family of Solvers created. -/// Each Solver in the family has constant value of this parameter. -template -struct PerformanceConfigConvOclBwdWrw2 - : PerfConfigBase> -{ - // Num waves involved a workgroup. - int n_waves = -1; // {1,2,4,8} - // Num values to read in a workitem (read_unit). - int read_size = -1; // [6..12] - // Num of output channels (top/bottom layer in forward/backward direction) - // that share the same input channel in single workgroup. - // Also represents number of output channels in single tile. - int n_out_channels_per_tile = -1; // {1,2,4,8} - // How many tiles of output channels are processed in a single workgroup? - // n_out_channels_in_lcl * n_out_channels_tiles = total number of - // output channels processed in single workgroup. - int n_out_channels_tiles = -1; // {1,2,4,8} - // Num of output rows processed in a single iteration of loop in a workitem - // (N_ALIGNED_OUT_SCAN_BLK). - int n_out_rows_in_lcl = -1; // [2..11] - - PerformanceConfigConvOclBwdWrw2(int nw, int rs, int nocpt, int noct, int noril) - : n_waves(nw), - read_size(rs), - n_out_channels_per_tile(nocpt), - n_out_channels_tiles(noct), - n_out_rows_in_lcl(noril) - { - } - PerformanceConfigConvOclBwdWrw2() {} - PerformanceConfigConvOclBwdWrw2(bool) : PerformanceConfigConvOclBwdWrw2(1, 6, 1, 1, 2) {} - // spare_set is not used in this solver. - - template - static void Visit(Self&& self, F f) - { - f(self.n_waves, "n_waves"); - f(self.read_size, "read_size"); - f(self.n_out_channels_per_tile, "n_out_channels_per_tile"); - f(self.n_out_channels_tiles, "n_out_channels_tiles"); - f(self.n_out_rows_in_lcl, "n_out_rows_in_lcl"); - } - - // clang-format off - int GetNumWaves() const { return n_waves; } - int GetReadSize() const { return read_size; } - int GetNumOutChannelsPerTile() const { return n_out_channels_per_tile; } - int GetNumOutChannelTiles() const { return n_out_channels_tiles; } - int GetNumOutRowsPerIterPerWork() const { return n_out_rows_in_lcl; } // clang-format on - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigConvOclBwdWrw2& other) const; -}; - -template -struct ConvOclBwdWrW2 : ConvTunableSolver> -{ - const std::string& SolverDbId() const override - { - return this->template GetSolverDbId>(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvOclBwdWrw2 - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvOclBwdWrw2&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigConvOclBwdWrw2 - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigConvOclBwdWrw2&) const override; - -protected: - bool IsApplicableBase(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; -}; - -// To suppress misleading clang warnings -#if defined(__clang__) && defined(CONV_OCL_DIR2D_BWDWRW_2_CPP) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wweak-template-vtables" -#endif - -extern template struct PerformanceConfigConvOclBwdWrw2<1>; -extern template struct PerformanceConfigConvOclBwdWrw2<2>; -extern template struct PerformanceConfigConvOclBwdWrw2<4>; -extern template struct PerformanceConfigConvOclBwdWrw2<8>; -extern template struct PerformanceConfigConvOclBwdWrw2<16>; - -extern template struct ConvOclBwdWrW2<1>; -extern template struct ConvOclBwdWrW2<2>; -extern template struct ConvOclBwdWrW2<4>; -extern template struct ConvOclBwdWrW2<8>; -extern template struct ConvOclBwdWrW2<16>; - -#if defined(__clang__) && defined(CONV_OCL_DIR2D_BWDWRW_2_CPP) -#pragma clang diagnostic pop -#endif - -/// A separate solver from ConvOclBwdWrW2 to disable auto-tuning for certain configs. -/// Basically, this is *hack* for non-group 3x3 and 1x1 cases. -/// It is assumed that Solutions provided by the ConvOclBwdWrW2 solver -/// would never beat 3x3 and 1x1 assembly WrW kernels, even after tuning. -struct ConvOclBwdWrW2NonTunable final : ConvOclBwdWrW2<1> -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - InvokerFactory GetInvokerFactory(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const - { - return *GetSolution(ctx, problem).invoker_factory; - } - -private: - // This function dervied from ConvOclBwdWrW2 is declared private - // so that this solver is not marked searchable/tunable. - using ConvOclBwdWrW2<1>::GetDefaultPerformanceConfig; - using ConvOclBwdWrW2<1>::GetSolution; - using ConvOclBwdWrW2<1>::GetInvokerFactory; -}; - -struct ConvOclBwdWrW53 final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvOclBwdWrW1x1 final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct fft final : ConvSolver -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct PerformanceImplicitGemmWrwV4R4Xdlops : PerfConfigBase -{ - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - int GemmMPerWave; - int GemmNPerWave; - int GemmKPack; - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmK; - bool use_spare_set; - - MIOPEN_INTERNALS_EXPORT - PerformanceImplicitGemmWrwV4R4Xdlops(int, int, int, int, int, int, bool, bool, bool); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops(); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops(bool spare); - PerformanceImplicitGemmWrwV4R4Xdlops(int a, int b, int c, int d, int e, int f, bool g, bool h) - : PerformanceImplicitGemmWrwV4R4Xdlops(a, b, c, d, e, f, g, h, false) - { - } - - template - static void Visit(Self&& self, F f) - { - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmKPack, "GemmKPack"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmK, "GemmBThreadCopyMoreGemmK"); - } - - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmWrwV4R4Xdlops& other) const; - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmSizeAndGemmKBlock(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGridSize(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmWrwV4R4Xdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmWrwV4R4Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmWrwV4R4Xdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; -}; - -struct PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm - : PerfConfigBase -{ - int GemmMPerBlock; - int GemmNPerBlock; - int GemmKPerBlock; - int GemmMPerWave; - int GemmNPerWave; - int GemmKPack; - int GemmMFactor; - int GemmNFactor; - int GemmKTotalFactor; - bool GemmAThreadCopyMoreGemmK; - bool GemmBThreadCopyMoreGemmK; - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm( - int, int, int, int, int, int, int, int, int, bool, bool); - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm(); - PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm(bool) - : PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm() - { - } - - template - static void Visit(Self&& self, F f) - { - f(self.GemmMPerBlock, "GemmMPerBlock"); - f(self.GemmNPerBlock, "GemmNPerBlock"); - f(self.GemmKPerBlock, "GemmKPerBlock"); - f(self.GemmMPerWave, "GemmMPerWave"); - f(self.GemmNPerWave, "GemmNPerWave"); - f(self.GemmKPack, "GemmKPack"); - f(self.GemmMFactor, "GemmMFactor"); - f(self.GemmNFactor, "GemmNFactor"); - f(self.GemmKTotalFactor, "GemmKTotalFactor"); - f(self.GemmAThreadCopyMoreGemmK, "GemmAThreadCopyMoreGemmK"); - f(self.GemmBThreadCopyMoreGemmK, "GemmBThreadCopyMoreGemmK"); - } - - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm& other) const; - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - MIOPEN_INTERNALS_EXPORT bool IsValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool IsReallyValid(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - IsFastToBeUsedForTuning(const ExecutionContext&, const miopen::conv::ProblemDescription&) const; - - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmSizeAndGemmKBlock(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple CalculateBlockSize() const; - std::tuple CalculateGridSize(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmABlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateGemmBBlockCopyPerformanceParameters(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT std::tuple - CalculateLdsNumberOfByte(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmWrwV4R4Xdlops_Padded_Gemm final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceImplicitGemmWrwV4R4Xdlops_Padded_Gemm - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; -}; - -struct PerformanceConvCkIgemmFwdV6r1DlopsNchw - : PerfConfigBase -{ - int ck_tunable_list_id; - - PerformanceConvCkIgemmFwdV6r1DlopsNchw(int a) : ck_tunable_list_id(a) {} - - PerformanceConvCkIgemmFwdV6r1DlopsNchw() : PerformanceConvCkIgemmFwdV6r1DlopsNchw(-1) {} - - PerformanceConvCkIgemmFwdV6r1DlopsNchw(bool) : PerformanceConvCkIgemmFwdV6r1DlopsNchw(0) {} - - template - static void Visit(Self&& self, F f) - { - f(self.ck_tunable_list_id, "ck_tunable_list_id"); - } - - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - bool operator==(const PerformanceConvCkIgemmFwdV6r1DlopsNchw& config) const - { - return ck_tunable_list_id == config.ck_tunable_list_id; - } -}; - -struct ConvCkIgemmFwdV6r1DlopsNchw final : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - bool IsDynamic() const override { return false; } - MIOPEN_INTERNALS_EXPORT PerformanceConvCkIgemmFwdV6r1DlopsNchw GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvCkIgemmFwdV6r1DlopsNchw&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConvCkIgemmFwdV6r1DlopsNchw - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConvCkIgemmFwdV6r1DlopsNchw&) const override; -}; - -struct ConvDirectNaiveConvFwd final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - /// Use very small fixed value enough to backup GEMM for cases when - /// GEMM is disabled. - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.01f; - } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvDirectNaiveConvBwd final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - /// Use very small fixed value enough to backup GEMM for cases when - /// GEMM is disabled. - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.01f; - } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct ConvDirectNaiveConvWrw final : ConvSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - /// Use very small fixed value enough to backup GEMM for cases when - /// GEMM is disabled. - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.01f; - } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct GemmFwdBase : ConvSolver -{ - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT float GetWti(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - -private: - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - friend struct GemmFwd1x1_0_2; - friend struct GemmFwd1x1_0_1_int8; - friend struct GemmFwd1x1_0_1; - friend struct GemmFwdRest; -}; - -struct GemmFwd1x1_0_2 final : GemmFwdBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - friend struct GemmFwdRest; -}; - -struct GemmFwd1x1_0_1_int8 final : GemmFwdBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - friend struct GemmFwdRest; -}; - -struct GemmFwd1x1_0_1 final : GemmFwdBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - friend struct GemmFwdRest; -}; - -struct GemmFwdRest final : GemmFwdBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct GemmBwdBase : ConvSolver -{ - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT float GetWti(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - -private: - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - friend struct GemmBwd1x1_stride2; - friend struct GemmBwd1x1_stride1; - friend struct GemmBwdRest; -}; - -struct GemmBwd1x1_stride2 final : GemmBwdBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - friend struct GemmBwdRest; -}; - -struct GemmBwd1x1_stride1 final : GemmBwdBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, - const miopen::conv::ProblemDescription& problem) const override; - - MIOPEN_INTERNALS_EXPORT ConvSolution GetSolution( - const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const override; - - friend struct GemmBwdRest; -}; - -struct GemmBwdRest final : GemmBwdBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct GemmWrwBase : ConvSolver -{ - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT float GetWti(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - -private: - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - friend struct GemmWrw1x1_stride1; - friend struct GemmWrwUniversal; -}; - -struct GemmWrw1x1_stride1 final : GemmWrwBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - friend struct GemmWrwUniversal; -}; - -struct GemmWrwUniversal final : GemmWrwBase -{ - const std::string& SolverDbId() const override { return GetSolverDbId(); } - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - bool MayNeedWorkspace() const override { return true; } - - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; -}; - -struct PerformanceConfigAsmImplicitGemmGTC : PerfConfigBase -{ - std::string direction; - std::string tensor_layout; - std::string precision; - int nxb; - int nxe; - - int gemm_m_per_block; - int gemm_n_per_block; - int gemm_k_per_block; - - int wave_tile_m; - int wave_tile_n; - int wave_tile_k; - int wave_step_m; - int wave_step_n; - int wave_repeat_m; - int wave_repeat_n; - - int multihead; - int vector_store; - int gemm_k_global_split; - int merge_e; - int tensor_a_pass_through; - - std::vector tensor_a_thread_lengths; - std::vector tensor_a_cluster_lengths; - std::vector tensor_b_thread_lengths; - std::vector tensor_b_cluster_lengths; - - bool use_spare_set; - int index; - - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTC(std::string dir, - std::string layout, - std::string prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false); - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTC(std::string dir, - std::string layout, - miopenDataType_t prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false); - PerformanceConfigAsmImplicitGemmGTC() - : PerformanceConfigAsmImplicitGemmGTC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - false) - { - } - PerformanceConfigAsmImplicitGemmGTC(bool spare) - : PerformanceConfigAsmImplicitGemmGTC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - spare) - { - } - - template - static void Visit(Self&& self, F f) - { - f(self.direction, "dir"); - f(self.tensor_layout, "lyt"); - f(self.precision, "pre"); - f(self.nxb, "nxb"); - f(self.nxe, "nxe"); - f(self.gemm_m_per_block, "mpb"); - f(self.gemm_n_per_block, "npb"); - f(self.gemm_k_per_block, "kpb"); - - f(self.wave_tile_m, "wtm"); - f(self.wave_tile_n, "wtn"); - f(self.wave_tile_k, "wtk"); - f(self.wave_step_m, "wsm"); - f(self.wave_step_n, "wsn"); - f(self.wave_repeat_m, "wrm"); - f(self.wave_repeat_n, "wrn"); - - f(self.multihead, "mh"); - f(self.vector_store, "vs"); - f(self.gemm_k_global_split, "gks"); - f(self.merge_e, "me"); - f(self.tensor_a_pass_through, "pta"); - - f(self.tensor_a_thread_lengths[0], "ta0"); - f(self.tensor_a_thread_lengths[1], "ta1"); - f(self.tensor_a_thread_lengths[2], "ta2"); - f(self.tensor_a_thread_lengths[3], "ta3"); - - f(self.tensor_a_cluster_lengths[0], "ca0"); - f(self.tensor_a_cluster_lengths[1], "ca1"); - f(self.tensor_a_cluster_lengths[2], "ca2"); - f(self.tensor_a_cluster_lengths[3], "ca3"); - - f(self.tensor_b_thread_lengths[0], "tb0"); - f(self.tensor_b_thread_lengths[1], "tb1"); - f(self.tensor_b_thread_lengths[2], "tb2"); - f(self.tensor_b_thread_lengths[3], "tb3"); - - f(self.tensor_b_cluster_lengths[0], "cb0"); - f(self.tensor_b_cluster_lengths[1], "cb1"); - f(self.tensor_b_cluster_lengths[2], "cb2"); - f(self.tensor_b_cluster_lengths[3], "cb3"); - f(self.index, "index"); - } - - // Chilrden must provide support for ComputedContainer. - void HeuristicInit(const ExecutionContext&) = delete; - bool SetNextValue(const miopen::conv::ProblemDescription&) = delete; - bool IsValidValue() const = delete; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const = delete; - - MIOPEN_INTERNALS_EXPORT bool IsDefaultConstructed() const; - MIOPEN_INTERNALS_EXPORT bool operator==(const PerformanceConfigAsmImplicitGemmGTC& other) const; - MIOPEN_INTERNALS_EXPORT void CopyParameters(const PerformanceConfigAsmImplicitGemmGTC& other); - MIOPEN_INTERNALS_EXPORT std::string ToString() const override; - MIOPEN_INTERNALS_EXPORT std::string ToKernelName(const ExecutionContext&) const; - MIOPEN_INTERNALS_EXPORT int BlockSize() const; -}; - -struct PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC : PerformanceConfigAsmImplicitGemmGTC -{ - PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC(std::string dir, - std::string layout, - std::string prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTC(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - wtm, - wtn, - wtk, - wsm, - wsn, - wrm, - wrn, - mh, - vs, - gks, - me, - pta, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC(std::string dir, - std::string layout, - miopenDataType_t prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTC(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - wtm, - wtn, - wtk, - wsm, - wsn, - wrm, - wrn, - mh, - vs, - gks, - me, - pta, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC() - : PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - false) - { - } - PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC(bool spare) - : PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - spare) - { - } - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription& config); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvAsmImplicitGemmGTCDynamicFwdXdlopsNHWC final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCFwdXdlopsNHWC&) const override; -}; - -struct PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC : PerformanceConfigAsmImplicitGemmGTC -{ - PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC(std::string dir, - std::string layout, - std::string prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTC(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - wtm, - wtn, - wtk, - wsm, - wsn, - wrm, - wrn, - mh, - vs, - gks, - me, - pta, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC(std::string dir, - std::string layout, - miopenDataType_t prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTC(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - wtm, - wtn, - wtk, - wsm, - wsn, - wrm, - wrn, - mh, - vs, - gks, - me, - pta, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC() - : PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - false) - { - } - PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC(bool spare) - : PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - spare) - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvAsmImplicitGemmGTCDynamicBwdXdlopsNHWC final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCBwdXdlopsNHWC&) const override; -}; - -struct PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC : PerformanceConfigAsmImplicitGemmGTC -{ - PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC(std::string dir, - std::string layout, - std::string prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTC(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - wtm, - wtn, - wtk, - wsm, - wsn, - wrm, - wrn, - mh, - vs, - gks, - me, - pta, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC(std::string dir, - std::string layout, - miopenDataType_t prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int wtm, - int wtn, - int wtk, - int wsm, - int wsn, - int wrm, - int wrn, - int mh, - int vs, - int gks, - int me, - int pta, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTC(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - wtm, - wtn, - wtk, - wsm, - wsn, - wrm, - wrn, - mh, - vs, - gks, - me, - pta, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC() - : PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - false) - { - } - PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC(bool spare) - : PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC("fwd", - "nchw", - "fp32", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - spare) - { - } - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT size_t ComputeKernelOccupancy() const; - -private: - void SetParamsForKSplit(const miopen::conv::ProblemDescription& problem, - const size_t& occupancy); -}; - -struct ConvAsmImplicitGemmGTCDynamicWrwXdlopsNHWC final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCWrwXdlopsNHWC&) const override; -}; - -struct PerformanceConfigAsmImplicitGemmGTCvector - : PerfConfigBase -{ - std::string direction; - std::string tensor_layout; - std::string precision; - int nxb; - int nxe; - - int gemm_m_per_block; - int gemm_n_per_block; - int gemm_k_per_block; - - int lanegroup_tile_m; - int lanegroup_tile_n; - int lanegroup_wave_m; - int lanegroup_wave_n; - int lanegroup_repeat_m; - int lanegroup_repeat_n; - - int vector_c; - - std::vector tensor_a_thread_lengths; - std::vector tensor_a_cluster_lengths; - std::vector tensor_b_thread_lengths; - std::vector tensor_b_cluster_lengths; - - bool use_spare_set; - int index; - - MIOPEN_INTERNALS_EXPORT - PerformanceConfigAsmImplicitGemmGTCvector(std::string dir, - std::string layout, - std::string prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int lgtm, - int lgtn, - int lgpwm, - int lgpwn, - int lgrm, - int lgrn, - int vec_c, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false); - - MIOPEN_INTERNALS_EXPORT - PerformanceConfigAsmImplicitGemmGTCvector(std::string dir, - std::string layout, - miopenDataType_t prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int lgtm, - int lgtn, - int lgpwm, - int lgpwn, - int lgrm, - int lgrn, - int vec_c, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false); - - PerformanceConfigAsmImplicitGemmGTCvector() - : PerformanceConfigAsmImplicitGemmGTCvector("fwd", - "nchwc_kcyxc", - "Half", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - false) - { - } - PerformanceConfigAsmImplicitGemmGTCvector(bool spare) - : PerformanceConfigAsmImplicitGemmGTCvector("fwd", - "nchwc_kcyxc", - "Half", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - spare) - { - } - - template - static void Visit(Self&& self, F f) - { - f(self.direction, "dir"); - f(self.tensor_layout, "lyt"); - f(self.precision, "pre"); - f(self.nxb, "nxb"); - f(self.nxe, "nxe"); - f(self.gemm_m_per_block, "mpb"); - f(self.gemm_n_per_block, "npb"); - f(self.gemm_k_per_block, "kpb"); - - f(self.lanegroup_tile_m, "lgtm"); - f(self.lanegroup_tile_n, "lgtn"); - f(self.lanegroup_wave_m, "lgpwm"); - f(self.lanegroup_wave_n, "lgpwn"); - f(self.lanegroup_repeat_m, "lgrm"); - f(self.lanegroup_repeat_n, "lgrn"); - - f(self.vector_c, "vec_c"); - - f(self.tensor_a_thread_lengths[0], "ta0"); - f(self.tensor_a_thread_lengths[1], "ta1"); - f(self.tensor_a_thread_lengths[2], "ta2"); - f(self.tensor_a_thread_lengths[3], "ta3"); - - f(self.tensor_a_cluster_lengths[0], "ca0"); - f(self.tensor_a_cluster_lengths[1], "ca1"); - f(self.tensor_a_cluster_lengths[2], "ca2"); - f(self.tensor_a_cluster_lengths[3], "ca3"); - - f(self.tensor_b_thread_lengths[0], "tb0"); - f(self.tensor_b_thread_lengths[1], "tb1"); - f(self.tensor_b_thread_lengths[2], "tb2"); - f(self.tensor_b_thread_lengths[3], "tb3"); - - f(self.tensor_b_cluster_lengths[0], "cb0"); - f(self.tensor_b_cluster_lengths[1], "cb1"); - f(self.tensor_b_cluster_lengths[2], "cb2"); - f(self.tensor_b_cluster_lengths[3], "cb3"); - f(self.index, "index"); - } - - // Chilrden must provide support for ComputedContainer. - void HeuristicInit(const ExecutionContext&) = delete; - bool SetNextValue(const miopen::conv::ProblemDescription&) = delete; - bool IsValidValue() const = delete; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription&) const = delete; - - MIOPEN_INTERNALS_EXPORT bool IsDefaultConstructed() const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigAsmImplicitGemmGTCvector& other) const; - MIOPEN_INTERNALS_EXPORT void - CopyParameters(const PerformanceConfigAsmImplicitGemmGTCvector& other); - MIOPEN_INTERNALS_EXPORT std::string ToString() const override; - MIOPEN_INTERNALS_EXPORT std::string ToKernelName(const ExecutionContext&) const; - MIOPEN_INTERNALS_EXPORT int BlockSize() const; -}; -struct PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC : PerformanceConfigAsmImplicitGemmGTCvector -{ - - PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC(std::string dir, - std::string layout, - std::string prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int lgtm, - int lgtn, - int lgpwm, - int lgpwn, - int lgrm, - int lgrn, - int vec_c, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTCvector(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - lgtm, - lgtn, - lgpwm, - lgpwn, - lgrm, - lgrn, - vec_c, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - - PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC(std::string dir, - std::string layout, - miopenDataType_t prec, - int b, - int e, - int mpb, - int npb, - int kpb, - int lgtm, - int lgtn, - int lgpwm, - int lgpwn, - int lgrm, - int lgrn, - int vec_c, - std::initializer_list ta_t, - std::initializer_list ta_c, - std::initializer_list tb_t, - std::initializer_list tb_c, - bool spare = false) - : PerformanceConfigAsmImplicitGemmGTCvector(dir, - layout, - prec, - b, - e, - mpb, - npb, - kpb, - lgtm, - lgtn, - lgpwm, - lgpwn, - lgrm, - lgrn, - vec_c, - ta_t, - ta_c, - tb_t, - tb_c, - spare) - { - } - - PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC() - : PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC("fwd", - "nchwc_kcyxc", - "Half", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - false) - { - } - PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC(bool spare) - : PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC("fwd", - "nchwc_kcyxc", - "Half", - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - {1, 1, 1, 1}, - spare) - { - } - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvAsmImplicitGemmGTCDynamicFwdDlopsNCHWC final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigAsmImplicitGemmGTCFwdDlopsNCHWC&) const override; -}; - -struct PerformanceConfigHipImplicitGemmFwdXdlops - : PerfConfigBaseCK -{ - int index = 0; - std::string kernel_id = ""; - std::vector valid_kernels; - - PerformanceConfigHipImplicitGemmFwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - - PerformanceConfigHipImplicitGemmFwdXdlops() = default; - - explicit PerformanceConfigHipImplicitGemmFwdXdlops(bool) - : PerformanceConfigHipImplicitGemmFwdXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmFwdXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmFwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmFwdXdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmFwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmFwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmFwdXdlops&) const override; - /// \anchor igemm_get_wti_magic_number - // Magic Number Alert: - // Naive convolutions have GetWti() that return very small value (0.01f). - // This allows MIOpen to use Naive Solvers if no other applicable Solvers - // have known WTIs. Right now this means that in case of find-db miss, - // the library will try to use Winograd or GEMM (whatever is faster according - // to their GetWti's), but if both are not applicable, the library will - // use Naive Solver - // Since we would like to us CK before naive, and use it instead (because - // we do expect that CK is faster than Naive), therefore we use a - // value bigger than 0.01f, e.g. 0.02f. - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemmBwdXdlops - : PerfConfigBaseCK -{ - int index = 0; - std::string kernel_id = ""; - std::vector valid_kernels; - - PerformanceConfigHipImplicitGemmBwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - - PerformanceConfigHipImplicitGemmBwdXdlops() = default; - - explicit PerformanceConfigHipImplicitGemmBwdXdlops(bool) - : PerformanceConfigHipImplicitGemmBwdXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmBwdXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmBwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmBwdXdlops GetDefaultPerformanceConfig( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmBwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmBwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmBwdXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemmGroupFwdXdlops - : PerfConfigBaseCK -{ - int index = 0; - std::string kernel_id = ""; - std::vector valid_kernels; - - PerformanceConfigHipImplicitGemmGroupFwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - - PerformanceConfigHipImplicitGemmGroupFwdXdlops() = default; - - explicit PerformanceConfigHipImplicitGemmGroupFwdXdlops(bool) - : PerformanceConfigHipImplicitGemmGroupFwdXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmGroupFwdXdlops& other) const; - MIOPEN_INTERNALS_EXPORT bool - IsModelApplicable(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const; - -private: -#if MIOPEN_ENABLE_AI_KERNEL_TUNING - std::vector heuristic_indexes; - std::unordered_map> heuristic_kernels; - template - bool RunParameterPredictionModel(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem); - void InitHeuristicKernelIDs(const std::string& type); - bool ModelApplyToken(int idx, std::string value, const std::string& arch); -#endif - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmGroupFwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupFwdXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmGroupFwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupFwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmGroupFwdXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemm3DGroupFwdXdlops - : PerfConfigBaseCK -{ - int index = 0; - std::string kernel_id = ""; - std::vector valid_kernels; - - PerformanceConfigHipImplicitGemm3DGroupFwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - - PerformanceConfigHipImplicitGemm3DGroupFwdXdlops() = default; - - explicit PerformanceConfigHipImplicitGemm3DGroupFwdXdlops(bool) - : PerformanceConfigHipImplicitGemm3DGroupFwdXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemm3DGroupFwdXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemm3DGroupFwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupFwdXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemm3DGroupFwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupFwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemm3DGroupFwdXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemm3DGroupWrwXdlops - : PerfConfigBaseCK -{ - int index; - std::string kernel_id; - std::vector valid_kernels; - PerformanceConfigHipImplicitGemm3DGroupWrwXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - PerformanceConfigHipImplicitGemm3DGroupWrwXdlops() - : PerformanceConfigHipImplicitGemm3DGroupWrwXdlops(0, "") - { - } - PerformanceConfigHipImplicitGemm3DGroupWrwXdlops(bool) - : PerformanceConfigHipImplicitGemm3DGroupWrwXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemm3DGroupWrwXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemm3DGroupWrwXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupWrwXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemm3DGroupWrwXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupWrwXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemm3DGroupWrwXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemm3DGroupBwdXdlops - : PerfConfigBaseCK -{ - int index; - std::string kernel_id; - std::vector valid_kernels; - PerformanceConfigHipImplicitGemm3DGroupBwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - PerformanceConfigHipImplicitGemm3DGroupBwdXdlops() - : PerformanceConfigHipImplicitGemm3DGroupBwdXdlops(0, "") - { - } - PerformanceConfigHipImplicitGemm3DGroupBwdXdlops(bool) - : PerformanceConfigHipImplicitGemm3DGroupBwdXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemm3DGroupBwdXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemm3DGroupBwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupBwdXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemm3DGroupBwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemm3DGroupBwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemm3DGroupBwdXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemmGroupBwdXdlops - : PerfConfigBaseCK -{ - int index; - std::string kernel_id; - std::vector valid_kernels; - PerformanceConfigHipImplicitGemmGroupBwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - PerformanceConfigHipImplicitGemmGroupBwdXdlops() - : PerformanceConfigHipImplicitGemmGroupBwdXdlops(0, "") - { - } - PerformanceConfigHipImplicitGemmGroupBwdXdlops(bool) - : PerformanceConfigHipImplicitGemmGroupBwdXdlops(0, "") - { - } - - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmGroupBwdXdlops& other) const; - MIOPEN_INTERNALS_EXPORT bool - IsModelApplicable(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const; - -private: -#if MIOPEN_ENABLE_AI_KERNEL_TUNING - std::vector heuristic_indexes; - std::unordered_map> heuristic_kernels; - template - bool RunParameterPredictionModel(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem); - void InitHeuristicKernelIDs(); - bool ModelApplyToken(int idx, std::string value); -#endif - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmGroupBwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupBwdXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmGroupBwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupBwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmGroupBwdXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemmGroupWrwXdlops - : PerfConfigBaseCK -{ - int index; - int split_k; - std::string kernel_id; - std::vector valid_kernels; - PerformanceConfigHipImplicitGemmGroupWrwXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - PerformanceConfigHipImplicitGemmGroupWrwXdlops() - : PerformanceConfigHipImplicitGemmGroupWrwXdlops(0, "") - { - } - PerformanceConfigHipImplicitGemmGroupWrwXdlops(bool) - : PerformanceConfigHipImplicitGemmGroupWrwXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const ExecutionContext&, - const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmGroupWrwXdlops& other) const; - MIOPEN_INTERNALS_EXPORT bool - IsModelApplicable(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem) const; - -private: -#if MIOPEN_ENABLE_AI_KERNEL_TUNING - std::vector heuristic_indexes; - std::unordered_map> heuristic_kernels; - template - bool RunParameterPredictionModel(const ExecutionContext& ctx, - const miopen::conv::ProblemDescription& problem); - void InitHeuristicKernelIDs(const std::string& type); - bool ModelApplyToken(int idx, - std::string value, - const std::string& arch, - const miopen::conv::ProblemDescription& problem); -#endif - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmGroupWrwXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupWrwXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool - IsValidPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmGroupWrwXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmGroupWrwXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmGroupWrwXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - - MIOPEN_INTERNALS_EXPORT size_t GetWorkspaceSize( - const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool MayNeedWorkspace() const override { return true; } - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops - : PerfConfigBaseCK -{ - int index = 0; - std::string kernel_id = ""; - std::vector valid_kernels; - - PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - - PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops() = default; - - explicit PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops(bool) - : PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmF16F8F16FwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmF16F8F16FwdXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops - : PerfConfigBaseCK -{ - int index; - std::string kernel_id; - std::vector valid_kernels; - PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops() - : PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops(0, "") - { - } - PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops(bool) - : PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmF16F8F16BwdXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmF16F8F16BwdXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -struct PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops - : PerfConfigBaseCK -{ - int index; - std::string kernel_id; - std::vector valid_kernels; - PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops(int idx, std::string kernl_id) - : index(idx), kernel_id(kernl_id) - { - } - PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops() - : PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops(0, "") - { - } - PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops(bool) - : PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops(0, "") - { - } - MIOPEN_INTERNALS_EXPORT void HeuristicInit(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool SetNextValue(const miopen::conv::ProblemDescription&); - MIOPEN_INTERNALS_EXPORT bool IsValidValue() const; - bool IsValid(const ExecutionContext&, const miopen::conv::ProblemDescription& problem) const - { - return IsValid(problem); - } - MIOPEN_INTERNALS_EXPORT bool IsValid(const miopen::conv::ProblemDescription&) const; - MIOPEN_INTERNALS_EXPORT bool - operator==(const PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops& other) const; - -private: - template - void Init(const miopen::conv::ProblemDescription&); - template - bool CheckIsSupportCKArgs(const miopen::conv::ProblemDescription&) const; -}; - -struct ConvHipImplicitGemmF16F8F16WrwXdlops final - : ConvTunableSolver -{ - const std::string& SolverDbId() const override - { - return GetSolverDbId(); - } - - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops - GetDefaultPerformanceConfig(const ExecutionContext&, - const miopen::conv::ProblemDescription&) const override; - MIOPEN_INTERNALS_EXPORT bool IsValidPerformanceConfig( - const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops&) const override; - MIOPEN_INTERNALS_EXPORT PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops - Search(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const AnyInvokeParams& invoke_ctx) const override; - MIOPEN_INTERNALS_EXPORT bool - IsApplicable(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override; - bool IsDynamic() const override { return true; } - MIOPEN_INTERNALS_EXPORT ConvSolution - GetSolution(const ExecutionContext&, - const miopen::conv::ProblemDescription&, - const PerformanceConfigHipImplicitGemmF16F8F16WrwXdlops&) const override; - /// \ref igemm_get_wti_magic_number - float GetWti(const ExecutionContext&, const miopen::conv::ProblemDescription&) const override - { - return 0.02f; - }; - -private: - template - bool CheckCKApplicability(const miopen::conv::ProblemDescription&) const; -}; - -} // namespace conv - // Use struct as a syntactic sugar to make the intent as clear as possible. struct ThisSolverIsDeprecatedStatic { diff --git a/src/include/miopen/solver/implicitgemm_ck_util.hpp b/src/include/miopen/solver/implicitgemm_ck_util.hpp index ff25d5f622..67a27bfbce 100644 --- a/src/include/miopen/solver/implicitgemm_ck_util.hpp +++ b/src/include/miopen/solver/implicitgemm_ck_util.hpp @@ -29,12 +29,15 @@ #include #include #include +#include #include #include #if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL #include #include +#include +#include #endif // MIOPEN_USE_COMPOSABLEKERNEL namespace miopen { @@ -61,6 +64,72 @@ using DeviceOpGWrw = ck::tensor_operation::device::DeviceGroupedConvBwdWeight< template using DeviceOpGWrwPtrs = ck::tensor_operation::device::instance::DeviceOperationInstanceFactory>; + +using InLayout = ck::tensor_layout::convolution::NDHWGC; +using WeiLayout = ck::tensor_layout::convolution::GKZYXC; +using OutLayout = ck::tensor_layout::convolution::NDHWGK; +using PassThrough = ck::tensor_operation::element_wise::PassThrough; +using Bilinear = ck::tensor_operation::element_wise::Bilinear; +using Scale = ck::tensor_operation::element_wise::Scale; + +template +using DeviceOpGBwdWeightDefault = + ck::tensor_operation::device::DeviceGroupedConvBwdWeight<3, + InLayout, + WeiLayout, + OutLayout, + DataType, + DataType, + DataType, + PassThrough, + PassThrough, + PassThrough>; + +template +using DeviceOpGBwdWeightBilinear = + ck::tensor_operation::device::DeviceGroupedConvBwdWeightMultipleD<3, + InLayout, + WeiLayout, + OutLayout, + ck::Tuple, + DataType, + DataType, + DataType, + ck::Tuple, + PassThrough, + Bilinear, + PassThrough>; + +template +using DeviceOpGBwdWeightScale = + ck::tensor_operation::device::DeviceGroupedConvBwdWeightMultipleD<3, + InLayout, + WeiLayout, + OutLayout, + ck::Tuple<>, + DataType, + DataType, + DataType, + ck::Tuple<>, + PassThrough, + Scale, + PassThrough>; + +template +using DeviceOpGBwdWeightDefaultPtrs = + ck::tensor_operation::device::instance::DeviceOperationInstanceFactory< + DeviceOpGBwdWeightDefault>; + +template +using DeviceOpGBwdWeightBilinearPtrs = + ck::tensor_operation::device::instance::DeviceOperationInstanceFactory< + DeviceOpGBwdWeightBilinear>; + +template +using DeviceOpGBwdWeightScalePtrs = + ck::tensor_operation::device::instance::DeviceOperationInstanceFactory< + DeviceOpGBwdWeightScale>; + } // namespace conv #endif @@ -124,22 +193,60 @@ std::vector FillValidKernelsIDs(const ProblemDescriptionType& probl return valid_kernels; } +#if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL +template +inline constexpr bool IsSplitKNeeded() +{ + return std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v> || + std::is_same_v>; +} +#endif + template + typename ProblemDescriptionType = miopen::conv::ProblemDescription, + bool CheckSplitK = false> bool IsCKArgsSupported(const ProblemDescriptionType& problem, const std::string& kernel_id) { #if MIOPEN_BACKEND_HIP && MIOPEN_USE_COMPOSABLEKERNEL if(!kernel_id.empty()) { auto conv_ptrs = DeviceOpType::GetInstances(); - if constexpr(std::is_same_v> || - std::is_same_v> || - std::is_same_v> || - std::is_same_v>) + if constexpr(IsSplitKNeeded() || CheckSplitK) { - auto pos = kernel_id.find_last_of('+'); - int split_k = std::stoi(kernel_id.substr(pos + 1)); + auto pos = kernel_id.find_last_of('+'); + if(pos == std::string::npos) + { + MIOPEN_LOG_WE("Unable to parse split_k from kernel_id for wrw: " << kernel_id); + return false; + } + + int split_k = 1; + try + { + split_k = std::stoi(kernel_id.substr(pos + 1)); + } + catch(std::exception& e) + { + MIOPEN_LOG_WE("Unable to parse split_k from kernel_id for wrw: " + << kernel_id << " : " << e.what()); + return false; + } + auto ptr_iter = FindConvPtrByID(conv_ptrs, kernel_id.substr(0, pos)); return (ptr_iter != conv_ptrs.end()) && CKArgsType{problem}.IsSupportedBySplitK(*ptr_iter, split_k); @@ -376,9 +483,10 @@ class TransposeInstance Run(handle, kernels, out_ptr, buf_handle.get()); } - void ZeroOutBuffer() + void ZeroOutBuffer(const Handle& handle) { - [[maybe_unused]] auto status = hipMemset(buf_handle.get(), 0, tensor_sz); + [[maybe_unused]] auto status = + hipMemsetAsync(buf_handle.get(), 0, tensor_sz, handle.GetStream()); assert(status == hipSuccess); } @@ -600,7 +708,8 @@ inline bool CKWrwRequireWorkspace( size_t K_per_group = K / G; return (alpha_beta_case == BILINEAR || alpha_beta_case == SCALE) || - (data_type == miopenHalf && (is_odd(C_per_group) || is_odd(K_per_group))); + ((data_type == miopenHalf || data_type == miopenBFloat16) && + (is_odd(C_per_group) || is_odd(K_per_group))); } /// \todo move to a cpp file @@ -632,6 +741,28 @@ inline size_t GetWorkspaceSizeLayoutTransformConv(const miopen::conv::ProblemDes return wt.GetSize(); } +inline void +ZeroOutTensor(const Handle& handle, const TensorDescriptor& tensorDesc, Data_t tensorData) +{ +#if MIOPEN_BACKEND_HIP + // SetTensor is required for non-packed tensors, but is also slower. + // Use faster clear if possible. + if(tensorDesc.IsPacked()) + { + auto status = hipMemsetAsync(tensorData, 0, tensorDesc.GetNumBytes(), handle.GetStream()); + if(status != hipSuccess) + { + MIOPEN_THROW_HIP_STATUS(status, "hipMemsetAsync() failed"); + } + } + else +#endif + { + auto zero = 0.0f; + SetTensor(handle, tensorDesc, tensorData, &zero); + } +} + template tr_ptrs = { &input1_tr_inst, &input2_tr_inst, &output_tr_inst}; @@ -747,10 +876,7 @@ ConvSolution InitInvokerFactoryNCHW(const ExecutionContext& ctx, auto invoker_ptr = sh_conv_ptr->MakeInvokerPointer(); std::unique_ptr argument_ptr; - if constexpr(std::is_same_v> || - std::is_same_v> || - std::is_same_v> || - std::is_same_v>) + if constexpr(IsSplitKNeeded()) { if(split_k.has_value()) { @@ -840,10 +966,7 @@ ConvSolution InitInvokerFactoryNHWC(const ExecutionContext&, const Handle& handle, const AnyInvokeParams& primitive_parameters) { const auto& data_ctx = primitive_parameters.CastTo(); std::unique_ptr argument_ptr; - if constexpr(std::is_same_v> || - std::is_same_v> || - std::is_same_v> || - std::is_same_v>) + if constexpr(IsSplitKNeeded()) { argument_ptr = ck_args.MakeArgPtr(sh_conv_ptr, data_ctx.tensors, @@ -865,9 +988,7 @@ ConvSolution InitInvokerFactoryNHWC(const ExecutionContext&, if(alpha_beta_case == DEFAULT) { - auto zero = 0.0f; - const auto& tensors = data_ctx.tensors; - SetTensor(handle, tensors.dwDesc, tensors.dw, &zero); + ZeroOutTensor(handle, data_ctx.tensors.dwDesc, data_ctx.tensors.dw); } // use captured value, other wise getting warning // "lambda capture is not used" since this variable is only used in assert. @@ -900,6 +1021,14 @@ ConvSolution InitInvokerFactoryNHWC(const ExecutionContext&, data_ctx.beta.GetAsFloat()); auto invoker_ptr = sh_conv_ptr->MakeInvokerPointer(); HipEventProfiler pfr(handle); + + // Zero out the buffer for output data since it won't always write all output + // values. + if constexpr(std::is_same_v) + { + ZeroOutTensor(handle, data_ctx.tensors.outDesc, data_ctx.tensors.out); + } + invoker_ptr->Run(argument_ptr.get(), {handle.GetStream(), false}); }; }; diff --git a/src/include/miopen/solver_id.hpp b/src/include/miopen/solver_id.hpp index 6ddd83bcef..76a13b051c 100644 --- a/src/include/miopen/solver_id.hpp +++ b/src/include/miopen/solver_id.hpp @@ -32,7 +32,6 @@ #include #include -#include namespace miopen { @@ -43,6 +42,7 @@ struct ForceInit namespace solver { struct AnySolver; +struct SolverBase; enum class Primitive { @@ -58,7 +58,13 @@ enum class Primitive Cat, Mha, Softmax, - Adam + Adam, + Item, + RoPE, + ReLU, + Kthvalue, + SoftMarginLoss, + MultiMarginLoss }; struct MIOPEN_INTERNALS_EXPORT Id @@ -73,6 +79,7 @@ struct MIOPEN_INTERNALS_EXPORT Id std::string ToString() const; AnySolver GetSolver() const; + const SolverBase* GetSolverBase() const; std::string GetAlgo(conv::Direction dir) const; miopenConvAlgorithm_t GetAlgo() const; Primitive GetPrimitive() const; diff --git a/src/include/miopen/sqlite_db.hpp b/src/include/miopen/sqlite_db.hpp index 71e3a31736..5b4f23179a 100644 --- a/src/include/miopen/sqlite_db.hpp +++ b/src/include/miopen/sqlite_db.hpp @@ -42,8 +42,6 @@ #include #include #include -#include -#include #include "sqlite3.h" #include #include @@ -240,7 +238,7 @@ class SQLiteBase if(!fs::create_directories(directory)) MIOPEN_LOG_W("Unable to create a directory: " << directory); else - fs::permissions(directory, fs::perms::all); + fs::permissions(directory, FS_ENUM_PERMS_ALL); } } sql = SQLite{filename_, is_system}; diff --git a/src/include/miopen/tensor.hpp b/src/include/miopen/tensor.hpp index f4d2b2dca7..87724df732 100644 --- a/src/include/miopen/tensor.hpp +++ b/src/include/miopen/tensor.hpp @@ -40,33 +40,44 @@ #include #include #include +#include #include #include +#include namespace miopen { template -auto tie_impl(T&& x, detail::seq) -> decltype(std::tie(x[Ns]...)) +constexpr auto tie_array_impl(T&& x, std::index_sequence) + MIOPEN_RETURNS(std::array{x[Ns]...}); + +template +constexpr auto tien_array(T&& x) + MIOPEN_RETURNS(tie_array_impl(std::forward(x), std::make_index_sequence())); + +template +auto tie_impl(T&& x, std::index_sequence) -> decltype(std::tie(x[Ns]...)) { assert(x.size() >= sizeof...(Ns)); return std::tie(x[Ns]...); } template -auto tie_impl(T&& x, U y, detail::seq) -> decltype(std::make_tuple(x[Ns]...)) +auto tie_impl(T&& x, U y, std::index_sequence) -> decltype(std::make_tuple(x[Ns]...)) { return std::make_tuple((Ns < x.size() ? x[Ns] : y)...); } template -auto tien(T&& x) MIOPEN_RETURNS(tie_impl(std::forward(x), typename detail::gens::type{})); +constexpr auto tien(T&& x) + MIOPEN_RETURNS(tie_impl(std::forward(x), std::make_index_sequence())); template auto tien(T&& x, U y) - MIOPEN_RETURNS(tie_impl(std::forward(x), y, typename detail::gens::type{})); + MIOPEN_RETURNS(tie_impl(std::forward(x), y, std::make_index_sequence())); template -auto tie_pick_impl(T&& x, detail::seq) +auto tie_pick_impl(T&& x, std::index_sequence) { #ifndef NDEBUG each_args([&](auto i) { assert(i < x.size()); }, Ns...); @@ -78,11 +89,12 @@ template struct tie_pick { template - auto operator()(T&& x) MIOPEN_RETURNS(tie_pick_impl(std::forward(x), detail::seq{})) + auto operator()(T&& x) + MIOPEN_RETURNS(tie_pick_impl(std::forward(x), std::index_sequence{})) }; template -auto create_tuple_impl(F f, detail::seq) +auto create_tuple_impl(F f, std::index_sequence) { return std::make_tuple(std::forward(f(Ns))...); } @@ -90,7 +102,7 @@ auto create_tuple_impl(F f, detail::seq) template auto create_tuple(F f) { - return create_tuple_impl(f, typename detail::gens::type{}); + return create_tuple_impl(f, std::make_index_sequence()); } inline std::size_t GetTypeSize(miopenDataType_t d) @@ -162,6 +174,10 @@ struct MIOPEN_INTERNALS_EXPORT TensorDescriptor : miopenTensorDescriptor TensorDescriptor(miopenDataType_t t, const std::vector& lens_in, const std::vector& strides_in); + TensorDescriptor(miopenDataType_t t, + miopenTensorLayout_t layout_in, + const std::vector& lens_in, + const std::vector& strides_in); TensorDescriptor(miopenDataType_t t, const std::initializer_list& lens_in, const std::initializer_list& strides_in); @@ -198,15 +214,21 @@ struct MIOPEN_INTERNALS_EXPORT TensorDescriptor : miopenTensorDescriptor int size); bool IsVectorized() const; - + static bool LayoutStrIsVectorized(const std::string& layout); const std::vector& GetLengths() const; const std::vector& GetStrides() const; unsigned GetNumDims() const; miopenDataType_t GetType() const; + // clang-format off + [[deprecated("Use GetLayoutEnum() instead")]] miopenTensorLayout_t GetLayout_t() const; + // clang-format on + const std::optional& GetLayoutEnum() const; + static std::string LayoutEnumToStr(miopenTensorLayout_t layout); static std::string GetLayoutStr(miopenTensorLayout_t layout); - std::string GetLayout_str() const; + const std::string& GetLayout_str() const; + bool IsDefaultLayout() const; std::size_t GetVectorLength() const; std::optional GetCastType() const; @@ -240,48 +262,26 @@ struct MIOPEN_INTERNALS_EXPORT TensorDescriptor : miopenTensorDescriptor std::string ToString() const; - bool IsPossibleLayout(const std::string& labels, const std::string& layout) const; + // For vectorized layouts storage_layout must be without the ending 'c' + // \todo make private + bool IsPossibleLayout(const std::string& storage_layout, const std::string& layout) const; + // Layout could be NCHW, NHWC, NCDHW, NDHWC, NCHWc, ... + bool IsPossibleLayout4D5D(const std::string& layout) const; - static inline std::vector find_permutation(const std::vector& lens, - const std::vector& strides) - { - std::vector result(lens.size()); - std::iota(result.begin(), result.end(), 0); - std::stable_sort(result.begin(), result.end(), by(std::greater<>{}, [&](auto x) { - return std::make_tuple(strides[x], lens[x]); - })); - return result; - } + static std::vector find_permutation(const std::vector& lens, + const std::vector& strides); - std::string GetLayout(std::string labels) const + std::string GetLayout(std::string storage_layout) const; + + static bool IsDefaultLayout(miopenTensorLayout_t layout, unsigned spatial_dims = 2) { - if(*(labels.end() - 1) != 'c') - { - if(labels.size() != strides.size()) - { - MIOPEN_THROW( - "Invalid labels size. Layout labels size must be equavalent to stride size"); - } - - // Copy construct the result string from labels. This allocates the space at one go - // and is faster than calling push_back in transform. - auto result = labels; - auto p = find_permutation(lens, strides); - std::transform(p.begin(), p.end(), result.begin(), [&](auto i) { return labels[i]; }); - return result; - } - else + switch (spatial_dims) { - const std::string base_label = labels.substr(0, labels.size() - 1); - if(base_label.size() != strides.size()) - { - MIOPEN_THROW( - "Invalid labels size. Layout labels size must be equavalent to stride size"); - } - auto result = base_label; - auto p = find_permutation(lens, strides); - std::transform(p.begin(), p.end(), result.begin(), [&](auto i) { return labels[i]; }); - return result + 'c'; + case 2: + case 3: + return layout == GetDefaultLayout(); + default: + MIOPEN_THROW(miopenStatusBadParm, "Spatial dimension count must be 2 or 3."); } } @@ -292,29 +292,32 @@ struct MIOPEN_INTERNALS_EXPORT TensorDescriptor : miopenTensorDescriptor friend void from_json(const nlohmann::json& j, TensorDescriptor& descriptor); protected: - static miopenTensorLayout_t GetDefaultLayout() { return miopenTensorNCHW; }; + static miopenTensorLayout_t GetDefaultLayout(unsigned spatial_dims = 2) + { + switch (spatial_dims) + { + case 2: return miopenTensorNCHW; + case 3: return miopenTensorNCDHW; + default: + MIOPEN_THROW(miopenStatusBadParm, "Spatial dimension count must be 2 or 3."); + } + }; private: TensorDescriptor(miopenDataType_t t, - miopenTensorLayout_t layout_in, + const std::optional& layout_in, const std::vector& lens_in, const std::vector& strides_in, bool use_strides); TensorDescriptor(miopenDataType_t t, - miopenTensorLayout_t layout_in, + const std::optional& layout_in, std::vector&& lens_in, std::vector&& strides_in, bool use_strides); void CheckArgsAndInit(bool use_strides); - void SetStrideNd(const std::string& layout); - void LensReorder(const std::string& layout); - - void CalculateStrides(); - void CalculateVectorLength(); - std::vector lens; std::vector strides; @@ -323,16 +326,54 @@ struct MIOPEN_INTERNALS_EXPORT TensorDescriptor : miopenTensorDescriptor miopenDataType_t type = miopenFloat; std::optional cast_type; - miopenTensorLayout_t tensorLayout = GetDefaultLayout(); + std::optional tensorLayout; + + // For GetLayoutEnum() + mutable std::optional cached_layout_enum; + mutable bool cached_layout_enum_calculated = false; + + // For GetLayout_str() + mutable std::string cached_layout_str; + + // For GetLayout + mutable std::vector cached_permutation; + + // For AllLengthsFitIntoInt() + mutable std::optional cached_lengths_fit_into_int; + // For AllDimsFitIntoInt() + mutable std::optional cached_strides_fit_into_int; }; template constexpr auto GetNCDHW(unsigned spatial_dims, const std::vector& data) +{ + if(spatial_dims == 3) + { + if(data.size() == 5) // NCDHW + return miopen::tien<5>(data, 1); + else if(data.size() == 3) // DHW + return std::make_tuple(static_cast(1), static_cast(1), data[0], data[1], data[2]); + else + MIOPEN_THROW("Invalid data length; must be 5 or 3 with 3 spatial dimensions"); + } + else + { + if(data.size() == 4) // NCHW + return std::make_tuple(data[0], data[1], static_cast(1), data[2], data[3]); + else if(data.size() == 2) // HW + return std::make_tuple(static_cast(1), static_cast(1), static_cast(1), data[0], data[1]); + else + MIOPEN_THROW("Invalid data length; must be 4 or 2 with 2 spatial dimensions"); + } +} + +template +constexpr auto GetNDHWC(unsigned spatial_dims, const std::vector& data) { if(spatial_dims == 3) return miopen::tien<5>(data, 1); else - return std::make_tuple(data[0], data[1], static_cast(1), data[2], data[3]); + return std::make_tuple(data[0], static_cast(1), data[1], data[2], data[3]); } } // namespace miopen diff --git a/src/include/miopen/tensor_view_utils.hpp b/src/include/miopen/tensor_view_utils.hpp new file mode 100644 index 0000000000..a75d0a380f --- /dev/null +++ b/src/include/miopen/tensor_view_utils.hpp @@ -0,0 +1,107 @@ +/******************************************************************************* + * + * MIT License + * + * Copyright (c) 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + *******************************************************************************/ + +#ifndef MIOPEN_TENSOR_VIEW_UTIL_HPP_ +#define MIOPEN_TENSOR_VIEW_UTIL_HPP_ + +#include "../../kernels/tensor_view.hpp" +#include + +namespace miopen { + +template +inline tensor_view_t get_inner_expanded_tv(const TensorDescriptor Desc) +{ + auto dims = Desc.GetLengths(); + auto strides = Desc.GetStrides(); + + tensor_view_t tensor_view{}; + for(size_t i = 0; i < N; ++i) + { + if(dims.empty()) + { + tensor_view.stride[i] = 0; + tensor_view.size[i] = 0; + } + else if(i < dims.size()) + { + tensor_view.stride[i] = strides[i]; + tensor_view.size[i] = dims[i]; + } + else + { + tensor_view.stride[i] = (i == 0 ? 1 : strides[i - 1]); + tensor_view.size[i] = 1; + } + } + return tensor_view; +} + +template +inline void slice_tv(tensor_view_t& tensor_view, int32_t sliceCount, const int32_t* slices) +{ + for(int32_t i = 0; i < sliceCount; i++) + { + int32_t dim = slices[4 * i + 0]; + int32_t start = slices[4 * i + 1]; + int32_t end = slices[4 * i + 2]; + int32_t step = slices[4 * i + 3]; + + if(end > static_cast(tensor_view.size[dim])) + end = tensor_view.size[dim]; + + auto len = end - start; + + tensor_view.size[dim] = (len + step - 1) / step; + tensor_view.stride[dim] *= step; + } +} + +template +inline tensor_view_t get_tv_without_dim(const tensor_view_t& origin_tv, int selected_dim) +{ + tensor_view_t res{}; + for(int i = 0; i < N; ++i) + { + if(i == selected_dim) + continue; + if(i < selected_dim) + { + res.size[i] = origin_tv.size[i]; + res.stride[i] = origin_tv.stride[i]; + } + else + { + res.size[i - 1] = origin_tv.size[i]; + res.stride[i - 1] = origin_tv.stride[i]; + } + } + return res; +} + +} // namespace miopen + +#endif // MIOPEN_TENSOR_VIEW_UTIL_HPP_ diff --git a/src/include/miopen/type_name.hpp b/src/include/miopen/type_name.hpp index e1541f9677..984b228525 100644 --- a/src/include/miopen/type_name.hpp +++ b/src/include/miopen/type_name.hpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (c) 2017 Advanced Micro Devices, Inc. + * Copyright (c) 2017-2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,29 +28,102 @@ #define GUARD_TYPE_NAME_HPP #include +#include +#if defined(_MSC_VER) && !defined(__clang__) && !defined(__GNUC__) +#include +#endif namespace miopen { -template -const std::string& get_type_name() +template +constexpr std::string_view type_name() { - static const std::string ret = -#if defined(_MSC_VER) && !defined(__clang__) - typeid(MIOpen_Private_TypeName_).name().substr(7); -#else - [](std::string name) { - const char parameter_name[] = "MIOpen_Private_TypeName_ ="; - - auto begin = name.find(parameter_name) + sizeof(parameter_name); -#if(defined(__GNUC__) && !defined(__clang__) && __GNUC__ == 4 && __GNUC_MINOR__ < 7) - auto length = name.find_last_of(",") - begin; -#else - auto length = name.find_first_of("];", begin) - begin; +#if defined(__clang__) || defined(__GNUC__) + // clang or gcc + constexpr auto full_name = std::string_view{__PRETTY_FUNCTION__}; +#elif defined(_MSC_VER) + // msvc + constexpr auto full_name = std::string_view{__FUNCSIG__}; +#endif + + // The substring with the data type name is located within the original string, between the + // prefix and the suffix, with the prefix always not at the beginning of the string and the + // suffix always at the end of the string. +#if defined(__clang__) + // clang + constexpr auto prefix = std::string_view{"[T = "}; + constexpr auto suffix = std::string_view{"]"}; +#elif defined(__GNUC__) + // gcc + constexpr auto prefix = std::string_view{"[with T = "}; + constexpr auto suffix = std::string_view{"; std::string_view = std::basic_string_view]"}; +#elif defined(_MSC_VER) + // msvc + constexpr auto prefix = std::string_view{"type_name<"}; + constexpr auto suffix = std::string_view{">(void)"}; +#endif + + constexpr auto prefix_pos = full_name.find(prefix); + static_assert(prefix_pos != std::string_view::npos); + + constexpr auto suffix_pos = full_name.rfind(suffix); + static_assert(suffix_pos != std::string_view::npos); + static_assert(suffix_pos == full_name.size() - suffix.size()); + + constexpr auto pos = prefix_pos + prefix.size(); + static_assert(pos < suffix_pos); + constexpr auto count = suffix_pos - pos; + + constexpr auto name = full_name.substr(pos, count); + +#if defined(__clang__) || defined(__GNUC__) + // clang or gcc + return name; +#elif defined(_MSC_VER) + // msvc + if constexpr(std::is_compound_v) + { + // For compound data types, the string contains the keyword 'class/struct/union/enum' before + // the data type name, separated by a space. + constexpr auto sep = std::string_view{" "}; + constexpr auto sep_pos = name.find(sep); + static_assert(sep_pos != std::string_view::npos); + static_assert(sep_pos != 0); // must not be at the 0 position + + constexpr auto name_pos = sep_pos + sep.size(); + constexpr auto tname = name.substr(name_pos); + static_assert(tname.size() > 0); + + return tname; + } + else + { + return name; + } #endif - name = name.substr(begin, length); - return name; - }(__PRETTY_FUNCTION__); -#endif // _MSC_VER +} + +template +constexpr std::string_view type_name_bare() +{ + constexpr auto name = type_name(); + constexpr auto pos = name.rfind(':'); + if constexpr(pos == std::string_view::npos) + { + return name; + } + else + { + constexpr auto bare_name = name.substr(pos + 1); + static_assert(bare_name.size() > 0); + return bare_name; + } +} + +template +const std::string& get_type_name() +{ + static const auto ret = std::string(type_name()); return ret; } diff --git a/src/include/miopen/type_traits.hpp b/src/include/miopen/utility/base64.hpp similarity index 65% rename from src/include/miopen/type_traits.hpp rename to src/include/miopen/utility/base64.hpp index c343c2a8e6..5071d20809 100644 --- a/src/include/miopen/type_traits.hpp +++ b/src/include/miopen/utility/base64.hpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (c) 2022 Advanced Micro Devices, Inc. + * Copyright (c) 2024 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,32 +26,19 @@ #pragma once -#include +#include +#include +#include namespace miopen { -namespace detail { +std::string base64Encode(const uint8_t* data, std::size_t length); -template -using void_t = void; - -template class Op, class... Args> -struct MemberDetector -{ - using value_t = std::false_type; - using type = Default; -}; - -template class Op, class... Args> -struct MemberDetector>, Op, Args...> +inline std::string base64Encode(const std::vector& data) { - using value_t = std::true_type; - using type = Op; -}; - -} // namespace detail + return base64Encode(data.data(), data.size()); +} -template