From 7708f1dff754b3bb93b8f83b3f2e6f406fafd0f7 Mon Sep 17 00:00:00 2001 From: Jieyang Chen Date: Thu, 24 Sep 2026 16:57:10 -0700 Subject: [PATCH 01/11] mgard-x: make MPI optional for MDR; only pmdr-x executables need it MGARD_ENABLE_MDR used find_package(MPI REQUIRED), so a machine without MPI could not build MDR-X at all, even though only the parallel pmdr-x-qoi and pmdr-x-qoi-io executables use MPI. Look MPI up without REQUIRED and build those two executables only when it is found; the MDR-X library, mdr-x and mdr-x-qoi build either way. The configure summary reports the result as "pmdr-x (MPI): ON/OFF" under MDR. Co-Authored-By: Claude Opus 5.5 (1M context) --- CMakeLists.txt | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ad2e6a358..e708203cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -133,6 +133,7 @@ if (MGARD_ENABLE_SERIAL OR $) install(TARGETS mgard-cpu) + set(MGARD_X_BUILD_PMDR OFF) if(MGARD_ENABLE_MDR) add_executable(mdr-x ${CMAKE_CURRENT_SOURCE_DIR}/src/mgard-x/Executables/mdr-x.cpp) target_link_libraries(mdr-x mgard-library ${CMAKE_DL_LIBS}) @@ -144,17 +145,24 @@ if (MGARD_ENABLE_SERIAL OR target_include_directories(mdr-x-qoi PUBLIC $ $) install(TARGETS mdr-x-qoi) - find_package(MPI REQUIRED) - add_executable(pmdr-x-qoi ${CMAKE_CURRENT_SOURCE_DIR}/src/mgard-x/Executables/pmdr-x-qoi.cpp) - target_link_libraries(pmdr-x-qoi mgard-library MPI::MPI_CXX ${CMAKE_DL_LIBS}) - target_include_directories(pmdr-x-qoi PUBLIC $ - $) - install(TARGETS pmdr-x-qoi) - add_executable(pmdr-x-qoi-io ${CMAKE_CURRENT_SOURCE_DIR}/src/mgard-x/Executables/pmdr-x-qoi-io.cpp) - target_link_libraries(pmdr-x-qoi-io mgard-library MPI::MPI_CXX ${CMAKE_DL_LIBS}) - target_include_directories(pmdr-x-qoi-io PUBLIC $ - $) - install(TARGETS pmdr-x-qoi-io) + # MPI is only needed by the parallel pmdr-x executables; the rest of MDR-X + # (library, mdr-x, mdr-x-qoi) builds without it. + find_package(MPI COMPONENTS CXX) + if(MPI_CXX_FOUND) + set(MGARD_X_BUILD_PMDR ON) + add_executable(pmdr-x-qoi ${CMAKE_CURRENT_SOURCE_DIR}/src/mgard-x/Executables/pmdr-x-qoi.cpp) + target_link_libraries(pmdr-x-qoi mgard-library MPI::MPI_CXX ${CMAKE_DL_LIBS}) + target_include_directories(pmdr-x-qoi PUBLIC $ + $) + install(TARGETS pmdr-x-qoi) + add_executable(pmdr-x-qoi-io ${CMAKE_CURRENT_SOURCE_DIR}/src/mgard-x/Executables/pmdr-x-qoi-io.cpp) + target_link_libraries(pmdr-x-qoi-io mgard-library MPI::MPI_CXX ${CMAKE_DL_LIBS}) + target_include_directories(pmdr-x-qoi-io PUBLIC $ + $) + install(TARGETS pmdr-x-qoi-io) + else() + message(STATUS "MPI not found: skipping pmdr-x-qoi and pmdr-x-qoi-io.") + endif() endif() endif() @@ -486,6 +494,11 @@ message(STATUS "SYCL: ${MGARD_ENABLE_SYCL}") list(POP_BACK CMAKE_MESSAGE_INDENT) message(STATUS "auto tuning: ${MGARD_ENABLE_AUTO_TUNING}") message(STATUS "MDR: ${MGARD_ENABLE_MDR}") +if(MGARD_ENABLE_MDR) + list(APPEND CMAKE_MESSAGE_INDENT " ") + message(STATUS "pmdr-x (MPI): ${MGARD_X_BUILD_PMDR}") + list(POP_BACK CMAKE_MESSAGE_INDENT) +endif() message(STATUS "timing: ${MGARD_ENABLE_TIMING}") message(STATUS "ASan: ${MGARD_ENABLE_ASAN}") message(STATUS "UBSan: ${MGARD_ENABLE_UBSAN}") From 0819577126bbab82ad2420a473c30b2c404a165a Mon Sep 17 00:00:00 2001 From: Jieyang Chen Date: Thu, 1 Oct 2026 15:46:04 -0700 Subject: [PATCH 02/11] mgard-x: store Hierarchy level volumes per level instead of dense _level_volumes and _level_volumes_reciprocal were dense (l_target + 1) x D x max_width arrays, but level l only fills level_shape[l][d] entries per dimension. In 1D max_width is N and l_target is ~log2(N), so the two arrays alone took ~50x the input size (1600 MB each for a 2^24-element float array) and dominated the 1D compression footprint (peak 120x the input). Pack each (level, dim) segment back to back and add a (l_target + 1) x D _level_volume_offsets table (Hierarchy::level_volume_offsets()); the total is sum_l sum_d level_shape[l][d], ~2N in 1D. The levelwise quantizer kernel reads level_volumes[offset(level, d) + idx[d]], which is the same element as before: idx[d] < level_shape[level][d] for a node of that level. EstimateMemoryFootprint uses the packed size. 1D float, SERIAL, peak device memory: 2^24 elements 7680 -> 1792 MB (multi-dim) and 7680 -> 2121 MB (hybrid). Compressed output is byte-identical to before across 1D-4D, float/double, s = inf/0/1, rel/abs, multi-dim and hybrid. Co-Authored-By: Claude Opus 5.5 --- include/mgard-x/Hierarchy/Hierarchy.h | 14 +++-- include/mgard-x/Hierarchy/Hierarchy.hpp | 51 ++++++++++++------- .../Quantization/LinearQuantization.hpp | 42 +++++++++------ 3 files changed, 69 insertions(+), 38 deletions(-) diff --git a/include/mgard-x/Hierarchy/Hierarchy.h b/include/mgard-x/Hierarchy/Hierarchy.h index aa608cd53..910240a72 100644 --- a/include/mgard-x/Hierarchy/Hierarchy.h +++ b/include/mgard-x/Hierarchy/Hierarchy.h @@ -40,7 +40,8 @@ template struct Hierarchy { Array<1, DIM, DeviceType> &unprocessed(SIZE idx, DIM &processed_n); Array<2, SIZE, DeviceType> &level_ranges(); Array<2, int, DeviceType> &level_marks(); - Array<3, T, DeviceType> &level_volumes(bool reciprocal); + Array<1, T, DeviceType> &level_volumes(bool reciprocal); + Array<2, SIZE, DeviceType> &level_volume_offsets(); data_structure_type data_structure(); bool is_initialized(); bool can_reuse(std::vector shape); @@ -80,9 +81,14 @@ template struct Hierarchy { // Pre-computed range array for fast quantization Array<2, SIZE, DeviceType> _level_ranges; Array<2, int, DeviceType> _level_marks; - // Pre-computed volume array for fast quantization - Array<3, T, DeviceType> _level_volumes; - Array<3, T, DeviceType> _level_volumes_reciprocal; + // Pre-computed volume array for fast quantization. Stored per level: the + // level_shape[l][d] volumes of each (level l, dim d) are packed back to + // back starting at _level_volume_offsets(l, d), so the total is + // sum_l sum_d level_shape[l][d] rather than (l_target + 1) * D * max_width + // (which in 1D is ~log2(N) * N). + Array<1, T, DeviceType> _level_volumes; + Array<1, T, DeviceType> _level_volumes_reciprocal; + Array<2, SIZE, DeviceType> _level_volume_offsets; // Indicating if it is uniform or non-uniform grid enum data_structure_type dstype; diff --git a/include/mgard-x/Hierarchy/Hierarchy.hpp b/include/mgard-x/Hierarchy/Hierarchy.hpp index a9b4acb64..d157320ea 100644 --- a/include/mgard-x/Hierarchy/Hierarchy.hpp +++ b/include/mgard-x/Hierarchy/Hierarchy.hpp @@ -403,26 +403,30 @@ void Hierarchy::init(std::vector shape, } } - { // volume for quantization - SIZE volumes_width = 0; - for (DIM d = 0; d < D; d++) { - volumes_width = std::max(volumes_width, _level_shape[_l_target][d]); + { // volume for quantization, packed per level (see Hierarchy.h) + std::vector offsets_h((_l_target + 1) * D); + SIZE total_volumes = 0; + for (SIZE l = 0; l < _l_target + 1; l++) { + for (DIM d = 0; d < D; d++) { + offsets_h[l * D + d] = total_volumes; + total_volumes += _level_shape[l][d]; + } } bool pitched = false; - _level_volumes = - Array<3, T, DeviceType>({_l_target + 1, D, volumes_width}, pitched); + _level_volume_offsets = + Array<2, SIZE, DeviceType>({_l_target + 1, D}, pitched); + _level_volume_offsets.load(offsets_h.data()); + _level_volumes = Array<1, T, DeviceType>({total_volumes}, pitched); _level_volumes_reciprocal = - Array<3, T, DeviceType>({_l_target + 1, D, volumes_width}, pitched); - SubArray<3, T, DeviceType> volumes_subarray(_level_volumes); - SubArray<3, T, DeviceType> volumes_reciprocal_subarray( - _level_volumes_reciprocal); + Array<1, T, DeviceType>({total_volumes}, pitched); for (SIZE l = 0; l < _l_target + 1; l++) { for (DIM d = 0; d < D; d++) { + SIZE offset = offsets_h[l * D + d]; calc_volume(_level_shape[l][d], _dist_array[l][d].data(), - volumes_subarray(l, d, 0), false); + _level_volumes.data() + offset, false); calc_volume(_level_shape[l][d], _dist_array[l][d].data(), - volumes_reciprocal_subarray(l, d, 0), true); + _level_volumes_reciprocal.data() + offset, true); } } } @@ -574,13 +578,15 @@ Hierarchy::EstimateMemoryFootprint(std::vector shape) { } } - { // volume for quantization - SIZE volumes_width = 0; - for (DIM d = 0; d < D; d++) { - volumes_width = std::max(volumes_width, _level_shape[_l_target][d]); + { // volume for quantization, packed per level + size_t total_volumes = 0; + for (SIZE l = 0; l < _l_target + 1; l++) { + for (DIM d = 0; d < D; d++) { + total_volumes += _level_shape[l][d]; + } } - estimate_memory_usgae += (_l_target + 1) * D * volumes_width * sizeof(T); - estimate_memory_usgae += (_l_target + 1) * D * volumes_width * sizeof(T); + estimate_memory_usgae += 2 * total_volumes * sizeof(T); + estimate_memory_usgae += (_l_target + 1) * D * sizeof(SIZE); } { // am and bm @@ -739,7 +745,7 @@ Array<2, int, DeviceType> &Hierarchy::level_marks() { } template -Array<3, T, DeviceType> & +Array<1, T, DeviceType> & Hierarchy::level_volumes(bool reciprocal) { if (!reciprocal) { return _level_volumes; @@ -748,6 +754,12 @@ Hierarchy::level_volumes(bool reciprocal) { } } +template +Array<2, SIZE, DeviceType> & +Hierarchy::level_volume_offsets() { + return _level_volume_offsets; +} + template data_structure_type Hierarchy::data_structure() { return dstype; @@ -865,6 +877,7 @@ Hierarchy::Hierarchy(const Hierarchy &hierarchy) { _ratio_array = hierarchy._ratio_array; _level_volumes = hierarchy._level_volumes; _level_volumes_reciprocal = hierarchy._level_volumes_reciprocal; + _level_volume_offsets = hierarchy._level_volume_offsets; _am_array = hierarchy._am_array; _bm_array = hierarchy._bm_array; _level_shape = hierarchy._level_shape; diff --git a/include/mgard-x/Quantization/LinearQuantization.hpp b/include/mgard-x/Quantization/LinearQuantization.hpp index 43bbef8e6..986fe3f1d 100644 --- a/include/mgard-x/Quantization/LinearQuantization.hpp +++ b/include/mgard-x/Quantization/LinearQuantization.hpp @@ -25,13 +25,15 @@ class LevelwiseLinearQuantizerNDFunctor : public Functor { SubArray<2, SIZE, DeviceType> level_ranges, SubArray<2, int, DeviceType> level_marks, SIZE l_target, SubArray<1, T, DeviceType> quantizers, - SubArray<3, T, DeviceType> level_volumes, bool calc_vol, + SubArray<1, T, DeviceType> level_volumes, + SubArray<2, SIZE, DeviceType> level_volume_offsets, bool calc_vol, SubArray v, SubArray quantized_v, bool prep_huffman, SIZE dict_size) : level_ranges(level_ranges), level_marks(level_marks), l_target(l_target), quantizers(quantizers), - level_volumes(level_volumes), calc_vol(calc_vol), v(v), + level_volumes(level_volumes), + level_volume_offsets(level_volume_offsets), calc_vol(calc_vol), v(v), quantized_v(quantized_v), prep_huffman(prep_huffman), dict_size(dict_size) { Functor(); @@ -81,7 +83,7 @@ class LevelwiseLinearQuantizerNDFunctor : public Functor { // Determine volume for (int d = D - 1; d >= 0; d--) { - volume *= *level_volumes(level, d, idx[d]); + volume *= *level_volumes(*level_volume_offsets(level, d) + idx[d]); } if (sizeof(T) == sizeof(double)) volume = sqrt(volume); @@ -130,7 +132,8 @@ class LevelwiseLinearQuantizerNDFunctor : public Functor { SubArray<2, int, DeviceType> level_marks; SIZE l_target; SubArray<1, T, DeviceType> quantizers; - SubArray<3, T, DeviceType> level_volumes; + SubArray<1, T, DeviceType> level_volumes; + SubArray<2, SIZE, DeviceType> level_volume_offsets; SubArray v; SubArray quantized_v; bool calc_vol; @@ -155,13 +158,15 @@ class LevelwiseLinearQuantizerKernel : public Kernel { SubArray<2, SIZE, DeviceType> level_ranges, SubArray<2, int, DeviceType> level_marks, SIZE l_target, SubArray<1, T, DeviceType> quantizers, - SubArray<3, T, DeviceType> level_volumes, bool calc_vol, + SubArray<1, T, DeviceType> level_volumes, + SubArray<2, SIZE, DeviceType> level_volume_offsets, bool calc_vol, SubArray v, SubArray quantized_v, bool prep_huffman, SIZE dict_size) : level_ranges(level_ranges), level_marks(level_marks), l_target(l_target), quantizers(quantizers), - level_volumes(level_volumes), calc_vol(calc_vol), v(v), + level_volumes(level_volumes), + level_volume_offsets(level_volume_offsets), calc_vol(calc_vol), v(v), quantized_v(quantized_v), prep_huffman(prep_huffman), dict_size(dict_size) {} @@ -173,8 +178,8 @@ class LevelwiseLinearQuantizerKernel : public Kernel { LevelwiseLinearQuantizerNDFunctor; FunctorType functor(level_ranges, level_marks, l_target, quantizers, - level_volumes, calc_vol, v, quantized_v, prep_huffman, - dict_size); + level_volumes, level_volume_offsets, calc_vol, v, + quantized_v, prep_huffman, dict_size); SIZE total_thread_z = v.shape(D - 3); SIZE total_thread_y = v.shape(D - 2); @@ -203,7 +208,8 @@ class LevelwiseLinearQuantizerKernel : public Kernel { SubArray<2, int, DeviceType> level_marks; SIZE l_target; SubArray<1, T, DeviceType> quantizers; - SubArray<3, T, DeviceType> level_volumes; + SubArray<1, T, DeviceType> level_volumes; + SubArray<2, SIZE, DeviceType> level_volume_offsets; bool calc_vol; SubArray v; SubArray quantized_v; @@ -330,8 +336,10 @@ class LinearQuantizer : public QuantizationInterface { SubArray<2, SIZE, DeviceType> level_ranges_subarray( hierarchy->level_ranges()); SubArray<2, int, DeviceType> level_marks_subarray(hierarchy->level_marks()); - SubArray<3, T, DeviceType> level_volumes_subarray( + SubArray<1, T, DeviceType> level_volumes_subarray( hierarchy->level_volumes(false)); + SubArray<2, SIZE, DeviceType> level_volume_offsets_subarray( + hierarchy->level_volume_offsets()); SubArray<1, T, DeviceType> quantizers_subarray(quantizers_array); T *quantizers = new T[hierarchy->l_target() + 1]; CalcQuantizers(total_elems, quantizers, ebtype, tol, s, norm, @@ -351,8 +359,9 @@ class LinearQuantizer : public QuantizationInterface { DeviceLauncher::Execute( LevelwiseLinearQuantizerKernel( level_ranges_subarray, level_marks_subarray, hierarchy->l_target(), - quantizers_subarray, level_volumes_subarray, calc_vol, - original_data, quantized_data, prep_huffman, config.huff_dict_size), + quantizers_subarray, level_volumes_subarray, + level_volume_offsets_subarray, calc_vol, original_data, + quantized_data, prep_huffman, config.huff_dict_size), queue_idx); if (log::level & log::TIME) { @@ -376,8 +385,10 @@ class LinearQuantizer : public QuantizationInterface { SubArray<2, SIZE, DeviceType> level_ranges_subarray( hierarchy->level_ranges()); SubArray<2, int, DeviceType> level_marks_subarray(hierarchy->level_marks()); - SubArray<3, T, DeviceType> level_volumes_subarray( + SubArray<1, T, DeviceType> level_volumes_subarray( hierarchy->level_volumes(true)); + SubArray<2, SIZE, DeviceType> level_volume_offsets_subarray( + hierarchy->level_volume_offsets()); // Must mirror the toggle used during Quantize so the dictionary shift is // undone exactly when it was applied. @@ -405,8 +416,9 @@ class LinearQuantizer : public QuantizationInterface { DeviceLauncher::Execute( LevelwiseLinearQuantizerKernel( level_ranges_subarray, level_marks_subarray, hierarchy->l_target(), - quantizers_subarray, level_volumes_subarray, calc_vol, - original_data, quantized_data, prep_huffman, config.huff_dict_size), + quantizers_subarray, level_volumes_subarray, + level_volume_offsets_subarray, calc_vol, original_data, + quantized_data, prep_huffman, config.huff_dict_size), queue_idx); DeviceRuntime::SyncQueue(queue_idx); From a5c9e25aaae9d257c9cb6a62c53d8ab566950b7b Mon Sep 17 00:00:00 2001 From: Jieyang Chen Date: Thu, 1 Oct 2026 16:19:15 -0700 Subject: [PATCH 03/11] mgard-x: move hierarchies into the compressor/refactor caches CompressorCache and RefactorCache inserted with (*hierarchy_cache)[key] = hierarchy, a copy assignment that deep-copies every device array (Hierarchy declared a copy constructor, so it had no implicit move). For a moment two full hierarchies were live: 1792 MB instead of 896 MB for a 2^24-element 1D float input. Give Hierarchy defaulted move constructor/assignment (Array's move hands over the buffers and leaves the source empty; moving into a populated Array frees it first) and an explicitly defaulted copy assignment, which declaring the moves would otherwise delete. Both caches now insert with std::move(hierarchy); shape is read before the move. mgard-x output is byte-identical across 1D-4D, float/double, s = inf/0/1, rel/abs, multi-dim and hybrid; mdr-x refactored components and reconstruction errors are unchanged. Co-Authored-By: Claude Opus 5.5 --- include/mgard-x/CompressionLowLevel/CompressorCache.hpp | 3 ++- include/mgard-x/Hierarchy/Hierarchy.h | 5 +++++ include/mgard-x/MDR-X/Refactor/RefactorCache.hpp | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/include/mgard-x/CompressionLowLevel/CompressorCache.hpp b/include/mgard-x/CompressionLowLevel/CompressorCache.hpp index 9a335f087..fdc27192b 100644 --- a/include/mgard-x/CompressionLowLevel/CompressorCache.hpp +++ b/include/mgard-x/CompressionLowLevel/CompressorCache.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "../Utilities/Types.h" @@ -60,7 +61,7 @@ class CompressorBundle { void InsertHierarchyCache(HierarchyType hierarchy) { std::vector shape = hierarchy.level_shape(hierarchy.l_target()); - (*hierarchy_cache)[GenHierarchyKey(shape)] = hierarchy; + (*hierarchy_cache)[GenHierarchyKey(shape)] = std::move(hierarchy); std::stringstream ss; for (int i = 0; i < shape.size(); i++) { ss << shape[i] << " "; diff --git a/include/mgard-x/Hierarchy/Hierarchy.h b/include/mgard-x/Hierarchy/Hierarchy.h index 910240a72..82405df2c 100644 --- a/include/mgard-x/Hierarchy/Hierarchy.h +++ b/include/mgard-x/Hierarchy/Hierarchy.h @@ -23,6 +23,11 @@ template struct Hierarchy { /* for Internal use only */ Hierarchy(); Hierarchy(const Hierarchy &hierarchy); + // Moves hand over the device arrays (a moved-from Array is left empty), so + // e.g. inserting into a hierarchy cache does not deep-copy every array. + Hierarchy(Hierarchy &&hierarchy) = default; + Hierarchy &operator=(const Hierarchy &hierarchy) = default; + Hierarchy &operator=(Hierarchy &&hierarchy) = default; SIZE total_num_elems(); SIZE level_num_elems(SIZE level); diff --git a/include/mgard-x/MDR-X/Refactor/RefactorCache.hpp b/include/mgard-x/MDR-X/Refactor/RefactorCache.hpp index c6aa73351..1dee9c880 100644 --- a/include/mgard-x/MDR-X/Refactor/RefactorCache.hpp +++ b/include/mgard-x/MDR-X/Refactor/RefactorCache.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include namespace mgard_x { @@ -54,7 +55,7 @@ class RefactorBundle { void InsertHierarchyCache(HierarchyType hierarchy) { std::vector shape = hierarchy.level_shape(hierarchy.l_target()); - (*hierarchy_cache)[GenHierarchyKey(shape)] = hierarchy; + (*hierarchy_cache)[GenHierarchyKey(shape)] = std::move(hierarchy); std::stringstream ss; for (int i = 0; i < shape.size(); i++) { ss << shape[i] << " "; From 35f03435d96dfa87af062a3d33ff981b7e4e2cda Mon Sep 17 00:00:00 2001 From: Jieyang Chen Date: Fri, 2 Oct 2026 09:11:14 -0700 Subject: [PATCH 04/11] mgard-x: add HybridHierarchy for the hybrid (local + global) decomposition The hybrid compressor used a full Hierarchy of the input grid, of which the block-local stage reads only the shape, plus a second Hierarchy for the global stage that HybridHierarchyDataRefactor::Adapt rebuilt on every call (once per subdomain). In 1D the input-sized hierarchy alone was ~14x the input. HybridHierarchy derives from Hierarchy. Its base part is shape-only (new protected Hierarchy::init_shape_only: one level equal to the input shape, no device arrays), so l_target()/level_shape()/total_num_elems() keep working for the pipelines and the cache. It stores the input shape, the local/global level counts, the padded fine/coarse block grid of each local level (computed once by compute_local_shapes, replacing four copies of the 8 -> 5 rule), and owns the global Hierarchy, built once, only when M > 0, on the coarsest local grid (the input grid when L == 0). - HybridHierarchyCompressor::HierarchyType is HybridHierarchy; the compressor, hybrid/block-local refactors and both quantizers take a HybridHierarchy& and read grids and the global hierarchy from it. check_config() rejects a config whose L/M differ from the hierarchy's. - CompressorCache::InHierarchyCache takes the Config and drops entries for which matches_config() fails (always true for a plain Hierarchy). - DomainDecomposer builds OperatorType::HierarchyType and sizes it with the new static HierarchyType::EstimateMemoryFootprint(shape, config). - Hybrid tests construct a HybridHierarchy. Hybrid peak device memory, 1D float 2^24 elements (SERIAL): L=1 2121 -> 1225 MB, L=1/M=1 2913 -> 1705 MB, L=2/M=2 2643 -> 1563 MB, L=0/M=2 3465 -> 1897 MB; with L=1 the hierarchy allocates no device memory. SERIAL output is byte-identical across 1D-4D, float/double, s = inf/0, multi-dim and hybrid L1/L2/L1G1/L2G2/L0G2. SERIAL and CUDA (RTX PRO 6000 Blackwell) unit tests pass. Co-Authored-By: Claude Opus 5.5 --- .../CompressionHighLevel/CPUPipelines.hpp | 8 +- .../CompressionHighLevel/GPUPipelines.hpp | 8 +- .../CompressionLowLevel/CompressorCache.hpp | 7 +- .../HybridHierarchyCompressor.h | 16 +- .../HybridHierarchyCompressor.hpp | 43 +++-- .../BlockLocalHierarchyDataRefactor.hpp | 20 +- .../HybridHierarchyDataRefactor.hpp | 46 ++--- .../DomainDecomposer/DomainDecomposer.hpp | 22 +-- include/mgard-x/Hierarchy/CMakeLists.txt | 1 + include/mgard-x/Hierarchy/Hierarchy.h | 13 ++ include/mgard-x/Hierarchy/Hierarchy.hpp | 36 ++++ include/mgard-x/Hierarchy/HybridHierarchy.h | 174 ++++++++++++++++++ .../HybridHierarchyLinearQuantization.hpp | 38 ++-- .../Quantization/LocalQuantization.hpp | 19 +- tests/mgard-x/src/test_HybridHierarchy.cpp.in | 30 +-- .../src/test_KernelFusionSwitch.cpp.in | 4 +- 16 files changed, 347 insertions(+), 138 deletions(-) create mode 100644 include/mgard-x/Hierarchy/HybridHierarchy.h diff --git a/include/mgard-x/CompressionHighLevel/CPUPipelines.hpp b/include/mgard-x/CompressionHighLevel/CPUPipelines.hpp index 9677ece15..c24b7f50d 100644 --- a/include/mgard-x/CompressionHighLevel/CPUPipelines.hpp +++ b/include/mgard-x/CompressionHighLevel/CPUPipelines.hpp @@ -18,13 +18,13 @@ enum compress_status_type compress_pipeline_cpu( Cache::cache.device_compressed_buffer; if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0), - domain_decomposer.uniform)) { + domain_decomposer.uniform, config)) { Cache::cache.ClearHierarchyCache(); } for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id), - domain_decomposer.uniform)) { + domain_decomposer.uniform, config)) { Cache::cache.InsertHierarchyCache( domain_decomposer.subdomain_hierarchy(id)); } @@ -290,13 +290,13 @@ enum compress_status_type decompress_pipeline_cpu( Cache::cache.device_compressed_buffer; if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0), - domain_decomposer.uniform)) { + domain_decomposer.uniform, config)) { Cache::cache.ClearHierarchyCache(); } for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id), - domain_decomposer.uniform)) { + domain_decomposer.uniform, config)) { Cache::cache.InsertHierarchyCache( domain_decomposer.subdomain_hierarchy(id)); } diff --git a/include/mgard-x/CompressionHighLevel/GPUPipelines.hpp b/include/mgard-x/CompressionHighLevel/GPUPipelines.hpp index 36ae915d4..df366a56a 100644 --- a/include/mgard-x/CompressionHighLevel/GPUPipelines.hpp +++ b/include/mgard-x/CompressionHighLevel/GPUPipelines.hpp @@ -18,13 +18,13 @@ enum compress_status_type compress_pipeline_gpu( Cache::cache.device_compressed_buffer; if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0), - domain_decomposer.uniform)) { + domain_decomposer.uniform, config)) { Cache::cache.ClearHierarchyCache(); } for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id), - domain_decomposer.uniform)) { + domain_decomposer.uniform, config)) { Cache::cache.InsertHierarchyCache( domain_decomposer.subdomain_hierarchy(id)); } @@ -277,13 +277,13 @@ enum compress_status_type decompress_pipeline_gpu( Cache::cache.device_compressed_buffer; if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0), - domain_decomposer.uniform)) { + domain_decomposer.uniform, config)) { Cache::cache.ClearHierarchyCache(); } for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id), - domain_decomposer.uniform)) { + domain_decomposer.uniform, config)) { Cache::cache.InsertHierarchyCache( domain_decomposer.subdomain_hierarchy(id)); } diff --git a/include/mgard-x/CompressionLowLevel/CompressorCache.hpp b/include/mgard-x/CompressionLowLevel/CompressorCache.hpp index fdc27192b..fa9766d70 100644 --- a/include/mgard-x/CompressionLowLevel/CompressorCache.hpp +++ b/include/mgard-x/CompressionLowLevel/CompressorCache.hpp @@ -43,7 +43,9 @@ class CompressorBundle { return ss.str(); } - bool InHierarchyCache(std::vector shape, bool uniform) { + // `config` matters for hierarchies that depend on more than the shape (a + // HybridHierarchy is built for given local/global level counts). + bool InHierarchyCache(std::vector shape, bool uniform, Config config) { if (!initialized) { return false; } else if (!uniform) { @@ -54,6 +56,9 @@ class CompressorBundle { } else if ((*hierarchy_cache)[GenHierarchyKey(shape)].data_structure() == data_structure_type::Cartesian_Grid_Non_Uniform) { return false; + } else if (!(*hierarchy_cache)[GenHierarchyKey(shape)].matches_config( + config)) { + return false; } else { return true; } diff --git a/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.h b/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.h index d64affba2..0180e51d3 100644 --- a/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.h +++ b/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.h @@ -14,7 +14,7 @@ // #include "CompressionLowLevelWorkspace.hpp" -#include "../Hierarchy/Hierarchy.h" +#include "../Hierarchy/HybridHierarchy.h" #include "../Lossless/Lossless.hpp" #include "../Quantization/HybridHierarchyLinearQuantization.hpp" #include "../Quantization/LocalQuantization.hpp" @@ -28,7 +28,7 @@ template class HybridHierarchyCompressor : public LossyCompressorInterface { public: - using HierarchyType = Hierarchy; + using HierarchyType = HybridHierarchy; // using BlockLocalHierarchyDataRefactorType = // data_refactoring::BlockLocalHierarchyDataRefactor; using HybridHierarchyDataRefactorType = @@ -42,10 +42,10 @@ class HybridHierarchyCompressor HybridHierarchyCompressor(); - HybridHierarchyCompressor(Hierarchy &hierarchy, + HybridHierarchyCompressor(HybridHierarchy &hierarchy, Config config); - void Adapt(Hierarchy &hierarchy, Config config, + void Adapt(HybridHierarchy &hierarchy, Config config, int queue_idx); static size_t EstimateMemoryFootprint(std::vector shape, Config config); @@ -106,11 +106,13 @@ class HybridHierarchyCompressor enum error_bound_type ebtype, T tol, T s, T &norm, Array &decompressed_data, int queue_idx); - static SIZE calculate_padded_size(Hierarchy &hierarchy, - Config config); + static SIZE calculate_padded_size(std::vector shape, Config config); + static SIZE + calculate_padded_size(HybridHierarchy &hierarchy, + Config config); bool initialized; - Hierarchy *hierarchy; + HybridHierarchy *hierarchy; Config config; bool orthogonal_projection = true; Array<1, T, DeviceType> norm_tmp_array; diff --git a/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.hpp b/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.hpp index 43cb42a01..de7b7b043 100644 --- a/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.hpp +++ b/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.hpp @@ -32,11 +32,11 @@ HybridHierarchyCompressor::HybridHierarchyCompressor() template HybridHierarchyCompressor::HybridHierarchyCompressor( - Hierarchy &hierarchy, Config config) + HybridHierarchy &hierarchy, Config config) : initialized(true), hierarchy(&hierarchy), config(config), hybrid_refactor(hierarchy, config), lossless_compressor(calculate_padded_size(hierarchy, config), config), - hybrid_quantizer(hierarchy, hybrid_refactor.global_hierarchy, config) { + hybrid_quantizer(hierarchy, config) { norm_array = Array<1, T, DeviceType>({1}); hybrid_decomposed_array = Array<1, T, DeviceType>({hybrid_refactor.DecomposedDataSize()}); @@ -70,15 +70,16 @@ HybridHierarchyCompressor::HybridHierarchyCompressor( template void HybridHierarchyCompressor::Adapt( - Hierarchy &hierarchy, Config config, int queue_idx) { + HybridHierarchy &hierarchy, Config config, + int queue_idx) { + hierarchy.check_config(config); this->initialized = true; this->hierarchy = &hierarchy; this->config = config; hybrid_refactor.Adapt(hierarchy, config, queue_idx); lossless_compressor.Adapt(calculate_padded_size(hierarchy, config), config, queue_idx); - hybrid_quantizer.Adapt(hierarchy, hybrid_refactor.global_hierarchy, config, - queue_idx); + hybrid_quantizer.Adapt(hierarchy, config, queue_idx); norm_array.resize({1}, queue_idx); hybrid_decomposed_array.resize({hybrid_refactor.DecomposedDataSize()}, queue_idx); @@ -115,8 +116,6 @@ void HybridHierarchyCompressor::Adapt( template size_t HybridHierarchyCompressor::EstimateMemoryFootprint( std::vector shape, Config config) { - Hierarchy hierarchy; - hierarchy.EstimateMemoryFootprint(shape); size_t size = 0; // size += // BlockLocalHierarchyDataRefactorType::EstimateMemoryFootprint(shape); @@ -138,7 +137,7 @@ size_t HybridHierarchyCompressor::EstimateMemoryFootprint( // + // " GB"); size += LosslessCompressorType::EstimateMemoryFootprint( - calculate_padded_size(hierarchy, config), config); + calculate_padded_size(shape, config), config); // log::info( // "Lossless space: " + // std::to_string((double)(LosslessCompressorType::EstimateMemoryFootprint( @@ -147,8 +146,8 @@ size_t HybridHierarchyCompressor::EstimateMemoryFootprint( // " GB"); size += sizeof(T); if (sizeof(QUANTIZED_INT) > sizeof(T)) { - size += sizeof(T) * calculate_padded_size(hierarchy, config); - size += sizeof(QUANTIZED_INT) * calculate_padded_size(hierarchy, config); + size += sizeof(T) * calculate_padded_size(shape, config); + size += sizeof(QUANTIZED_INT) * calculate_padded_size(shape, config); } return size; } @@ -427,19 +426,18 @@ void HybridHierarchyCompressor::Decompress( // Only calculating padding to 8x8x8 for once template SIZE HybridHierarchyCompressor::calculate_padded_size( - Hierarchy &hierarchy, Config config) { + std::vector shape, Config config) { int L = config.num_local_refactoring_level; SIZE total_num_elems_1D = 1; if (L > 0) { - std::vector coarse_shape = - hierarchy.level_shape(hierarchy.l_target()); + std::vector> fine_shapes, coarse_shapes; + HybridHierarchy::compute_local_shapes( + shape, L, fine_shapes, coarse_shapes); for (int l = 0; l < L; l++) { SIZE last_level_size = 1, curr_level_size = 1; for (DIM d = 0; d < D; d++) { - coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 8; - last_level_size *= coarse_shape[d]; - coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 5; - curr_level_size *= coarse_shape[d]; + last_level_size *= fine_shapes[l][d]; + curr_level_size *= coarse_shapes[l][d]; } total_num_elems_1D += (last_level_size - curr_level_size); if (l == L - 1) { @@ -447,12 +445,21 @@ SIZE HybridHierarchyCompressor::calculate_padded_size( } } } else { - total_num_elems_1D = hierarchy.total_num_elems(); + total_num_elems_1D = 1; + for (DIM d = 0; d < D; d++) { + total_num_elems_1D *= shape[d]; + } } return total_num_elems_1D; } +template +SIZE HybridHierarchyCompressor::calculate_padded_size( + HybridHierarchy &hierarchy, Config config) { + return calculate_padded_size(hierarchy.input_shape(), config); +} + } // namespace mgard_x #endif diff --git a/include/mgard-x/DataRefactoring/BlockLocalHierarchyDataRefactor.hpp b/include/mgard-x/DataRefactoring/BlockLocalHierarchyDataRefactor.hpp index 38d434c20..858bb8d5e 100644 --- a/include/mgard-x/DataRefactoring/BlockLocalHierarchyDataRefactor.hpp +++ b/include/mgard-x/DataRefactoring/BlockLocalHierarchyDataRefactor.hpp @@ -1,3 +1,4 @@ +#include "../Hierarchy/HybridHierarchy.h" #include "InCacheBlock/DataRefactoring.h" #include "MultiDimension/DataRefactoring.h" @@ -14,9 +15,10 @@ class BlockLocalHierarchyDataRefactor { BlockLocalHierarchyDataRefactor() : initialized(false) {} // Removing all checks for L since processed in HybridHierarchyDataRefactor - BlockLocalHierarchyDataRefactor(Hierarchy &hierarchy, + BlockLocalHierarchyDataRefactor(HybridHierarchy &hierarchy, Config config) : initialized(true), hierarchy(&hierarchy), config(config) { + hierarchy.check_config(config); this->L = config.num_local_refactoring_level; compute_local_ranges(); w_array = Array<1, T, DeviceType>({DecomposedCoeffSize()}); @@ -26,8 +28,9 @@ class BlockLocalHierarchyDataRefactor { coarse_buffers[1] = Array(fine_shapes[0]); } - void Adapt(Hierarchy &hierarchy, Config config, + void Adapt(HybridHierarchy &hierarchy, Config config, int queue_idx) { + hierarchy.check_config(config); this->initialized = true; this->hierarchy = &hierarchy; this->config = config; @@ -69,8 +72,7 @@ class BlockLocalHierarchyDataRefactor { } void compute_local_ranges() { - // Get original shape from hierarchy - coarse_shape = hierarchy->level_shape(hierarchy->l_target()); + coarse_shape = hierarchy->input_shape(); fine_num_elems.clear(); coarse_num_elems.clear(); local_coeff_size.clear(); @@ -79,12 +81,10 @@ class BlockLocalHierarchyDataRefactor { for (int l = 0; l < this->L; ++l) { SIZE last_level_size = 1, curr_level_size = 1; - std::vector fine_shape(D); + std::vector fine_shape = hierarchy->local_fine_shape(l); + coarse_shape = hierarchy->local_coarse_shape(l); for (DIM d = 0; d < D; ++d) { - coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 8; - last_level_size *= coarse_shape[d]; - fine_shape[d] = coarse_shape[d]; - coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 5; + last_level_size *= fine_shape[d]; curr_level_size *= coarse_shape[d]; } fine_num_elems.push_back(last_level_size); @@ -461,7 +461,7 @@ class BlockLocalHierarchyDataRefactor { SIZE accumulated_local_coeff_size = 0; bool initialized; SIZE L; - Hierarchy *hierarchy; + HybridHierarchy *hierarchy; Config config; std::vector fine_num_elems; diff --git a/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactor.hpp b/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactor.hpp index 7d8a2939c..0a685e552 100644 --- a/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactor.hpp +++ b/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactor.hpp @@ -5,6 +5,7 @@ * Date: September 21, 2026 */ +#include "../Hierarchy/HybridHierarchy.h" #include "../RuntimeX/Utilities/Exceptions.h" #include "BlockLocalHierarchyDataRefactor.hpp" #include "DataRefactor.hpp" @@ -24,14 +25,14 @@ class HybridHierarchyDataRefactor : public HybridHierarchyDataRefactorInterface { public: HybridHierarchyDataRefactor() : initialized(false) {} - HybridHierarchyDataRefactor(Hierarchy &hierarchy, + HybridHierarchyDataRefactor(HybridHierarchy &hierarchy, Config config) : initialized(true), hierarchy(&hierarchy), config(config) { this->L = config.num_local_refactoring_level; this->M = config.num_global_refactoring_level; } - void Adapt(Hierarchy &hierarchy, Config config, + void Adapt(HybridHierarchy &hierarchy, Config config, int queue_idx) { this->initialized = true; this->hierarchy = &hierarchy; @@ -43,30 +44,18 @@ class HybridHierarchyDataRefactor if (this->L == 0 && this->M == 0) { throw ProcessingException("Both L and M cannot be zero"); } + hierarchy.check_config(config); if (this->L > 0) { local_refactor.Adapt(hierarchy, config, queue_idx); } if (this->M > 0) { - if (this->L > 0) { - // With local, global adapt from the output shape of local - std::vector global_hierarchy_shape = - local_refactor.coarse_shapes[this->L - 1]; - Config global_config; - global_config.max_larget_level = this->M; - this->global_hierarchy = - Hierarchy(global_hierarchy_shape, global_config); - global_refactor.Adapt(this->global_hierarchy, global_config, queue_idx); - } else { - // Without local, global directly adapt to original shape - Config global_config; - global_config.max_larget_level = this->M; - - this->global_hierarchy = Hierarchy( - hierarchy.level_shape(hierarchy.l_target()), global_config); - global_refactor.Adapt(global_hierarchy, global_config, queue_idx); - } + // The global hierarchy is built (once) by the HybridHierarchy, on the + // coarsest local grid or, without a local stage, on the input grid. + global_refactor.Adapt( + hierarchy.global_hierarchy(), + HybridHierarchy::global_config(this->M), queue_idx); } } @@ -110,9 +99,7 @@ class HybridHierarchyDataRefactor // decompose+quantize path can run it separately from the local stage. void DecomposeGlobal(SubArray<1, T, DeviceType> decomposed_data, int queue_idx, bool orthogonal_projection = true) { - std::vector global_shape = - (this->L > 0) ? local_refactor.coarse_shapes[this->L - 1] - : hierarchy->level_shape(hierarchy->l_target()); + std::vector global_shape = hierarchy->global_shape(); SubArray global_input_data(global_shape, decomposed_data.data()); for (DIM d = 0; d < D; d++) { @@ -129,9 +116,7 @@ class HybridHierarchyDataRefactor // dequantize+recompose path can run it separately from the local stage. void RecomposeGlobal(SubArray<1, T, DeviceType> decomposed_data, int queue_idx, bool orthogonal_projection = true) { - std::vector global_shape = - (this->L > 0) ? local_refactor.coarse_shapes[this->L - 1] - : hierarchy->level_shape(hierarchy->l_target()); + std::vector global_shape = hierarchy->global_shape(); SubArray global_input_data(global_shape, decomposed_data.data()); for (DIM d = 0; d < D; d++) { @@ -157,8 +142,7 @@ class HybridHierarchyDataRefactor } if (this->L == 0) { // Pure Global (In-Place) - std::vector original_shape = - hierarchy->level_shape(hierarchy->l_target()); + std::vector original_shape = hierarchy->input_shape(); SubArray global_input_data(original_shape, decomposed_data.data()); for (DIM d = 0; d < D; d++) { @@ -206,8 +190,7 @@ class HybridHierarchyDataRefactor } if (this->L == 0) { // Pure Global (In-Place) - std::vector original_shape = - hierarchy->level_shape(hierarchy->l_target()); + std::vector original_shape = hierarchy->input_shape(); SubArray global_input_data(original_shape, decomposed_data.data()); for (DIM d = 0; d < D; d++) { @@ -253,8 +236,7 @@ class HybridHierarchyDataRefactor } bool initialized; - Hierarchy *hierarchy; - Hierarchy global_hierarchy; + HybridHierarchy *hierarchy; Config config; SIZE L; // Number of local levels diff --git a/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp b/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp index 164b1c6c6..5be08cf7b 100644 --- a/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp +++ b/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp @@ -47,12 +47,11 @@ class DomainDecomposer { " GB"); using HierarchyType = typename OperatorType::HierarchyType; - HierarchyType hierarchy; - estimate_memory_usgae += hierarchy.EstimateMemoryFootprint(shape); - log::dbg( - "Hierarchy space: " + - std::to_string((double)hierarchy.EstimateMemoryFootprint(shape) / 1e9) + - " GB"); + size_t hierarchy_space = + HierarchyType::EstimateMemoryFootprint(shape, config); + estimate_memory_usgae += hierarchy_space; + log::dbg("Hierarchy space: " + + std::to_string((double)hierarchy_space / 1e9) + " GB"); // For prefetching if (enable_prefetch) { @@ -250,9 +249,10 @@ class DomainDecomposer { return true; } - Hierarchy subdomain_hierarchy(int subdomain_id) { + typename OperatorType::HierarchyType subdomain_hierarchy(int subdomain_id) { + using HierarchyType = typename OperatorType::HierarchyType; if (uniform) { - return Hierarchy(subdomain_shape(subdomain_id), config); + return HierarchyType(subdomain_shape(subdomain_id), config); } else { if (config.domain_decomposition == domain_decomposition_type::MaxDim || config.domain_decomposition == domain_decomposition_type::Variable) { @@ -266,8 +266,7 @@ class DomainDecomposer { chunk_shape[_domain_decomposed_dim], 0); DeviceRuntime::SyncQueue(0); chunk_coords[_domain_decomposed_dim] = decompose_dim_coord; - Hierarchy hierarchy(chunk_shape, chunk_coords, - config); + HierarchyType hierarchy(chunk_shape, chunk_coords, config); delete[] decompose_dim_coord; return hierarchy; } else if (config.domain_decomposition == @@ -283,8 +282,7 @@ class DomainDecomposer { chunk_shape[d], 0); DeviceRuntime::SyncQueue(0); } - Hierarchy hierarchy(chunk_shape, chunk_coords, - config); + HierarchyType hierarchy(chunk_shape, chunk_coords, config); for (int d = D - 1; d >= 0; d--) delete[] chunk_coords[d]; return hierarchy; diff --git a/include/mgard-x/Hierarchy/CMakeLists.txt b/include/mgard-x/Hierarchy/CMakeLists.txt index d02867c67..ea510512a 100644 --- a/include/mgard-x/Hierarchy/CMakeLists.txt +++ b/include/mgard-x/Hierarchy/CMakeLists.txt @@ -1,5 +1,6 @@ list(APPEND MGARD_X_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/Hierarchy.h ${CMAKE_CURRENT_SOURCE_DIR}/Hierarchy.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/HybridHierarchy.h ) set(MGARD_X_HEADER ${MGARD_X_HEADER} PARENT_SCOPE) \ No newline at end of file diff --git a/include/mgard-x/Hierarchy/Hierarchy.h b/include/mgard-x/Hierarchy/Hierarchy.h index 82405df2c..6cf480384 100644 --- a/include/mgard-x/Hierarchy/Hierarchy.h +++ b/include/mgard-x/Hierarchy/Hierarchy.h @@ -50,10 +50,23 @@ template struct Hierarchy { data_structure_type data_structure(); bool is_initialized(); bool can_reuse(std::vector shape); + // Whether a cached hierarchy is still valid for `config`. A plain hierarchy + // does not depend on it; HybridHierarchy hides this to compare its level + // counts. + bool matches_config(Config config) { return true; } size_t EstimateMemoryFootprint(std::vector shape); + static size_t EstimateMemoryFootprint(std::vector shape, + Config config); ~Hierarchy(); +protected: + // Describe only the input grid: one level equal to `shape`, so l_target(), + // level_shape() and total_num_elems() work, but no device arrays (coords, + // dist/ratio, am/bm, volumes, ranges/marks) are built. For hierarchies such + // as HybridHierarchy whose decomposition does not use them. + void init_shape_only(std::vector shape, data_structure_type dstype); + private: // Shape of the finest grid std::vector shape; diff --git a/include/mgard-x/Hierarchy/Hierarchy.hpp b/include/mgard-x/Hierarchy/Hierarchy.hpp index d157320ea..06610fa50 100644 --- a/include/mgard-x/Hierarchy/Hierarchy.hpp +++ b/include/mgard-x/Hierarchy/Hierarchy.hpp @@ -823,6 +823,42 @@ bool Hierarchy::can_reuse(std::vector shape) { template Hierarchy::Hierarchy() {} +template +size_t +Hierarchy::EstimateMemoryFootprint(std::vector shape, + Config config) { + Hierarchy hierarchy; + return hierarchy.EstimateMemoryFootprint(shape); +} + +template +void Hierarchy::init_shape_only(std::vector shape, + data_structure_type dstype) { + int ret = check_shape(shape); + if (ret == -1) { + throw std::runtime_error( + "Number of dimensions mismatch. mgard_x::Hierarchy not initialized!"); + } + if (ret == -2) { + throw std::runtime_error("Size of any dimension cannot be smaller than 3. " + "mgard_x::Hierarchy not initialized!"); + } + this->shape = shape; + this->dstype = dstype; + _l_target = 0; + _level_shape = {shape}; + _total_num_elems = 1; + for (DIM d = 0; d < D; d++) { + _total_num_elems *= shape[d]; + } + _linearized_width = 1; + for (int d = D - 2; d >= 0; d--) { + _linearized_width *= shape[d]; + } + _level_num_elems = {_total_num_elems}; + initialized = true; +} + template Hierarchy::Hierarchy(std::vector shape, Config config) { int ret = check_shape(shape); diff --git a/include/mgard-x/Hierarchy/HybridHierarchy.h b/include/mgard-x/Hierarchy/HybridHierarchy.h new file mode 100644 index 000000000..c1eaab1c8 --- /dev/null +++ b/include/mgard-x/Hierarchy/HybridHierarchy.h @@ -0,0 +1,174 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: October 1, 2026 + */ + +#ifndef MGARD_X_HYBRID_HIERARCHY_H +#define MGARD_X_HYBRID_HIERARCHY_H + +#include "../RuntimeX/Utilities/Exceptions.h" +#include "Hierarchy.h" + +namespace mgard_x { + +// Hierarchy of the hybrid (BlockMGARD) decomposition: L block-local levels on +// the input grid, then M global multilevel levels on the coarsest local grid +// (or on the input grid when L == 0). +// +// The block-local stage needs nothing from a Hierarchy beyond the input shape +// and the padded per-level block grids, so the base Hierarchy only describes +// the input grid (one level, no device arrays -- see init_shape_only). The +// only full Hierarchy is the global one, built here once, on the grid the +// global stage actually decomposes, and only when M > 0. +template +class HybridHierarchy : public Hierarchy { +public: + HybridHierarchy() {} + + HybridHierarchy(std::vector shape, Config config) { + init(shape, config, data_structure_type::Cartesian_Grid_Uniform); + } + + // Coordinates are accepted for parity with Hierarchy (DomainDecomposer + // builds hierarchies either way) but not used: the block-local transform + // assumes uniform spacing and the global stage is built on a uniform grid. + // The grid is still marked non-uniform, so the compressor cache treats it + // the same as a non-uniform Hierarchy. + HybridHierarchy(std::vector shape, std::vector coords, + Config config) { + init(shape, config, data_structure_type::Cartesian_Grid_Non_Uniform); + } + + // Shape of the input data. + std::vector input_shape() { return this->level_shape(0); } + + SIZE num_local_levels() { return L; } + SIZE num_global_levels() { return M; } + + // Block-local level l (0 = finest) operates on local_fine_shape(l): the + // previous grid padded up to whole blocks. It produces local_coarse_shape(l), + // MGARDX_HYBRID_LOCAL_COARSE_SIZE nodes per block per dimension. + std::vector local_fine_shape(SIZE l) { return fine_shapes[l]; } + std::vector local_coarse_shape(SIZE l) { return coarse_shapes[l]; } + + // Grid the global stage decomposes: the coarsest local grid, or the input + // grid when there is no local stage. + std::vector global_shape() { + return L > 0 ? coarse_shapes[L - 1] : input_shape(); + } + + // Hierarchy of the global stage. Only exists when M > 0. + Hierarchy &global_hierarchy() { + if (M == 0) { + throw ProcessingException( + "HybridHierarchy: no global hierarchy without global levels."); + } + return global; + } + + // Config the global stage is built and run with: defaults, limited to M + // levels. + static Config global_config(SIZE M) { + Config config; + config.max_larget_level = M; + return config; + } + + bool matches_config(Config config) { + return L == (SIZE)config.num_local_refactoring_level && + M == (SIZE)config.num_global_refactoring_level; + } + + // The hybrid components take their level counts from the config and their + // grids from this hierarchy, so the two have to agree. + void check_config(Config config) { + if (!matches_config(config)) { + throw ProcessingException( + "HybridHierarchy was built for " + std::to_string(L) + + " local / " + std::to_string(M) + " global levels, but config has " + + std::to_string(config.num_local_refactoring_level) + " / " + + std::to_string(config.num_global_refactoring_level) + "."); + } + } + + // Padded block grids of the L local levels of an input of `shape`. + static void compute_local_shapes(std::vector shape, SIZE L, + std::vector> &fine, + std::vector> &coarse) { + fine.clear(); + coarse.clear(); + std::vector curr = shape; + for (SIZE l = 0; l < L; l++) { + std::vector f(D), c(D); + for (DIM d = 0; d < D; d++) { + SIZE num_blocks = (curr[d] - 1) / MGARDX_HYBRID_LOCAL_BLOCK_SIZE + 1; + f[d] = num_blocks * MGARDX_HYBRID_LOCAL_BLOCK_SIZE; + c[d] = num_blocks * MGARDX_HYBRID_LOCAL_COARSE_SIZE; + } + fine.push_back(f); + coarse.push_back(c); + curr = c; + } + } + + // Device memory of a HybridHierarchy for `shape`: only the global + // hierarchy, if any (the base part allocates nothing). Uses the Hierarchy + // estimate, which assumes all levels, so it is an upper bound for M levels. + static size_t EstimateMemoryFootprint(std::vector shape, + Config config) { + SIZE L = config.num_local_refactoring_level; + SIZE M = config.num_global_refactoring_level; + if (M == 0) { + return 0; + } + std::vector global_input = shape; + if (L > 0) { + std::vector> fine, coarse; + compute_local_shapes(shape, L, fine, coarse); + global_input = coarse[L - 1]; + } + return Hierarchy::EstimateMemoryFootprint( + global_input, global_config(M)); + } + + // Same, for this hierarchy's own level counts. + size_t EstimateMemoryFootprint(std::vector shape) { + Config config; + config.num_local_refactoring_level = L; + config.num_global_refactoring_level = M; + return EstimateMemoryFootprint(shape, config); + } + +private: + void init(std::vector shape, Config config, + data_structure_type dstype) { + if (config.num_local_refactoring_level < 0 || + config.num_global_refactoring_level < 0) { + throw ProcessingException( + "HybridHierarchy: level counts cannot be negative."); + } + L = config.num_local_refactoring_level; + M = config.num_global_refactoring_level; + if (L == 0 && M == 0) { + throw ProcessingException( + "HybridHierarchy: local and global levels cannot both be zero."); + } + this->init_shape_only(shape, dstype); + compute_local_shapes(shape, L, fine_shapes, coarse_shapes); + if (M > 0) { + global = Hierarchy(global_shape(), global_config(M)); + } + } + + SIZE L = 0; + SIZE M = 0; + std::vector> fine_shapes; + std::vector> coarse_shapes; + Hierarchy global; +}; + +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/Quantization/HybridHierarchyLinearQuantization.hpp b/include/mgard-x/Quantization/HybridHierarchyLinearQuantization.hpp index dc692309b..2a9ba8af6 100644 --- a/include/mgard-x/Quantization/HybridHierarchyLinearQuantization.hpp +++ b/include/mgard-x/Quantization/HybridHierarchyLinearQuantization.hpp @@ -26,11 +26,10 @@ class HybridHierarchyQuantizer public: HybridHierarchyQuantizer() : initialized(false) {} - HybridHierarchyQuantizer(Hierarchy &hierarchy, - Hierarchy &global_hierarchy, + HybridHierarchyQuantizer(HybridHierarchy &hierarchy, Config config) - : initialized(true), hierarchy(&hierarchy), - global_hierarchy(&global_hierarchy), config(config) { + : initialized(true), hierarchy(&hierarchy), config(config) { + hierarchy.check_config(config); this->L = config.num_local_refactoring_level; this->M = config.num_global_refactoring_level; @@ -39,12 +38,11 @@ class HybridHierarchyQuantizer } } - void Adapt(Hierarchy &hierarchy, - Hierarchy &global_hierarchy, Config config, + void Adapt(HybridHierarchy &hierarchy, Config config, int queue_idx) { + hierarchy.check_config(config); this->initialized = true; this->hierarchy = &hierarchy; - this->global_hierarchy = &global_hierarchy; this->config = config; this->L = config.num_local_refactoring_level; this->M = config.num_global_refactoring_level; @@ -63,7 +61,10 @@ class HybridHierarchyQuantizer } if (this->M > 0) { - global_quantizer.Adapt(global_hierarchy, config, queue_idx); + global_hierarchy = &hierarchy.global_hierarchy(); + global_quantizer.Adapt(*global_hierarchy, config, queue_idx); + } else { + global_hierarchy = nullptr; } } @@ -570,21 +571,9 @@ class HybridHierarchyQuantizer return; } - // Get original shape from hierarchy - std::vector coarse_shape = - hierarchy->level_shape(hierarchy->l_target()); - for (SIZE l = 0; l < this->L; ++l) { - std::vector fine_shape(D); - for (DIM d = 0; d < D; ++d) { - // Round up to multiple of 8 for fine shape - coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 8; - fine_shape[d] = coarse_shape[d]; - // Compute next coarse shape (8->5 mapping) - coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 5; - } - fine_shapes.push_back(fine_shape); - coarse_shapes.push_back(coarse_shape); + fine_shapes.push_back(hierarchy->local_fine_shape(l)); + coarse_shapes.push_back(hierarchy->local_coarse_shape(l)); } } @@ -807,8 +796,9 @@ class HybridHierarchyQuantizer SIZE L; // Number of local levels SIZE M; // Number of global levels - Hierarchy *hierarchy; - Hierarchy *global_hierarchy; + HybridHierarchy *hierarchy; + // hierarchy->global_hierarchy() when M > 0. + Hierarchy *global_hierarchy = nullptr; Config config; bool orthogonal_projection = true; diff --git a/include/mgard-x/Quantization/LocalQuantization.hpp b/include/mgard-x/Quantization/LocalQuantization.hpp index 8dd97e32d..69afaad07 100644 --- a/include/mgard-x/Quantization/LocalQuantization.hpp +++ b/include/mgard-x/Quantization/LocalQuantization.hpp @@ -1,6 +1,7 @@ #ifndef MGARD_X_LOCAL_QUANTIZATION_TEMPLATE #define MGARD_X_LOCAL_QUANTIZATION_TEMPLATE +#include "../Hierarchy/HybridHierarchy.h" #include "../RuntimeX/RuntimeX.h" #include "QuantizationInterface.hpp" @@ -292,8 +293,9 @@ template class LocalQuantizer : public QuantizationInterface { public: LocalQuantizer() : initialized(false) {} - LocalQuantizer(Hierarchy &hierarchy, Config config) + LocalQuantizer(HybridHierarchy &hierarchy, Config config) : initialized(true), hierarchy(&hierarchy), config(config) { + hierarchy.check_config(config); this->L = config.num_local_refactoring_level; this->M = config.num_global_refactoring_level; compute_local_ranges(); @@ -301,8 +303,9 @@ class LocalQuantizer : public QuantizationInterface { } // Add logic to determine if roi or not - void Adapt(Hierarchy &hierarchy, Config config, + void Adapt(HybridHierarchy &hierarchy, Config config, int queue_idx) { + hierarchy.check_config(config); this->initialized = true; this->hierarchy = &hierarchy; this->config = config; @@ -322,7 +325,7 @@ class LocalQuantizer : public QuantizationInterface { } void compute_local_ranges() { - coarse_shape = hierarchy->level_shape(hierarchy->l_target()); + coarse_shape = hierarchy->input_shape(); fine_num_elems.clear(); coarse_num_elems.clear(); @@ -332,12 +335,10 @@ class LocalQuantizer : public QuantizationInterface { for (int l = 0; l < this->L; ++l) { SIZE last_level_size = 1, curr_level_size = 1; - std::vector fine_shape(D); + std::vector fine_shape = hierarchy->local_fine_shape(l); + coarse_shape = hierarchy->local_coarse_shape(l); for (DIM d = 0; d < D; ++d) { - coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 8; - fine_shape[d] = coarse_shape[d]; - last_level_size *= coarse_shape[d]; - coarse_shape[d] = ((coarse_shape[d] - 1) / 8 + 1) * 5; + last_level_size *= fine_shape[d]; curr_level_size *= coarse_shape[d]; } fine_num_elems.push_back(last_level_size); @@ -760,7 +761,7 @@ class LocalQuantizer : public QuantizationInterface { bool initialized; SIZE L; SIZE M; - Hierarchy *hierarchy; + HybridHierarchy *hierarchy; Config config; bool orthogonal_projection = true; diff --git a/tests/mgard-x/src/test_HybridHierarchy.cpp.in b/tests/mgard-x/src/test_HybridHierarchy.cpp.in index b3ba6d439..3d5bc9525 100644 --- a/tests/mgard-x/src/test_HybridHierarchy.cpp.in +++ b/tests/mgard-x/src/test_HybridHierarchy.cpp.in @@ -65,7 +65,7 @@ void run_block_local_roundtrip(std::vector shape, int num_local_levels, Config config; config.num_local_refactoring_level = num_local_levels; - Hierarchy hierarchy(shape, config); + HybridHierarchy hierarchy(shape, config); data_refactoring::BlockLocalHierarchyDataRefactor refactor; refactor.Adapt(hierarchy, config, 0); @@ -112,7 +112,7 @@ void run_hybrid_refactor_roundtrip(std::vector shape, Config config; config.num_local_refactoring_level = num_local_levels; config.num_global_refactoring_level = num_global_levels; - Hierarchy hierarchy(shape, config); + HybridHierarchy hierarchy(shape, config); data_refactoring::HybridHierarchyDataRefactor refactor; refactor.Adapt(hierarchy, config, 0); @@ -168,7 +168,7 @@ void run_hybrid_compressor_test( // Separate compressor and decompressor objects so the round trip only // communicates through the serialized bytes (and the norm, as it would be // carried in metadata in real usage). - Hierarchy hierarchy(shape, config); + HybridHierarchy hierarchy(shape, config); HybridHierarchyCompressor compressor; compressor.Adapt(hierarchy, config, 0); HybridHierarchyCompressor decompressor; @@ -358,7 +358,7 @@ void run_block_local_multilinear_test(std::vector shape) { Config config; config.num_local_refactoring_level = 1; - Hierarchy hierarchy(shape, config); + HybridHierarchy hierarchy(shape, config); data_refactoring::BlockLocalHierarchyDataRefactor refactor; refactor.Adapt(hierarchy, config, 0); @@ -707,7 +707,7 @@ TEMPLATE_TEST_CASE( config.num_local_refactoring_level = 1; config.num_global_refactoring_level = 0; config.projection_mode = compression_projection_mode_type::Hierarchical; - Hierarchy hierarchy(shape, config); + HybridHierarchy hierarchy(shape, config); HybridHierarchyCompressor compressor; compressor.Adapt(hierarchy, config, 0); DeviceRuntime::SyncQueue(0); @@ -747,7 +747,7 @@ static std::vector decomposed_to_host(Array<1, T, DevType> &array, SIZE n) { template static void adapt_refactor( data_refactoring::HybridHierarchyDataRefactor<3, T, DevType> &refactor, - Hierarchy<3, T, DevType> &hierarchy, Config config) { + HybridHierarchy<3, T, DevType> &hierarchy, Config config) { refactor.Adapt(hierarchy, config, 0); DeviceRuntime::SyncQueue(0); } @@ -772,7 +772,7 @@ void run_hybrid_stage_split_test(std::vector shape, int num_local_levels, Config config; config.num_local_refactoring_level = num_local_levels; config.num_global_refactoring_level = num_global_levels; - Hierarchy hierarchy(shape, config); + HybridHierarchy hierarchy(shape, config); data_refactoring::HybridHierarchyDataRefactor monolithic; data_refactoring::HybridHierarchyDataRefactor staged; @@ -857,7 +857,7 @@ void run_hybrid_standalone_recompose_test(std::vector shape, Config config; config.num_local_refactoring_level = num_local_levels; config.num_global_refactoring_level = num_global_levels; - Hierarchy hierarchy(shape, config); + HybridHierarchy hierarchy(shape, config); data_refactoring::HybridHierarchyDataRefactor compressor_side; adapt_refactor(compressor_side, hierarchy, config); @@ -870,7 +870,7 @@ void run_hybrid_standalone_recompose_test(std::vector shape, DeviceRuntime::SyncQueue(0); // Hand the decomposed buffer to a refactor with no decomposition history. - Hierarchy decompress_hierarchy(shape, config); + HybridHierarchy decompress_hierarchy(shape, config); data_refactoring::HybridHierarchyDataRefactor decompressor_side; adapt_refactor(decompressor_side, decompress_hierarchy, config); @@ -901,16 +901,16 @@ void run_hybrid_shape_contract_test(std::vector shape, Config config; config.num_local_refactoring_level = num_local_levels; config.num_global_refactoring_level = num_global_levels; - Hierarchy hierarchy(shape, config); + HybridHierarchy hierarchy(shape, config); data_refactoring::HybridHierarchyDataRefactor refactor; adapt_refactor(refactor, hierarchy, config); std::vector local_coarsest = refactor.local_refactor.coarse_shapes[num_local_levels - 1]; + Hierarchy &global_hierarchy = hierarchy.global_hierarchy(); std::vector global_finest = - refactor.global_hierarchy.level_shape( - refactor.global_hierarchy.l_target()); + global_hierarchy.level_shape(global_hierarchy.l_target()); REQUIRE(global_finest.size() == local_coarsest.size()); for (DIM d = 0; d < D; d++) { @@ -1000,7 +1000,7 @@ TEST_CASE("Hybrid refactor rejects having no levels at all", Config config; config.num_local_refactoring_level = 1; - Hierarchy hierarchy(shape, config); + HybridHierarchy hierarchy(shape, config); Config empty_config; empty_config.num_local_refactoring_level = 0; @@ -1368,7 +1368,7 @@ void run_roi_pipeline_test(std::vector shape, int num_local_levels, // up only in a separate decompressor process, and an in-process check // caught it or not depending on which allocation it happened to get -- // it caught it for double and missed it for float. - Hierarchy hierarchy(shape, config); + HybridHierarchy hierarchy(shape, config); HybridHierarchyCompressor compressor; compressor.Adapt(hierarchy, config, 0); HybridHierarchyCompressor decompressor; @@ -1476,7 +1476,7 @@ TEMPLATE_TEST_CASE("Hybrid refuses an s-norm bound it cannot bound", config.decomposition = decomposition_type::Hybrid; config.num_local_refactoring_level = L; config.num_global_refactoring_level = M; - Hierarchy<3, TestType, TestDevice> hierarchy(shape, config); + HybridHierarchy<3, TestType, TestDevice> hierarchy(shape, config); HybridHierarchyCompressor<3, TestType, TestDevice> compressor; compressor.Adapt(hierarchy, config, 0); DeviceRuntime::SyncQueue(0); diff --git a/tests/mgard-x/src/test_KernelFusionSwitch.cpp.in b/tests/mgard-x/src/test_KernelFusionSwitch.cpp.in index 2fbb14af8..9af223f0b 100644 --- a/tests/mgard-x/src/test_KernelFusionSwitch.cpp.in +++ b/tests/mgard-x/src/test_KernelFusionSwitch.cpp.in @@ -65,11 +65,11 @@ void round_trip(std::vector shape, int num_local_levels, hybrid_config(num_local_levels, num_global_levels); decompress_config.fuse_dequantize_recompose = fuse_decompress; - Hierarchy compress_hierarchy(shape, compress_config); + HybridHierarchy compress_hierarchy(shape, compress_config); HybridHierarchyCompressor compressor; compressor.Adapt(compress_hierarchy, compress_config, 0); - Hierarchy decompress_hierarchy(shape, decompress_config); + HybridHierarchy decompress_hierarchy(shape, decompress_config); HybridHierarchyCompressor decompressor; decompressor.Adapt(decompress_hierarchy, decompress_config, 0); DeviceRuntime::SyncQueue(0); From d4d3d121371db8202cd42233edbc1103fe542aae Mon Sep 17 00:00:00 2001 From: Jieyang Chen Date: Sun, 4 Oct 2026 15:29:02 -0700 Subject: [PATCH 05/11] mgard-x: merge max_larget_level into num_global_refactoring_level max_larget_level capped the levels of the multi-dimensional Hierarchy and num_global_refactoring_level set the levels of the hybrid global stage, which is itself a multi-dimensional Hierarchy capped the same way. Merge them into num_global_refactoring_level: it now caps every multilevel hierarchy (multi-dim, MDR-X, hybrid global stage). Both level counts are now SIZE; defaults are num_local_refactoring_level = 1 and num_global_refactoring_level = 3 (previously: all levels for multi-dim, 0 global levels for hybrid). Non-hybrid headers did not record the level count -- the reader rebuilt the hierarchy from the shape -- so a non-default cap could not be decompressed. Record it for every compressed and MDR-X stream in a new optional FunctionDecomposition.num_global_refactoring_levels. On read it sets the Config; streams without it (everything written before) were decomposed with all levels and are read that way. Hybrid streams still take L/M from their HybridHierarchy message. The mgard-x CLI -ll/-gl now default to the Config defaults instead of hard-coded 1/0, and -gl also applies to multi-dim compression. Behavior change: default compression now uses at most 3 global levels. Under rel 1e-3, multi-dim L-inf CR rises (1D 2.89 -> 3.44, 3D 2.57 -> 2.85); multi-dim s = 0 in 1D drops (4.08 -> 3.65); all bounds hold. Hybrid with an s-norm bound now warns that it falls back from 3 global levels to 0 by default. Files from the previous build (multi-dim and hybrid, 1D-4D, float/double, SERIAL and CUDA) decompress byte-identically, and new compression with matching levels reproduces the old output; old MDR-X streams reconstruct identically. SERIAL and CUDA unit tests pass. Co-Authored-By: Claude Opus 5.5 --- doc/MGARD-X.md | 7 ++-- examples/mgard-x/MDR-X/CUDA/refactor.cu | 2 +- examples/mgard-x/MDR-X/HIP/refactor.cpp | 2 +- examples/mgard-x/MDR-X/SERIAL/refactor.cpp | 2 +- .../CompressionHighLevel.hpp | 7 +++- .../HybridHierarchyCompressor.hpp | 4 +- include/mgard-x/Config/Config.h | 9 +++-- .../HybridHierarchyDataRefactor.hpp | 2 +- include/mgard-x/Hierarchy/Hierarchy.hpp | 8 ++-- include/mgard-x/Hierarchy/HybridHierarchy.h | 11 ++---- include/mgard-x/MDRHighLevel/MDRHighLevel.hpp | 9 +++++ include/mgard-x/Metadata/Metadata.hpp | 4 ++ src/mgard-x/Config/Config.cpp | 3 +- src/mgard-x/Executables/mgard-x-autotuner.cpp | 2 +- src/mgard-x/Executables/mgard-x.cpp | 39 ++++++++++--------- src/mgard-x/Metadata/Metadata.cpp | 22 ++++++++++- src/mgard.proto | 5 +++ tests/mgard-x/src/test_HybridMetadata.cpp.in | 36 +++++++++++++++-- 18 files changed, 120 insertions(+), 54 deletions(-) diff --git a/doc/MGARD-X.md b/doc/MGARD-X.md index dee9fa35f..5e3cb37da 100644 --- a/doc/MGARD-X.md +++ b/doc/MGARD-X.md @@ -128,8 +128,8 @@ An executable ```mgard-x``` will be built after building the MGARD-X library. To + ```zerorle-rans```: zero-run-length encoding followed by byte-alphabet rANS + ```symbol-rans```: symbol-alphabet rANS + ```-hh / --hybrid``` use the hybrid hierarchy (1D-3D only) - + ```-ll / --local-levels ``` number of block-local refactoring levels (default: 1) - + ```-gl / --global-levels ``` number of global refactoring levels (default: 0) + + ```-ll / --local-levels ``` number of hybrid block-local refactoring levels (default: 1) + + ```-gl / --global-levels ``` number of global refactoring levels, multi-dim or the hybrid global stage (default: 3) + ```-hp / --hybrid-projection ``` select the projection mode used by the hybrid hierarchy (default: orthogonal) + ```orthogonal```: retain the L2 projection (default) + ```hierarchical```: remove the L2 projection (L∞ only) @@ -177,13 +177,12 @@ An executable ```mgard-x``` will be built after building the MGARD-X library. To + ```mgard_x::decomposition_type::SingleDim```: 1D-at-a-time decomposition + ```mgard_x::decomposition_type::Hybrid```: hybrid hierarchy with block-local and global decomposition (1D-3D only; high-level APIs only) + ```num_local_refactoring_level```: controls the number of block-local refactoring levels in the hybrid hierarchy (***Default: 1***) - + ```num_global_refactoring_level```: controls the number of global refactoring levels in the hybrid hierarchy (***Default: 0***) + + ```num_global_refactoring_level```: controls the max number of global refactoring levels: of the multi-dimensional hierarchy, and of the global stage of the hybrid hierarchy. Recorded in the compressed header, so decompression needs no setting (***Default: 3***) + ```hybrid_projection_mode```: controls the projection mode used by the hybrid hierarchy: + ```mgard_x::hybrid_projection_mode_type::Orthogonal```: retain the L2 projection (***Default***) + ```mgard_x::hybrid_projection_mode_type::Hierarchical```: remove the L2 projection (L∞ only) + ```fuse_decompose_quantize```: controls fused local decomposition and quantization for the hybrid hierarchy (***Default: true***) + ```fuse_dequantize_recompose```: controls fused local dequantization and recomposition for the hybrid hierarchy (***Default: true***) - + ```max_larget_level```: controls max level of multi-level decomposition (***Default: 0 (no limit)***) + ```max_memory_footprint```(for high-level APIs only): controls maximum memory footprint in bytes (***Default: inf (no limit)***) + ```adjust_shape```(for high-level APIs only): controls whether or not to enable shape adjustment for better decomposition performance (***Default: false***) ## Using high-level APIs diff --git a/examples/mgard-x/MDR-X/CUDA/refactor.cu b/examples/mgard-x/MDR-X/CUDA/refactor.cu index d946c5367..956bd9f33 100644 --- a/examples/mgard-x/MDR-X/CUDA/refactor.cu +++ b/examples/mgard-x/MDR-X/CUDA/refactor.cu @@ -166,7 +166,7 @@ int main(int argc, char **argv) { mgard_x::log::level |= mgard_x::log::INFO; mgard_x::Config config; - config.max_larget_level = target_level; + config.num_global_refactoring_level = target_level; mgard_x::Hierarchy hierarchy(dims, config); test(filename, num_bitplanes, hierarchy, metadata_file, diff --git a/examples/mgard-x/MDR-X/HIP/refactor.cpp b/examples/mgard-x/MDR-X/HIP/refactor.cpp index 2ac338f75..7f4bcd97d 100644 --- a/examples/mgard-x/MDR-X/HIP/refactor.cpp +++ b/examples/mgard-x/MDR-X/HIP/refactor.cpp @@ -153,7 +153,7 @@ int main(int argc, char **argv) { mgard_x::log::level |= mgard_x::log::INFO; mgard_x::Config config; - config.max_larget_level = target_level; + config.num_global_refactoring_level = target_level; mgard_x::Hierarchy hierarchy(dims, config); test(filename, num_bitplanes, hierarchy, metadata_file, diff --git a/examples/mgard-x/MDR-X/SERIAL/refactor.cpp b/examples/mgard-x/MDR-X/SERIAL/refactor.cpp index 63a828f63..839300bbe 100644 --- a/examples/mgard-x/MDR-X/SERIAL/refactor.cpp +++ b/examples/mgard-x/MDR-X/SERIAL/refactor.cpp @@ -153,7 +153,7 @@ int main(int argc, char **argv) { mgard_x::log::level |= mgard_x::log::INFO; mgard_x::Config config; - config.max_larget_level = target_level; + config.num_global_refactoring_level = target_level; mgard_x::Hierarchy hierarchy(dims, config); test(filename, num_bitplanes, hierarchy, metadata_file, diff --git a/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp b/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp index 9ea2438b2..97b96508a 100644 --- a/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp +++ b/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp @@ -239,12 +239,15 @@ general_compress_pipeline(std::vector shape, T tol, T s, // FillForCompression does not carry the BlockDelta block size; set it here so // it is persisted in (and restored from) the metadata header. m.block_delta_block_size = config.block_delta_block_size; + // The decompressor rebuilds the hierarchy, so it needs the level cap. + m.has_num_global_refactoring_level = true; + m.num_global_refactoring_level = config.num_global_refactoring_level; // Same for the hybrid (BlockMGARD) parameters: the decompressor cannot // reconstruct the buffer layout or the per-block quantization steps without // them. if (config.decomposition == decomposition_type::Hybrid) { - m.hybrid_num_local_levels = (uint64_t)config.num_local_refactoring_level; - m.hybrid_num_global_levels = (uint64_t)config.num_global_refactoring_level; + m.hybrid_num_local_levels = config.num_local_refactoring_level; + m.hybrid_num_global_levels = config.num_global_refactoring_level; m.hybrid_local_block_size = MGARDX_HYBRID_LOCAL_BLOCK_SIZE; m.hybrid_projection_mode = config.projection_mode; m.hybrid_enable_roi = config.enable_roi; diff --git a/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.hpp b/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.hpp index de7b7b043..c71a584c5 100644 --- a/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.hpp +++ b/include/mgard-x/CompressionLowLevel/HybridHierarchyCompressor.hpp @@ -427,13 +427,13 @@ void HybridHierarchyCompressor::Decompress( template SIZE HybridHierarchyCompressor::calculate_padded_size( std::vector shape, Config config) { - int L = config.num_local_refactoring_level; + SIZE L = config.num_local_refactoring_level; SIZE total_num_elems_1D = 1; if (L > 0) { std::vector> fine_shapes, coarse_shapes; HybridHierarchy::compute_local_shapes( shape, L, fine_shapes, coarse_shapes); - for (int l = 0; l < L; l++) { + for (SIZE l = 0; l < L; l++) { SIZE last_level_size = 1, curr_level_size = 1; for (DIM d = 0; d < D; d++) { last_level_size *= fine_shapes[l][d]; diff --git a/include/mgard-x/Config/Config.h b/include/mgard-x/Config/Config.h index 113c3dd0e..c56b8e462 100644 --- a/include/mgard-x/Config/Config.h +++ b/include/mgard-x/Config/Config.h @@ -24,7 +24,6 @@ struct Config { enum lossless_type lossless; int log_level; bool auto_pin_host_buffers; - SIZE max_larget_level; SIZE max_memory_footprint; SIZE total_num_bitplanes; SIZE block_size; @@ -33,8 +32,12 @@ struct Config { bool mdr_adaptive_resolution; bool adjust_shape; bool compress_with_dryrun; - int num_local_refactoring_level; - int num_global_refactoring_level; + // Block-local levels of the hybrid hierarchy (L). Default 1. + SIZE num_local_refactoring_level; + // Global refactoring levels (M): caps the levels of every multilevel + // hierarchy -- the plain multi-dimensional one and the global stage of the + // hybrid one. Recorded in the file header. Default 3. + SIZE num_global_refactoring_level; bool auto_cache_release; cpu_parallelization_mode cpu_mode; bool mdr_qoi_mode; diff --git a/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactor.hpp b/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactor.hpp index 0a685e552..fbeb989ec 100644 --- a/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactor.hpp +++ b/include/mgard-x/DataRefactoring/HybridHierarchyDataRefactor.hpp @@ -72,7 +72,7 @@ class HybridHierarchyDataRefactor if (M > 0) { // Calculate Coarest shape from local std::vector coarest_shape = shape; - for (int l = 0; l < config.num_local_refactoring_level; l++) { + for (SIZE l = 0; l < config.num_local_refactoring_level; l++) { for (DIM d = 0; d < D; d++) { coarest_shape[d] = ((coarest_shape[d] - 1) / 8 + 1) * 5; } diff --git a/include/mgard-x/Hierarchy/Hierarchy.hpp b/include/mgard-x/Hierarchy/Hierarchy.hpp index 06610fa50..217461a7b 100644 --- a/include/mgard-x/Hierarchy/Hierarchy.hpp +++ b/include/mgard-x/Hierarchy/Hierarchy.hpp @@ -270,7 +270,7 @@ void Hierarchy::calc_volume(SIZE dof, T *dist, T *volume, template void Hierarchy::init(std::vector shape, std::vector coords, - SIZE max_larget_level) { + SIZE max_level) { this->shape = shape; @@ -292,7 +292,7 @@ void Hierarchy::init(std::vector shape, } _l_target = nlevel - 1; - _l_target = std::min(_l_target, max_larget_level); + _l_target = std::min(_l_target, max_level); for (int l = 0; l < _l_target + 1; l++) { std::vector curr_level_shape(D); @@ -877,7 +877,7 @@ Hierarchy::Hierarchy(std::vector shape, Config config) { dstype = data_structure_type::Cartesian_Grid_Uniform; std::vector coords = create_uniform_coords(shape, config.normalize_coordinates); - init(shape, coords, config.max_larget_level); + init(shape, coords, config.num_global_refactoring_level); assert(uniform_coords_created); assert(coords.size() == D); for (int d = 0; d < D; d++) @@ -898,7 +898,7 @@ Hierarchy::Hierarchy(std::vector shape, } dstype = data_structure_type::Cartesian_Grid_Non_Uniform; - init(shape, coords, config.max_larget_level); + init(shape, coords, config.num_global_refactoring_level); } template diff --git a/include/mgard-x/Hierarchy/HybridHierarchy.h b/include/mgard-x/Hierarchy/HybridHierarchy.h index c1eaab1c8..7447331d1 100644 --- a/include/mgard-x/Hierarchy/HybridHierarchy.h +++ b/include/mgard-x/Hierarchy/HybridHierarchy.h @@ -72,13 +72,13 @@ class HybridHierarchy : public Hierarchy { // levels. static Config global_config(SIZE M) { Config config; - config.max_larget_level = M; + config.num_global_refactoring_level = M; return config; } bool matches_config(Config config) { - return L == (SIZE)config.num_local_refactoring_level && - M == (SIZE)config.num_global_refactoring_level; + return L == config.num_local_refactoring_level && + M == config.num_global_refactoring_level; } // The hybrid components take their level counts from the config and their @@ -144,11 +144,6 @@ class HybridHierarchy : public Hierarchy { private: void init(std::vector shape, Config config, data_structure_type dstype) { - if (config.num_local_refactoring_level < 0 || - config.num_global_refactoring_level < 0) { - throw ProcessingException( - "HybridHierarchy: level counts cannot be negative."); - } L = config.num_local_refactoring_level; M = config.num_global_refactoring_level; if (L == 0 && M == 0) { diff --git a/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp b/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp index 3a45cab19..73446a0be 100644 --- a/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp +++ b/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp @@ -54,6 +54,11 @@ void load(Config &config, Metadata &metadata) { config.huff_dict_size = metadata.huff_dict_size; config.huff_block_size = metadata.huff_block_size; config.total_num_bitplanes = metadata.number_bitplanes; + // Streams from before the level cap was recorded used all levels. + config.num_global_refactoring_level = + metadata.has_num_global_refactoring_level + ? (SIZE)metadata.num_global_refactoring_level + : std::numeric_limits::max(); } template @@ -145,6 +150,10 @@ void MDRefactor(std::vector shape, const void *original_data, coords); } + // The reconstructor rebuilds the hierarchy, so it needs the level cap. + m.has_num_global_refactoring_level = true; + m.num_global_refactoring_level = config.num_global_refactoring_level; + uint32_t metadata_size; Byte *metadata = m.Serialize(metadata_size); refactored_metadata.header.resize(metadata_size); diff --git a/include/mgard-x/Metadata/Metadata.hpp b/include/mgard-x/Metadata/Metadata.hpp index 5549156a5..28791f947 100644 --- a/include/mgard-x/Metadata/Metadata.hpp +++ b/include/mgard-x/Metadata/Metadata.hpp @@ -36,6 +36,10 @@ struct MetadataBase { enum decomposition_type decomposition; uint32_t l_target = 0; + // Config::num_global_refactoring_level the data was decomposed with. Not + // present in streams written before it was recorded (see InitializeConfig). + bool has_num_global_refactoring_level = false; + uint64_t num_global_refactoring_level = 0; // Hybrid (block-local + global) hierarchy, a.k.a. BlockMGARD. Only // meaningful when `decomposition == decomposition_type::Hybrid`; these are diff --git a/src/mgard-x/Config/Config.cpp b/src/mgard-x/Config/Config.cpp index 37901afe6..0bd5a3a01 100644 --- a/src/mgard-x/Config/Config.cpp +++ b/src/mgard-x/Config/Config.cpp @@ -45,7 +45,6 @@ Config::Config() { normalize_coordinates = true; lossless = lossless_type::Huffman; log_level = log::ERR; - max_larget_level = std::numeric_limits::max(); // no limit auto_pin_host_buffers = true; max_memory_footprint = std::numeric_limits::max(); // no limit total_num_bitplanes = 32; @@ -56,7 +55,7 @@ Config::Config() { adjust_shape = false; compress_with_dryrun = false; num_local_refactoring_level = 1; - num_global_refactoring_level = 0; + num_global_refactoring_level = 3; enable_roi = false; projection_mode = compression_projection_mode_type::Auto; auto_cache_release = false; diff --git a/src/mgard-x/Executables/mgard-x-autotuner.cpp b/src/mgard-x/Executables/mgard-x-autotuner.cpp index 7d7d98f11..aea7bfc8a 100644 --- a/src/mgard-x/Executables/mgard-x-autotuner.cpp +++ b/src/mgard-x/Executables/mgard-x-autotuner.cpp @@ -35,7 +35,7 @@ int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, mgard_x::Config config; config.dev_type = dev_type; config.lossless = mgard_x::lossless_type::Huffman; - config.max_larget_level = 0; + config.num_global_refactoring_level = 0; enum mgard_x::error_bound_type mode = mgard_x::error_bound_type::REL; double tol = 0.1; diff --git a/src/mgard-x/Executables/mgard-x.cpp b/src/mgard-x/Executables/mgard-x.cpp index 891ce94df..384cc7835 100644 --- a/src/mgard-x/Executables/mgard-x.cpp +++ b/src/mgard-x/Executables/mgard-x.cpp @@ -47,8 +47,9 @@ void print_usage_message(std::string error) { \t\t -l / --lossless : lossless compression\n\ \t\t -d / --device : device type\n\ \t\t (optional) -hh / --hybrid: use hybrid (block-local + global) hierarchy\n\ -\t\t (optional) -ll / --local-levels : number of local refactoring levels (default: 1)\n\ -\t\t (optional) -gl / --global-levels : number of global refactoring levels (default: 0)\n\ +\t\t (optional) -ll / --local-levels : number of hybrid block-local refactoring levels (default: 1)\n\ +\t\t (optional) -gl / --global-levels : number of global refactoring levels,\n\ +\t\t\t multi-dim or the hybrid global stage (default: 3)\n\ \t\t (optional) -pm / --projection-mode :\n\ \t\t\t transform basis, plain or hybrid (default: auto -- hierarchical\n\ \t\t\t under an L-inf bound, orthogonal otherwise; hierarchical is L-inf only)\n\ @@ -560,8 +561,10 @@ int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, enum mgard_x::error_bound_type mode, std::string lossless, std::string domain_decomposition, mgard_x::SIZE block_size, enum mgard_x::device_type dev_type, int verbose, - mgard_x::SIZE max_memory_footprint, int num_local_levels, - int num_global_levels, bool use_hybrid, bool warm_up, + mgard_x::SIZE max_memory_footprint, + mgard_x::SIZE num_local_levels, + mgard_x::SIZE num_global_levels, bool use_hybrid, + bool warm_up, bool kernel_fusion, mgard_x::compression_projection_mode_type projection_mode) { mgard_x::Config config; @@ -591,8 +594,6 @@ int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, exit(-1); } - // config.max_larget_level = 1; - // config.compressor = mgard_x::compressor_type::ZFP; if (domain_decomposition == "block") { @@ -774,9 +775,9 @@ int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, // kept for debugging a file whose header disagrees with the data. struct DecompressOverrides { bool has_local_levels = false; - int num_local_levels = 0; + mgard_x::SIZE num_local_levels = 0; bool has_global_levels = false; - int num_global_levels = 0; + mgard_x::SIZE num_global_levels = 0; bool has_roi = false; bool enable_roi = false; // Also used, independently of any override, as the reference map for the @@ -931,10 +932,11 @@ bool try_compression(int argc, char *argv[]) { argc, argv, "Max memory", "-m", "--max-memory"); } - int num_local_levels = 1; // default value + mgard_x::SIZE num_local_levels = + mgard_x::Config().num_local_refactoring_level; if (has_arg(argc, argv, "-ll", "--local-levels")) { - num_local_levels = - get_arg(argc, argv, "Local levels", "-ll", "--local-levels"); + num_local_levels = get_arg(argc, argv, "Local levels", "-ll", + "--local-levels"); } bool use_hybrid = has_arg(argc, argv, "-hh", "--hybrid"); @@ -961,10 +963,11 @@ bool try_compression(int argc, char *argv[]) { // Fusion is on by default; the flag selects the separate-pass path. bool kernel_fusion = !has_arg(argc, argv, "-nkf", "--no-kernel-fusion"); - int num_global_levels = 0; // default value + mgard_x::SIZE num_global_levels = + mgard_x::Config().num_global_refactoring_level; if (has_arg(argc, argv, "-gl", "--global-levels")) { - num_global_levels = - get_arg(argc, argv, "Global levels", "-gl", "--global-levels"); + num_global_levels = get_arg(argc, argv, "Global levels", + "-gl", "--global-levels"); } std::string domain_decomposition = "max-dim"; @@ -1030,13 +1033,13 @@ bool try_decompression(int argc, char *argv[]) { } if (has_arg(argc, argv, "-ll", "--local-levels")) { overrides.has_local_levels = true; - overrides.num_local_levels = - get_arg(argc, argv, "Local levels", "-ll", "--local-levels"); + overrides.num_local_levels = get_arg( + argc, argv, "Local levels", "-ll", "--local-levels"); } if (has_arg(argc, argv, "-gl", "--global-levels")) { overrides.has_global_levels = true; - overrides.num_global_levels = - get_arg(argc, argv, "Global levels", "-gl", "--global-levels"); + overrides.num_global_levels = get_arg( + argc, argv, "Global levels", "-gl", "--global-levels"); } // Optional: original data file for error verification std::string original_file; diff --git a/src/mgard-x/Metadata/Metadata.cpp b/src/mgard-x/Metadata/Metadata.cpp index 05f0afcfb..823f9fdbf 100644 --- a/src/mgard-x/Metadata/Metadata.cpp +++ b/src/mgard-x/Metadata/Metadata.cpp @@ -12,6 +12,8 @@ #include +#include + using mgard_x::SERIALIZED_TYPE; using SerializedIter = std::vector::const_iterator; @@ -251,6 +253,12 @@ void MetadataBase::InitializeConfig(Config &config) { if (block_delta_block_size != 0) { config.block_delta_block_size = block_delta_block_size; } + // The decomposition has to be rebuilt with the levels it was made with. + // Streams from before the level cap was recorded used all levels. + config.num_global_refactoring_level = + has_num_global_refactoring_level + ? (SIZE)num_global_refactoring_level + : std::numeric_limits::max(); // The hybrid parameters are only meaningful for a hybrid file, and // Deserialize refuses to produce a hybrid file without them, so a Hybrid // decomposition here always carries a full set. For non-hybrid files leave @@ -258,8 +266,8 @@ void MetadataBase::InitializeConfig(Config &config) { // with zeros would break a subsequent hybrid compression through the same // Config. if (decomposition == decomposition_type::Hybrid) { - config.num_local_refactoring_level = (int)hybrid_num_local_levels; - config.num_global_refactoring_level = (int)hybrid_num_global_levels; + config.num_local_refactoring_level = (SIZE)hybrid_num_local_levels; + config.num_global_refactoring_level = (SIZE)hybrid_num_global_levels; config.projection_mode = hybrid_projection_mode; config.enable_roi = hybrid_enable_roi; if (hybrid_enable_roi) { @@ -565,6 +573,10 @@ std::vector MetadataBase::Serialize() { // the level count) from the shape, and under domain decomposition there is // no single value to record. Written for format compatibility only. function_decomposition.set_l_target(l_target); + if (has_num_global_refactoring_level) { + function_decomposition.set_num_global_refactoring_levels( + num_global_refactoring_level); + } } { // Quantization @@ -896,6 +908,12 @@ void MetadataBase::Deserialize( "in MGARD-X."); } l_target = function_decomposition.l_target(); + has_num_global_refactoring_level = + function_decomposition.has_num_global_refactoring_levels(); + num_global_refactoring_level = + has_num_global_refactoring_level + ? function_decomposition.num_global_refactoring_levels() + : 0; } { // Quantization diff --git a/src/mgard.proto b/src/mgard.proto index 0a20ab806..07d97e769 100644 --- a/src/mgard.proto +++ b/src/mgard.proto @@ -150,6 +150,11 @@ message FunctionDecomposition { uint64 L_target = 3; // Required when `hierarchy == HYBRID_HIERARCHY`, absent otherwise. HybridHierarchy hybrid_hierarchy = 4; + // Cap on the number of global refactoring levels + // (Config::num_global_refactoring_level): of the multi-dimensional + // hierarchy, and of the global stage of a hybrid one. Absent from streams + // written before it was recorded, which were decomposed with all levels. + optional uint64 num_global_refactoring_levels = 5; } message Quantization { diff --git a/tests/mgard-x/src/test_HybridMetadata.cpp.in b/tests/mgard-x/src/test_HybridMetadata.cpp.in index ecb6f4924..d051e1def 100644 --- a/tests/mgard-x/src/test_HybridMetadata.cpp.in +++ b/tests/mgard-x/src/test_HybridMetadata.cpp.in @@ -6,6 +6,7 @@ #include "catch2/catch_test_macros.hpp" #include +#include #include #include "mgard-x/Config/Config.h" @@ -184,25 +185,52 @@ TEST_CASE("an irregular ROI map still round trips exactly", } } -TEST_CASE("a non-hybrid file leaves the caller's hybrid Config alone", +TEST_CASE("a non-hybrid file leaves the caller's local levels alone", "[HybridMetadata]") { std::vector shape = {64, 64, 64}; Metadata out = base_metadata(shape); out.decomposition = decomposition_type::MultiDim; + out.has_num_global_refactoring_level = true; + out.num_global_refactoring_level = 5; std::vector bytes = out.MetadataBase::Serialize(); Metadata in; in.MetadataBase::Deserialize(bytes); REQUIRE(in.decomposition == decomposition_type::MultiDim); + REQUIRE(in.has_num_global_refactoring_level); + REQUIRE(in.num_global_refactoring_level == 5); Config config; config.num_local_refactoring_level = 3; config.num_global_refactoring_level = 2; in.InitializeConfig(config); - // Untouched: overwriting these with zeros would break a later hybrid - // compression driven by the same Config. + // Untouched: the local levels only mean something to a hybrid file, and + // overwriting them would break a later hybrid compression driven by the + // same Config. REQUIRE(config.num_local_refactoring_level == 3); - REQUIRE(config.num_global_refactoring_level == 2); + // The global levels also cap the multi-dimensional hierarchy, so the + // decompressor has to rebuild it with the recorded value. + REQUIRE(config.num_global_refactoring_level == 5); +} + +TEST_CASE("a file without recorded global levels decomposes with all levels", + "[HybridMetadata]") { + // Streams written before the cap was recorded used every level. + std::vector shape = {64, 64, 64}; + Metadata out = base_metadata(shape); + out.decomposition = decomposition_type::MultiDim; + out.has_num_global_refactoring_level = false; + + std::vector bytes = out.MetadataBase::Serialize(); + Metadata in; + in.MetadataBase::Deserialize(bytes); + REQUIRE_FALSE(in.has_num_global_refactoring_level); + + Config config; + config.num_global_refactoring_level = 2; + in.InitializeConfig(config); + REQUIRE(config.num_global_refactoring_level == + std::numeric_limits::max()); } TEST_CASE("the quantized integer width is recorded and checked", From 8ac56b0455766701915362bd637f1887a34d0a7b Mon Sep 17 00:00:00 2001 From: Jieyang Chen Date: Sat, 10 Oct 2026 10:55:00 -0700 Subject: [PATCH 06/11] mgard-x: fix subdomain stream, MDR reconstruction and zstd decoding bugs - Serializer: Serialize aligns each value to sizeof(T) but Deserialize did not skip the padding. A subdomain stored uncompressed whose byte count is not a multiple of 8 (e.g. a 9x3x5x25 float block) shifted every later subdomain size field, and decompression failed or read garbage. Deserialize now aligns the same way. - MDR reconstruction sized level_signs to the raw per-level element count, while the bitplane decoder writes a sign for every slot of its last 32-value batch. The refactoring Resize already rounded up; the reconstruction Resize now does too (it takes the reconstructor for its BATCH_SIZE). The overrun corrupted the heap on SERIAL (free(): invalid pointer) and wrote out of bounds on CUDA whenever a later subdomain grew a level's buffer. - MDR reconstruction copied the last subdomain from abs((current_buffer - 1) % 3), which is 1 instead of 2 when current_buffer is 0: with a multiple of 3 subdomains the last one was written back from the wrong buffer and the error bound failed. - MDR refactor/reconstruct sized their reused buffers from subdomain 0; size them from the largest subdomain, as the compression pipelines do. - Zstd: read the stored size as size_t (it was truncated to 32 bits) and check ZSTD_decompress, so a corrupt stream fails instead of handing garbage to the Huffman decoder. - Compression output buffer: reserve room for non-uniform coordinates, which the header stores as doubles, and fail with OutputTooLargeFailure instead of overflowing when the header does not fit. - Metadata: delete[] for the coordinate copies allocated with new[]. - mgard-x CLI: check the status of decompress() in both paths. Co-Authored-By: Claude Opus 5.5 --- .../CompressionHighLevel.hpp | 20 +++++++++++++++ include/mgard-x/Lossless/Zstd.hpp | 17 ++++++++++--- .../MDRHighLevel/ReconstructPipeline.hpp | 25 +++++++++++-------- .../MDRHighLevel/ReconstructPipelineQoI.hpp | 9 ++++--- .../mgard-x/MDRHighLevel/RefactorPipeline.hpp | 11 +++++--- include/mgard-x/Metadata/Metadata.hpp | 4 +-- .../RuntimeX/DataStructures/MDRData.hpp | 12 ++++++--- .../mgard-x/RuntimeX/Utilities/Serializer.hpp | 5 +++- src/mgard-x/Executables/mgard-x.cpp | 15 ++++++++--- 9 files changed, 87 insertions(+), 31 deletions(-) diff --git a/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp b/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp index 97b96508a..801d2f4a9 100644 --- a/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp +++ b/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp @@ -193,6 +193,13 @@ general_compress_pipeline(std::vector shape, T tol, T s, size_t output_buffer_size; if (!output_pre_allocated) { output_buffer_size = total_num_elem * sizeof(T) + OUTPUT_SAFTY_OVERHEAD; + // The header stores non-uniform coordinates as doubles, which can exceed + // the safety margin (8 MB for a 1M-point 1D grid). + if (!uniform) { + for (DIM d = 0; d < D; d++) { + output_buffer_size += shape[d] * sizeof(double); + } + } if (MemoryManager::IsDevicePointer(original_data)) { DeviceRuntime::SelectDevice( MemoryManager::GetPointerDevice(original_data)); @@ -304,6 +311,19 @@ general_compress_pipeline(std::vector shape, T tol, T s, SERIALIZED_TYPE *serizalied_meta = m.Serialize(metadata_size); MemoryManager::Free(serizalied_meta); + if (metadata_size > output_buffer_size) { + log::err("output buffer of " + std::to_string(output_buffer_size) + + " bytes cannot hold the " + std::to_string(metadata_size) + + "-byte header."); + if (!input_previously_pinned && config.auto_pin_host_buffers) { + MemoryManager::HostUnregister((void *)original_data); + } + if (!output_previously_pinned && config.auto_pin_host_buffers) { + MemoryManager::HostUnregister((void *)compressed_data); + } + return compress_status_type::OutputTooLargeFailure; + } + SIZE byte_offset = metadata_size; Byte *compressed_subdomain_data = (Byte *)compressed_data + byte_offset; SIZE compressed_subdomain_size = output_buffer_size - metadata_size; diff --git a/include/mgard-x/Lossless/Zstd.hpp b/include/mgard-x/Lossless/Zstd.hpp index 34e5e729d..46037a75a 100644 --- a/include/mgard-x/Lossless/Zstd.hpp +++ b/include/mgard-x/Lossless/Zstd.hpp @@ -114,8 +114,9 @@ template class Zstd { queue_idx); DeviceRuntime::SyncQueue(queue_idx); - uint32_t actual_out_count = 0; - actual_out_count = *reinterpret_cast(in_data); + // Written as a size_t by Compress; reading it into a narrower type would + // truncate streams of 4 GiB or more. + size_t actual_out_count = *reinterpret_cast(in_data); // FIX: Only resize out_data if needed, do not touch in_data which contains // the compressed data we just copied. The original code called Resize() @@ -131,8 +132,16 @@ template class Zstd { } DeviceRuntime::SyncQueue(queue_idx); - ZSTD_decompress(out_data, actual_out_count, in_data + sizeof(size_t), - input_count - sizeof(size_t)); + // A corrupt or truncated stream must fail here: an unchecked failure + // leaves out_data as garbage that the next stage misreads (MGARD 1.6.0 + // crashed in the Huffman decoder this way). + size_t const decompressed_size = + ZSTD_decompress(out_data, actual_out_count, in_data + sizeof(size_t), + input_count - sizeof(size_t)); + CHECK_ZSTD(decompressed_size); + CHECK(decompressed_size == actual_out_count, + "Zstd decompressed %zu bytes, header says %zu", decompressed_size, + actual_out_count); output_data.resize({(SIZE)actual_out_count}); MemoryManager::Copy1D(output_data.data(), out_data, diff --git a/include/mgard-x/MDRHighLevel/ReconstructPipeline.hpp b/include/mgard-x/MDRHighLevel/ReconstructPipeline.hpp index 5e21d8d5c..ccca1b348 100644 --- a/include/mgard-x/MDRHighLevel/ReconstructPipeline.hpp +++ b/include/mgard-x/MDRHighLevel/ReconstructPipeline.hpp @@ -38,16 +38,19 @@ void reconstruct_pipeline( } } - HierarchyType &hierarchy = - Cache::cache.GetHierarchyCache(domain_decomposer.subdomain_shape(0)); + // Size the reused buffers for the largest subdomain: they are not grown + // when a later subdomain is bigger than the first. + std::vector largest_shape = + domain_decomposer.subdomain_shape(domain_decomposer.largest_subdomain()); + HierarchyType &hierarchy = Cache::cache.GetHierarchyCache(largest_shape); log::info("Adjust device buffers"); mdr_data[0].Resize(reconstructor, hierarchy, 0); mdr_data[1].Resize(reconstructor, hierarchy, 0); mdr_data[2].Resize(reconstructor, hierarchy, 0); - device_subdomain_buffer[0].resize(domain_decomposer.subdomain_shape(0), 0); - device_subdomain_buffer[1].resize(domain_decomposer.subdomain_shape(0), 0); - device_subdomain_buffer[2].resize(domain_decomposer.subdomain_shape(0), 0); + device_subdomain_buffer[0].resize(largest_shape, 0); + device_subdomain_buffer[1].resize(largest_shape, 0); + device_subdomain_buffer[2].resize(largest_shape, 0); Timer timer_series; if (timing_pipeline) @@ -55,8 +58,8 @@ void reconstruct_pipeline( // Prefetch the first subdomain int current_buffer = 0; int current_queue = 0; - mdr_data[current_buffer].Resize(refactored_metadata.metadata[0], - current_queue); + mdr_data[current_buffer].Resize( + reconstructor, refactored_metadata.metadata[0], current_queue); device_subdomain_buffer[current_buffer].resize( domain_decomposer.subdomain_shape(0), current_queue); mdr_data[current_buffer].CopyFromRefactoredData( @@ -86,7 +89,8 @@ void reconstruct_pipeline( // Prefetch the next subdomain next_subdomain_id = curr_subdomain_id + 1; mdr_data[next_buffer].Resize( - refactored_metadata.metadata[next_subdomain_id], next_queue); + reconstructor, refactored_metadata.metadata[next_subdomain_id], + next_queue); device_subdomain_buffer[next_buffer].resize( domain_decomposer.subdomain_shape(next_subdomain_id), next_queue); @@ -162,8 +166,9 @@ void reconstruct_pipeline( } // Copy the last subdomain - int previous_buffer = std::abs((current_buffer - 1) % 3); - int previous_queue = std::abs((current_queue - 1) % 3); + // (+ 3: in C++ (0 - 1) % 3 is -1, and abs(-1) is not the previous index.) + int previous_buffer = (current_buffer + 3 - 1) % 3; + int previous_queue = (current_queue + 3 - 1) % 3; SIZE prev_subdomain_id = domain_decomposer.num_subdomains() - 1; // Update level signs for future progressive reconstruction mdr_data[previous_buffer].CopyToRefactoredSigns( diff --git a/include/mgard-x/MDRHighLevel/ReconstructPipelineQoI.hpp b/include/mgard-x/MDRHighLevel/ReconstructPipelineQoI.hpp index ad56ed208..68979b98a 100644 --- a/include/mgard-x/MDRHighLevel/ReconstructPipelineQoI.hpp +++ b/include/mgard-x/MDRHighLevel/ReconstructPipelineQoI.hpp @@ -111,7 +111,7 @@ void reconstruct_pipeline_qoi( Cache::cache.InsertHierarchyCache( domain_decomposer.subdomain_hierarchy(id)); } - mdr_data[id].Resize(refactored_metadata.metadata[id], 0); + mdr_data[id].Resize(reconstructor, refactored_metadata.metadata[id], 0); device_subdomain_buffer[id].resize(domain_decomposer.subdomain_shape(id), 0); // Reset all signs to 0 for the initial QOI reconstruction @@ -120,8 +120,11 @@ void reconstruct_pipeline_qoi( } } - HierarchyType &hierarchy = - Cache::cache.GetHierarchyCache(domain_decomposer.subdomain_shape(0)); + // Size the reused buffers for the largest subdomain: they are not grown + // when a later subdomain is bigger than the first. + std::vector largest_shape = + domain_decomposer.subdomain_shape(domain_decomposer.largest_subdomain()); + HierarchyType &hierarchy = Cache::cache.GetHierarchyCache(largest_shape); reconstructor.Adapt(hierarchy, config, 0); int current_buffer = 0; diff --git a/include/mgard-x/MDRHighLevel/RefactorPipeline.hpp b/include/mgard-x/MDRHighLevel/RefactorPipeline.hpp index ae7de5a18..caa264fd5 100644 --- a/include/mgard-x/MDRHighLevel/RefactorPipeline.hpp +++ b/include/mgard-x/MDRHighLevel/RefactorPipeline.hpp @@ -43,13 +43,16 @@ void refactor_pipeline( } } - HierarchyType &hierarchy = - Cache::cache.GetHierarchyCache(domain_decomposer.subdomain_shape(0)); + // Size the reused buffers for the largest subdomain: they are not grown + // when a later subdomain is bigger than the first. + std::vector largest_shape = + domain_decomposer.subdomain_shape(domain_decomposer.largest_subdomain()); + HierarchyType &hierarchy = Cache::cache.GetHierarchyCache(largest_shape); log::info("Adjust device buffers"); refactor.Adapt(hierarchy, config, 0); - device_subdomain_buffer[0].resize(domain_decomposer.subdomain_shape(0), 0); - device_subdomain_buffer[1].resize(domain_decomposer.subdomain_shape(0), 0); + device_subdomain_buffer[0].resize(largest_shape, 0); + device_subdomain_buffer[1].resize(largest_shape, 0); mdr_data[0].Resize(refactor, hierarchy, 0); mdr_data[1].Resize(refactor, hierarchy, 0); DeviceRuntime::SyncDevice(); diff --git a/include/mgard-x/Metadata/Metadata.hpp b/include/mgard-x/Metadata/Metadata.hpp index 28791f947..1e34e3b46 100644 --- a/include/mgard-x/Metadata/Metadata.hpp +++ b/include/mgard-x/Metadata/Metadata.hpp @@ -168,7 +168,7 @@ template struct Metadata : MetadataBase { coord[i] = (double)coord_h[i]; } this->coords.push_back(coord); - delete coord_h; + delete[] coord_h; } this->dstype = data_structure_type::Cartesian_Grid_Non_Uniform; } @@ -232,7 +232,7 @@ template struct Metadata : MetadataBase { coord[i] = (double)coord_h[i]; } this->coords.push_back(coord); - delete coord_h; + delete[] coord_h; } this->dstype = data_structure_type::Cartesian_Grid_Non_Uniform; } diff --git a/include/mgard-x/RuntimeX/DataStructures/MDRData.hpp b/include/mgard-x/RuntimeX/DataStructures/MDRData.hpp index 1a60ed3e7..261c2f803 100644 --- a/include/mgard-x/RuntimeX/DataStructures/MDRData.hpp +++ b/include/mgard-x/RuntimeX/DataStructures/MDRData.hpp @@ -59,13 +59,19 @@ template class MDRData { } // Resize before reconstruction - void Resize(MDRMetadata &mdr_metadata, int queue_idx) { + template + void Resize(ReconstructorType &reconstructor, MDRMetadata &mdr_metadata, + int queue_idx) { compressed_bitplanes.resize(mdr_metadata.num_levels); level_signs.resize(mdr_metadata.num_levels); for (int level_idx = 0; level_idx < mdr_metadata.num_levels; level_idx++) { compressed_bitplanes[level_idx].resize(mdr_metadata.num_bitplanes); - level_signs[level_idx].resize({mdr_metadata.level_num_elems[level_idx]}, - queue_idx); + // Rounded up to the decoder's batch size, as in the refactoring Resize + // above: the decoder writes a sign for every slot of its last batch. + level_signs[level_idx].resize( + {round_up((SIZE)mdr_metadata.level_num_elems[level_idx], + ReconstructorType::BATCH_SIZE)}, + queue_idx); for (int bitplane_idx = 0; bitplane_idx < mdr_metadata.num_bitplanes; bitplane_idx++) { compressed_bitplanes[level_idx][bitplane_idx].resize( diff --git a/include/mgard-x/RuntimeX/Utilities/Serializer.hpp b/include/mgard-x/RuntimeX/Utilities/Serializer.hpp index 78f372df8..c08a4c3a8 100644 --- a/include/mgard-x/RuntimeX/Utilities/Serializer.hpp +++ b/include/mgard-x/RuntimeX/Utilities/Serializer.hpp @@ -61,7 +61,10 @@ template void Deserialize(Byte *serialize_ptr, T *&data_ptr, SIZE count, SIZE &byte_offset, bool zero_copy, int queue_idx) { using Mem = MemoryManager; - // align_byte_offset(byte_offset); + // Serialize aligns each value to sizeof(T); skip the same padding here, or + // a record whose byte count is not a multiple of 8 (e.g. a subdomain stored + // uncompressed) shifts every size field after it. + align_byte_offset(byte_offset); if (zero_copy) { data_ptr = (T *)(serialize_ptr + byte_offset); } else { diff --git a/src/mgard-x/Executables/mgard-x.cpp b/src/mgard-x/Executables/mgard-x.cpp index 384cc7835..54ba58802 100644 --- a/src/mgard-x/Executables/mgard-x.cpp +++ b/src/mgard-x/Executables/mgard-x.cpp @@ -746,8 +746,11 @@ int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, void *decompressed_data = malloc(original_size * sizeof(T)); mgard_x::pin_memory(decompressed_data, original_size * sizeof(T), config); - mgard_x::decompress(compressed_data, compressed_size, decompressed_data, - config, true); + ret = mgard_x::decompress(compressed_data, compressed_size, + decompressed_data, config, true); + if (ret != mgard_x::compress_status_type::Success) { + throw std::runtime_error("Decompression failed"); + } if (config.enable_roi) { print_statistics_roi(s, mode, shape, original_data, @@ -820,8 +823,12 @@ int launch_decompress( mgard_x::data_type dtype; void *decompressed_data; - mgard_x::decompress(compressed_data, compressed_size, decompressed_data, - shape, dtype, config, false); + mgard_x::compress_status_type ret = + mgard_x::decompress(compressed_data, compressed_size, decompressed_data, + shape, dtype, config, false); + if (ret != mgard_x::compress_status_type::Success) { + throw std::runtime_error("Decompression failed"); + } int elem_size = 0; if (dtype == mgard_x::data_type::Double) { From a5dc55e79b5d5026d64c92612eaa41f7e325ce9f Mon Sep 17 00:00:00 2001 From: Jieyang Chen Date: Sat, 10 Oct 2026 10:55:23 -0700 Subject: [PATCH 07/11] mgard-x: decompress MGARD 1.6.0 data through a separate compatibility pipeline Data written by MGARD 1.6.x differs from the current format in many places (hierarchy geometry and level volumes, Huffman layout, basis, defaults such as an unlimited level cap). Rather than branch on the version throughout the pipeline, add a complete copy of the 1.6.0 decompression path in its own tree and namespace (include/mgard-x-160, src/mgard-x-160, namespace mgard_x_160), kept faithful to 1.6.0. - Compression always writes the current format; only decompression is routed. DynamicAPI's decompress overloads and release_cache first read just the header versions (new MetadataBase::DeserializeVersion, which does not validate the rest of the header) and send streams written by 1.6.x or earlier (WrittenByV160OrEarlier) to mgard_x_160 through Bridge.cpp.in, which converts the Config and results between the two namespaces. The compatibility decoder starts from the 1.6.0 defaults and takes from the caller only the settings 1.6.0 took. - CMake option MGARD_ENABLE_DATA_COMPATIBILITY (default OFF; it roughly doubles the MGARD-X build). Without it, 1.6.0 data is reported with the new status DataCompatibilityNotEnabledFailure. - The bug fixes made to the current pipeline also apply to the compatibility one: MaxDim slab offsets along a dimension other than the slowest, zstd size truncation and unchecked decompression, and the Serialize/Deserialize alignment mismatch. Files that 1.6.0 itself could not decode because of these now decode. Verified against a build of MGARD 1.6.0: 61 CLI-written files (1D-5D, float/double, huffman/huffman-zstd, s = inf/0/1, rel/abs) decompress byte-identically on SERIAL and within 1.5e-6 of range on CUDA; 68 API-only files (reorder, non-uniform, SingleDim, MaxDim and Block decomposition, CPU lossless, double) and 60 zstd files likewise. Current data is unaffected by the routing, and an OFF build reports 1.6.0 data with the new status. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 10 + .../CompressionHighLevel/CPUPipelines.hpp | 535 ++ .../CompressionHighLevel.h | 85 + .../CompressionHighLevel.hpp | 1051 ++++ .../ErrorToleranceCalculator.hpp | 148 + .../CompressionHighLevel/GPUPipelines.hpp | 590 +++ .../CompressionHighLevel/ShapeAdjustment.hpp | 79 + .../CompressionLowLevel/Compressor.h | 123 + .../CompressionLowLevel/Compressor.hpp | 300 ++ .../CompressionLowLevel/CompressorCache.hpp | 153 + .../HybridHierarchyCompressor.h | 133 + .../HybridHierarchyCompressor.hpp | 465 ++ .../LossyCompressorInterface.hpp | 24 + .../CompressionLowLevel/NormCalculator.hpp | 84 + include/mgard-x-160/Config/Config.h | 71 + .../BlockLocalHierarchyDataRefactor.hpp | 484 ++ ...ockLocalHierarchyDataRefactorInterface.hpp | 17 + .../DataRefactoring/DataRefactor.hpp | 151 + .../DataRefactoring/DataRefactorInterface.hpp | 31 + .../HybridHierarchyDataRefactor.hpp | 253 + .../HybridHierarchyDataRefactorInterface.hpp | 28 + .../InCacheBlock/Autocorrelation8x8x8.hpp | 239 + .../InCacheBlock/DataRefactoring.h | 71 + .../InCacheBlock/DataRefactoring.hpp | 150 + .../InCacheBlock/Decompose8.hpp | 331 ++ .../InCacheBlock/Decompose8x8.hpp | 368 ++ .../InCacheBlock/Decompose8x8x8.hpp | 673 +++ .../InCacheBlock/IndexTable3x3x3.hpp | 371 ++ .../InCacheBlock/IndexTable5x5x5.hpp | 790 +++ .../InCacheBlock/IndexTable8x8x8.hpp | 2522 +++++++++ .../InCacheBlock/IndexTableLowDim.hpp | 373 ++ .../InCacheBlock/Recompose8.hpp | 310 ++ .../InCacheBlock/Recompose8x8.hpp | 345 ++ .../InCacheBlock/Recompose8x8x8.hpp | 663 +++ .../Coefficient/CalcCoefficients3D.hpp | 101 + .../Coefficient/CalcCoefficientsND.hpp | 239 + .../Coefficient/CalcCoefficientsPointers.hpp | 96 + .../Coefficient/CoefficientsRestore3D.hpp | 99 + .../Coefficient/CoefficientsRestoreND.hpp | 259 + .../MultiDimension/Coefficient/GPKFunctor.h | 29 + .../Coefficient/GridProcessingKernel.hpp | 4661 +++++++++++++++++ .../Coefficient/GridProcessingKernel3D.hpp | 2413 +++++++++ .../MultiDimension/CopyND/AddND.hpp | 38 + .../MultiDimension/CopyND/Copy3D.hpp | 49 + .../MultiDimension/CopyND/CopyND.hpp | 38 + .../CopyND/LevelwiseProcessingKernel.hpp | 376 ++ .../MultiDimension/CopyND/SubtractND.hpp | 38 + .../Correction/CalcCorrection3D.hpp | 193 + .../Correction/CalcCorrectionND.hpp | 269 + .../MultiDimension/Correction/IPKFunctor.h | 55 + .../Correction/IterativeProcessingKernel.hpp | 1993 +++++++ .../IterativeProcessingKernel3D.hpp | 1169 +++++ .../MultiDimension/Correction/LPKFunctor.h | 71 + .../Correction/LinearProcessingKernel.hpp | 1513 ++++++ .../Correction/LinearProcessingKernel3D.hpp | 1098 ++++ .../MultiDimension/DataRefactoring.h | 146 + .../MultiDimension/DataRefactoring.hpp | 325 ++ .../Coefficient/CalcCoefficients.hpp | 42 + .../Coefficient/CoefficientKernel.hpp | 219 + .../Coefficient/CoefficientsRestore.hpp | 42 + .../Correction/CalcCorrection.hpp | 102 + .../Correction/MassTransKernel.hpp | 191 + .../SingleDimension/DataRefactoring.h | 74 + .../SingleDimension/DataRefactoring.hpp | 197 + .../DomainDecomposer/DomainDecomposer.hpp | 855 +++ include/mgard-x-160/Hierarchy/Hierarchy.h | 129 + include/mgard-x-160/Hierarchy/Hierarchy.hpp | 874 ++++ .../mgard-x-160/Hierarchy/HybridHierarchy.h | 169 + .../Lossless/BlockDelta/BlockDelta.hpp | 494 ++ .../Lossless/BlockDelta/BlockDeltaFused.hpp | 204 + .../Lossless/BlockDelta/BlockDeltaKernels.hpp | 544 ++ include/mgard-x-160/Lossless/CPU.hpp | 283 + include/mgard-x-160/Lossless/LZ4/LZ4.hpp | 329 ++ include/mgard-x-160/Lossless/LZ4/LZ4Fused.hpp | 177 + .../mgard-x-160/Lossless/LZ4/LZ4Kernels.hpp | 505 ++ include/mgard-x-160/Lossless/Lossless.hpp | 279 + .../Lossless/LosslessCompressorInterface.hpp | 21 + .../Lossless/ParallelHuffman/Condense.hpp | 92 + .../Lossless/ParallelHuffman/Decode.hpp | 207 + .../Lossless/ParallelHuffman/Deflate.hpp | 148 + .../ParallelHuffman/DictionaryShift.hpp | 80 + .../ParallelHuffman/EncodeFixedLen.hpp | 96 + .../ParallelHuffman/EntropyCalculator.hpp | 60 + .../ParallelHuffman/FillArraySequence.hpp | 69 + .../Lossless/ParallelHuffman/GenerateCL.hpp | 936 ++++ .../Lossless/ParallelHuffman/GenerateCW.hpp | 319 ++ .../Lossless/ParallelHuffman/GetCodebook.hpp | 156 + .../ParallelHuffman/GetFirstNonzeroIndex.hpp | 78 + .../Lossless/ParallelHuffman/Histogram.hpp | 251 + .../Lossless/ParallelHuffman/Huffman.hpp | 638 +++ .../ParallelHuffman/HuffmanWorkspace.hpp | 379 ++ .../ParallelHuffman/OutlierSeparator.hpp | 186 + .../ParallelHuffman/ParallelDeflate.hpp | 412 ++ .../ParallelHuffman/ReorderByIndex.hpp | 91 + .../Lossless/ParallelHuffman/ReverseArray.hpp | 81 + .../Lossless/ParallelRLE/Convert.hpp | 86 + .../Lossless/ParallelRLE/Decode.hpp | 109 + .../Lossless/ParallelRLE/Encode.hpp | 105 + .../ParallelRLE/RunLengthEncoding.hpp | 361 ++ .../Lossless/ParallelRLE/StartMarks.hpp | 94 + .../Lossless/ParallelRLE/StartPositions.hpp | 99 + .../Lossless/ParallelRLE/ZeroDecode.hpp | 172 + .../Lossless/ParallelRLE/ZeroEncode.hpp | 114 + .../ParallelRLE/ZeroRunLengthEncoding.hpp | 287 + .../Lossless/ParallelRLE/ZeroStartMarks.hpp | 91 + .../ParallelRLE/ZeroStartPositions.hpp | 102 + .../Lossless/SymbolRans/SymbolRans.hpp | 186 + include/mgard-x-160/Lossless/Zstd.hpp | 165 + include/mgard-x-160/Lossless/rANS/Rans.hpp | 510 ++ .../mgard-x-160/Lossless/rANS/RansCommon.hpp | 179 + .../mgard-x-160/Lossless/rANS/RansDecode.hpp | 133 + .../mgard-x-160/Lossless/rANS/RansEncode.hpp | 207 + .../Lossless/rANS/RansInterleaved.hpp | 448 ++ include/mgard-x-160/Metadata/Metadata.hpp | 288 + .../HybridHierarchyLinearQuantization.hpp | 830 +++ .../Quantization/LinearQuantization.hpp | 618 +++ .../Quantization/LocalQuantization.hpp | 787 +++ .../Quantization/QuantizationInterface.hpp | 25 + .../RuntimeX/AutoTuners/AutoTuner.h | 270 + .../RuntimeX/AutoTuners/AutoTunerCuda.h | 106 + .../RuntimeX/AutoTuners/AutoTunerHip.h | 106 + .../RuntimeX/AutoTuners/AutoTunerKokkos.h | 384 ++ .../RuntimeX/AutoTuners/AutoTunerOpenmp.h | 105 + .../RuntimeX/AutoTuners/AutoTunerSerial.h | 105 + .../RuntimeX/AutoTuners/AutoTunerSycl.h | 106 + .../RuntimeX/DataStructures/Array.h | 69 + .../RuntimeX/DataStructures/Array.hpp | 375 ++ .../RuntimeX/DataStructures/MDRData.hpp | 199 + .../RuntimeX/DataStructures/MDRMetadata.hpp | 210 + .../RuntimeX/DataStructures/SubArray.hpp | 377 ++ .../RuntimeX/DataStructures/SubArrayCopy.hpp | 54 + include/mgard-x-160/RuntimeX/DataTypes.h | 137 + .../RuntimeX/DeviceAdapters/DeviceAdapter.h | 354 ++ .../DeviceAdapters/DeviceAdapterCuda.h | 2792 ++++++++++ .../DeviceAdapters/DeviceAdapterHip.h | 2580 +++++++++ .../DeviceAdapters/DeviceAdapterKokkos.h | 382 ++ .../DeviceAdapters/DeviceAdapterOpenmp.h | 1575 ++++++ .../DeviceAdapters/DeviceAdapterSerial.h | 1501 ++++++ .../DeviceAdapters/DeviceAdapterSycl.h | 2000 +++++++ .../mgard-x-160/RuntimeX/Functors/Functor.h | 300 ++ include/mgard-x-160/RuntimeX/Kernel/Kernel.h | 17 + include/mgard-x-160/RuntimeX/RuntimeX.h | 70 + include/mgard-x-160/RuntimeX/RuntimeXPublic.h | 22 + include/mgard-x-160/RuntimeX/Tasks/Task.h | 82 + .../RuntimeX/Utilities/CheckShape.hpp | 30 + .../RuntimeX/Utilities/Exceptions.h | 53 + include/mgard-x-160/RuntimeX/Utilities/Log.h | 54 + .../RuntimeX/Utilities/OffsetCalculators.hpp | 61 + .../RuntimeX/Utilities/Serializer.hpp | 79 + .../RuntimeX/Utilities/SubArrayPrinter.hpp | 535 ++ .../mgard-x-160/RuntimeX/Utilities/Timer.hpp | 54 + .../mgard-x-160/Utilities/ErrorCalculator.h | 125 + .../mgard-x-160/Utilities/ProjectionMode.h | 52 + include/mgard-x-160/Utilities/Types.h | 133 + .../CompressionHighLevel/CMakeLists.txt | 1 + .../CompressionHighLevel.hpp | 8 + .../CompressionHighLevel/DataCompatibility.h | 47 + include/mgard-x/Metadata/Metadata.hpp | 18 + include/mgard-x/Utilities/Types.h | 5 +- src/mgard-x-160/CMakeLists.txt | 13 + .../CompressionHighLevel/Bridge.cpp.in | 103 + .../CompressionHighLevel/CMakeLists.txt | 10 + .../CompressionHighLevel/Decompress.cpp.in | 39 + .../CompressionLowLevel/CMakeLists.txt | 6 + .../CompressionLowLevel/Compressor.cpp.in | 18 + src/mgard-x-160/Config/CMakeLists.txt | 4 + src/mgard-x-160/Config/Config.cpp | 73 + .../DataRefactoring/CMakeLists.txt | 7 + .../MultiDimension/CMakeLists.txt | 10 + .../MultiDimension/Coefficient/CMakeLists.txt | 10 + .../Coefficient/CalcCoefficients3D.cpp.in | 27 + .../Coefficient/CalcCoefficientsND.cpp.in | 28 + .../Coefficient/CoefficientsRestore3D.cpp.in | 27 + .../Coefficient/CoefficientsRestoreND.cpp.in | 28 + .../MultiDimension/CopyND/AddND.cpp.in | 25 + .../MultiDimension/CopyND/CMakeLists.txt | 9 + .../MultiDimension/CopyND/Copy3D.cpp.in | 25 + .../MultiDimension/CopyND/CopyND.cpp.in | 25 + .../MultiDimension/CopyND/SubtractND.cpp.in | 25 + .../MultiDimension/Correction/CMakeLists.txt | 7 + .../Correction/CalcCorrection3D.cpp.in | 27 + .../Correction/CalcCorrectionND.cpp.in | 27 + .../MultiDimension/Decompose.cpp.in | 27 + .../MultiDimension/Recompose.cpp.in | 27 + .../SingleDimension/CMakeLists.txt | 9 + .../Coefficient/CMakeLists.txt | 7 + .../Coefficient/CalcCoefficients.cpp.in | 27 + .../Coefficient/CoefficientsRestore.cpp.in | 27 + .../SingleDimension/Correction/CMakeLists.txt | 6 + .../Correction/CalcCorrection.cpp.in | 27 + .../SingleDimension/Decompose.cpp.in | 26 + .../SingleDimension/Recompose.cpp.in | 26 + src/mgard-x-160/Hierarchy/CMakeLists.txt | 6 + src/mgard-x-160/Hierarchy/Hierarchy.cpp.in | 15 + src/mgard-x-160/Metadata/CMakeLists.txt | 4 + src/mgard-x-160/Metadata/Metadata.cpp | 1035 ++++ .../RuntimeX/AutoTuners/AutoTunerCuda.cu | 17 + .../RuntimeX/AutoTuners/AutoTunerHip.cpp | 17 + .../RuntimeX/AutoTuners/AutoTunerOpenmp.cpp | 17 + .../RuntimeX/AutoTuners/AutoTunerSerial.cpp | 17 + .../RuntimeX/AutoTuners/AutoTunerSycl.cpp | 17 + .../RuntimeX/AutoTuners/CMakeLists.txt | 32 + src/mgard-x-160/RuntimeX/CMakeLists.txt | 9 + .../RuntimeX/DeviceAdapters/CMakeLists.txt | 30 + .../DeviceAdapters/DeviceAdapterCuda.cu | 24 + .../DeviceAdapters/DeviceAdapterHip.cpp | 24 + .../DeviceAdapters/DeviceAdapterKokkos.cpp | 23 + .../DeviceAdapters/DeviceAdapterOpenmp.cpp | 24 + .../DeviceAdapters/DeviceAdapterSerial.cpp | 24 + .../DeviceAdapters/DeviceAdapterSycl.cpp | 25 + .../RuntimeX/Utilities/CMakeLists.txt | 3 + src/mgard-x-160/RuntimeX/Utilities/Log.cpp | 77 + .../CompressionHighLevel/Decompress.cpp.in | 1 + .../CompressionHighLevel/DynamicAPI.cpp | 131 +- src/mgard-x/Metadata/Metadata.cpp | 26 + 215 files changed, 61181 insertions(+), 26 deletions(-) create mode 100644 include/mgard-x-160/CompressionHighLevel/CPUPipelines.hpp create mode 100644 include/mgard-x-160/CompressionHighLevel/CompressionHighLevel.h create mode 100644 include/mgard-x-160/CompressionHighLevel/CompressionHighLevel.hpp create mode 100644 include/mgard-x-160/CompressionHighLevel/ErrorToleranceCalculator.hpp create mode 100644 include/mgard-x-160/CompressionHighLevel/GPUPipelines.hpp create mode 100644 include/mgard-x-160/CompressionHighLevel/ShapeAdjustment.hpp create mode 100644 include/mgard-x-160/CompressionLowLevel/Compressor.h create mode 100644 include/mgard-x-160/CompressionLowLevel/Compressor.hpp create mode 100644 include/mgard-x-160/CompressionLowLevel/CompressorCache.hpp create mode 100644 include/mgard-x-160/CompressionLowLevel/HybridHierarchyCompressor.h create mode 100644 include/mgard-x-160/CompressionLowLevel/HybridHierarchyCompressor.hpp create mode 100644 include/mgard-x-160/CompressionLowLevel/LossyCompressorInterface.hpp create mode 100644 include/mgard-x-160/CompressionLowLevel/NormCalculator.hpp create mode 100644 include/mgard-x-160/Config/Config.h create mode 100644 include/mgard-x-160/DataRefactoring/BlockLocalHierarchyDataRefactor.hpp create mode 100644 include/mgard-x-160/DataRefactoring/BlockLocalHierarchyDataRefactorInterface.hpp create mode 100644 include/mgard-x-160/DataRefactoring/DataRefactor.hpp create mode 100644 include/mgard-x-160/DataRefactoring/DataRefactorInterface.hpp create mode 100644 include/mgard-x-160/DataRefactoring/HybridHierarchyDataRefactor.hpp create mode 100644 include/mgard-x-160/DataRefactoring/HybridHierarchyDataRefactorInterface.hpp create mode 100644 include/mgard-x-160/DataRefactoring/InCacheBlock/Autocorrelation8x8x8.hpp create mode 100644 include/mgard-x-160/DataRefactoring/InCacheBlock/DataRefactoring.h create mode 100644 include/mgard-x-160/DataRefactoring/InCacheBlock/DataRefactoring.hpp create mode 100644 include/mgard-x-160/DataRefactoring/InCacheBlock/Decompose8.hpp create mode 100644 include/mgard-x-160/DataRefactoring/InCacheBlock/Decompose8x8.hpp create mode 100644 include/mgard-x-160/DataRefactoring/InCacheBlock/Decompose8x8x8.hpp create mode 100644 include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTable3x3x3.hpp create mode 100644 include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTable5x5x5.hpp create mode 100644 include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTable8x8x8.hpp create mode 100644 include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTableLowDim.hpp create mode 100644 include/mgard-x-160/DataRefactoring/InCacheBlock/Recompose8.hpp create mode 100644 include/mgard-x-160/DataRefactoring/InCacheBlock/Recompose8x8.hpp create mode 100644 include/mgard-x-160/DataRefactoring/InCacheBlock/Recompose8x8x8.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficients3D.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsND.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsPointers.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestore3D.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestoreND.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/GPKFunctor.h create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel3D.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/AddND.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/Copy3D.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/CopyND.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/LevelwiseProcessingKernel.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/SubtractND.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrection3D.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrectionND.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Correction/IPKFunctor.h create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel3D.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Correction/LPKFunctor.h create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel3D.hpp create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/DataRefactoring.h create mode 100644 include/mgard-x-160/DataRefactoring/MultiDimension/DataRefactoring.hpp create mode 100644 include/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CalcCoefficients.hpp create mode 100644 include/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CoefficientKernel.hpp create mode 100644 include/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CoefficientsRestore.hpp create mode 100644 include/mgard-x-160/DataRefactoring/SingleDimension/Correction/CalcCorrection.hpp create mode 100644 include/mgard-x-160/DataRefactoring/SingleDimension/Correction/MassTransKernel.hpp create mode 100644 include/mgard-x-160/DataRefactoring/SingleDimension/DataRefactoring.h create mode 100644 include/mgard-x-160/DataRefactoring/SingleDimension/DataRefactoring.hpp create mode 100644 include/mgard-x-160/DomainDecomposer/DomainDecomposer.hpp create mode 100644 include/mgard-x-160/Hierarchy/Hierarchy.h create mode 100644 include/mgard-x-160/Hierarchy/Hierarchy.hpp create mode 100644 include/mgard-x-160/Hierarchy/HybridHierarchy.h create mode 100644 include/mgard-x-160/Lossless/BlockDelta/BlockDelta.hpp create mode 100644 include/mgard-x-160/Lossless/BlockDelta/BlockDeltaFused.hpp create mode 100644 include/mgard-x-160/Lossless/BlockDelta/BlockDeltaKernels.hpp create mode 100644 include/mgard-x-160/Lossless/CPU.hpp create mode 100644 include/mgard-x-160/Lossless/LZ4/LZ4.hpp create mode 100644 include/mgard-x-160/Lossless/LZ4/LZ4Fused.hpp create mode 100644 include/mgard-x-160/Lossless/LZ4/LZ4Kernels.hpp create mode 100644 include/mgard-x-160/Lossless/Lossless.hpp create mode 100644 include/mgard-x-160/Lossless/LosslessCompressorInterface.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/Condense.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/Decode.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/Deflate.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/DictionaryShift.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/EncodeFixedLen.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/EntropyCalculator.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/FillArraySequence.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/GenerateCL.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/GenerateCW.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/GetCodebook.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/GetFirstNonzeroIndex.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/Histogram.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/Huffman.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/HuffmanWorkspace.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/OutlierSeparator.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/ParallelDeflate.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/ReorderByIndex.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelHuffman/ReverseArray.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelRLE/Convert.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelRLE/Decode.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelRLE/Encode.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelRLE/RunLengthEncoding.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelRLE/StartMarks.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelRLE/StartPositions.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelRLE/ZeroDecode.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelRLE/ZeroEncode.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelRLE/ZeroRunLengthEncoding.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelRLE/ZeroStartMarks.hpp create mode 100644 include/mgard-x-160/Lossless/ParallelRLE/ZeroStartPositions.hpp create mode 100644 include/mgard-x-160/Lossless/SymbolRans/SymbolRans.hpp create mode 100644 include/mgard-x-160/Lossless/Zstd.hpp create mode 100644 include/mgard-x-160/Lossless/rANS/Rans.hpp create mode 100644 include/mgard-x-160/Lossless/rANS/RansCommon.hpp create mode 100644 include/mgard-x-160/Lossless/rANS/RansDecode.hpp create mode 100644 include/mgard-x-160/Lossless/rANS/RansEncode.hpp create mode 100644 include/mgard-x-160/Lossless/rANS/RansInterleaved.hpp create mode 100644 include/mgard-x-160/Metadata/Metadata.hpp create mode 100644 include/mgard-x-160/Quantization/HybridHierarchyLinearQuantization.hpp create mode 100644 include/mgard-x-160/Quantization/LinearQuantization.hpp create mode 100644 include/mgard-x-160/Quantization/LocalQuantization.hpp create mode 100644 include/mgard-x-160/Quantization/QuantizationInterface.hpp create mode 100644 include/mgard-x-160/RuntimeX/AutoTuners/AutoTuner.h create mode 100644 include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerCuda.h create mode 100644 include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerHip.h create mode 100644 include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerKokkos.h create mode 100644 include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerOpenmp.h create mode 100644 include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSerial.h create mode 100644 include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSycl.h create mode 100644 include/mgard-x-160/RuntimeX/DataStructures/Array.h create mode 100644 include/mgard-x-160/RuntimeX/DataStructures/Array.hpp create mode 100644 include/mgard-x-160/RuntimeX/DataStructures/MDRData.hpp create mode 100644 include/mgard-x-160/RuntimeX/DataStructures/MDRMetadata.hpp create mode 100644 include/mgard-x-160/RuntimeX/DataStructures/SubArray.hpp create mode 100644 include/mgard-x-160/RuntimeX/DataStructures/SubArrayCopy.hpp create mode 100644 include/mgard-x-160/RuntimeX/DataTypes.h create mode 100644 include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapter.h create mode 100644 include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h create mode 100644 include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterHip.h create mode 100644 include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h create mode 100644 include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h create mode 100644 include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h create mode 100644 include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h create mode 100644 include/mgard-x-160/RuntimeX/Functors/Functor.h create mode 100644 include/mgard-x-160/RuntimeX/Kernel/Kernel.h create mode 100644 include/mgard-x-160/RuntimeX/RuntimeX.h create mode 100644 include/mgard-x-160/RuntimeX/RuntimeXPublic.h create mode 100644 include/mgard-x-160/RuntimeX/Tasks/Task.h create mode 100644 include/mgard-x-160/RuntimeX/Utilities/CheckShape.hpp create mode 100644 include/mgard-x-160/RuntimeX/Utilities/Exceptions.h create mode 100644 include/mgard-x-160/RuntimeX/Utilities/Log.h create mode 100644 include/mgard-x-160/RuntimeX/Utilities/OffsetCalculators.hpp create mode 100644 include/mgard-x-160/RuntimeX/Utilities/Serializer.hpp create mode 100644 include/mgard-x-160/RuntimeX/Utilities/SubArrayPrinter.hpp create mode 100644 include/mgard-x-160/RuntimeX/Utilities/Timer.hpp create mode 100644 include/mgard-x-160/Utilities/ErrorCalculator.h create mode 100644 include/mgard-x-160/Utilities/ProjectionMode.h create mode 100644 include/mgard-x-160/Utilities/Types.h create mode 100644 include/mgard-x/CompressionHighLevel/DataCompatibility.h create mode 100644 src/mgard-x-160/CMakeLists.txt create mode 100644 src/mgard-x-160/CompressionHighLevel/Bridge.cpp.in create mode 100644 src/mgard-x-160/CompressionHighLevel/CMakeLists.txt create mode 100644 src/mgard-x-160/CompressionHighLevel/Decompress.cpp.in create mode 100644 src/mgard-x-160/CompressionLowLevel/CMakeLists.txt create mode 100644 src/mgard-x-160/CompressionLowLevel/Compressor.cpp.in create mode 100644 src/mgard-x-160/Config/CMakeLists.txt create mode 100644 src/mgard-x-160/Config/Config.cpp create mode 100644 src/mgard-x-160/DataRefactoring/CMakeLists.txt create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/CMakeLists.txt create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CMakeLists.txt create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficients3D.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsND.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestore3D.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestoreND.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/AddND.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/CMakeLists.txt create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/Copy3D.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/CopyND.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/SubtractND.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/Correction/CMakeLists.txt create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrection3D.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrectionND.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/Decompose.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/MultiDimension/Recompose.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/SingleDimension/CMakeLists.txt create mode 100644 src/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CMakeLists.txt create mode 100644 src/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CalcCoefficients.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CoefficientsRestore.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/SingleDimension/Correction/CMakeLists.txt create mode 100644 src/mgard-x-160/DataRefactoring/SingleDimension/Correction/CalcCorrection.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/SingleDimension/Decompose.cpp.in create mode 100644 src/mgard-x-160/DataRefactoring/SingleDimension/Recompose.cpp.in create mode 100644 src/mgard-x-160/Hierarchy/CMakeLists.txt create mode 100644 src/mgard-x-160/Hierarchy/Hierarchy.cpp.in create mode 100644 src/mgard-x-160/Metadata/CMakeLists.txt create mode 100644 src/mgard-x-160/Metadata/Metadata.cpp create mode 100644 src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerCuda.cu create mode 100644 src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerHip.cpp create mode 100644 src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerOpenmp.cpp create mode 100644 src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSerial.cpp create mode 100644 src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSycl.cpp create mode 100644 src/mgard-x-160/RuntimeX/AutoTuners/CMakeLists.txt create mode 100644 src/mgard-x-160/RuntimeX/CMakeLists.txt create mode 100644 src/mgard-x-160/RuntimeX/DeviceAdapters/CMakeLists.txt create mode 100644 src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterCuda.cu create mode 100644 src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterHip.cpp create mode 100644 src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.cpp create mode 100644 src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.cpp create mode 100644 src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSerial.cpp create mode 100644 src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSycl.cpp create mode 100644 src/mgard-x-160/RuntimeX/Utilities/CMakeLists.txt create mode 100644 src/mgard-x-160/RuntimeX/Utilities/Log.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e708203cc..f435d0071 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,6 +47,8 @@ option(MGARD_ENABLE_UNSTRUCTURED "Enable experimental unstructured mesh support. option(MGARD_ENABLE_TIMING "Enable timing." OFF) +option(MGARD_ENABLE_DATA_COMPATIBILITY "Also build the separate MGARD 1.6.0-compatible decompression pipeline (src/mgard-x-160) so data compressed by MGARD 1.6.x can be decompressed. Roughly doubles MGARD-X build time." OFF) + include(CTest) # Some boilerplate to setup nice output directories @@ -106,6 +108,9 @@ if (MGARD_ENABLE_SERIAL OR include(MgardXGenerateSource) add_subdirectory (src/mgard-x) + if (MGARD_ENABLE_DATA_COMPATIBILITY) + add_subdirectory (src/mgard-x-160) + endif() add_subdirectory (include/mgard-x) add_subdirectory (include/mgard-x/MDR) add_subdirectory (include/mgard-x/MDR-X) @@ -169,6 +174,10 @@ endif() #Adding library here so we can set compile definitions for it. add_library(mgard-library) +if(MGARD_ENABLE_DATA_COMPATIBILITY) + target_compile_definitions(mgard-library PUBLIC MGARD_ENABLE_DATA_COMPATIBILITY) +endif() + if(MGARD_ENABLE_SERIAL) target_compile_definitions(mgard-library PUBLIC MGARD_ENABLE_SERIAL) set_source_files_properties(${MGARD_X_SERIAL_SRC} PROPERTIES LANGUAGE CXX) @@ -499,6 +508,7 @@ if(MGARD_ENABLE_MDR) message(STATUS "pmdr-x (MPI): ${MGARD_X_BUILD_PMDR}") list(POP_BACK CMAKE_MESSAGE_INDENT) endif() +message(STATUS "1.6.0 data: ${MGARD_ENABLE_DATA_COMPATIBILITY}") message(STATUS "timing: ${MGARD_ENABLE_TIMING}") message(STATUS "ASan: ${MGARD_ENABLE_ASAN}") message(STATUS "UBSan: ${MGARD_ENABLE_UBSAN}") diff --git a/include/mgard-x-160/CompressionHighLevel/CPUPipelines.hpp b/include/mgard-x-160/CompressionHighLevel/CPUPipelines.hpp new file mode 100644 index 000000000..1e41b2347 --- /dev/null +++ b/include/mgard-x-160/CompressionHighLevel/CPUPipelines.hpp @@ -0,0 +1,535 @@ +namespace mgard_x_160 { + +template +enum compress_status_type compress_pipeline_cpu( + DomainDecomposer &domain_decomposer, + T local_tol, T s, T &norm, enum error_bound_type local_ebtype, + Config &config, Byte *compressed_subdomain_data, + SIZE &compressed_subdomain_size) { + Timer timer_series; + if (log::level & log::TIME) + timer_series.start(); + using Cache = CompressorCache; + using HierarchyType = typename CompressorType::HierarchyType; + CompressorType *compressor = Cache::cache.compressor; + Array *device_subdomain_buffer = + Cache::cache.device_subdomain_buffer; + Array<1, Byte, DeviceType> *device_compressed_buffer = + Cache::cache.device_compressed_buffer; + + if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0), + domain_decomposer.uniform, config)) { + Cache::cache.ClearHierarchyCache(); + } + + for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { + if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id), + domain_decomposer.uniform, config)) { + Cache::cache.InsertHierarchyCache( + domain_decomposer.subdomain_hierarchy(id)); + } + } + log::info("Adjust device buffers and hierarchy"); + for (int i = 0; i < domain_decomposer.num_subdomains(); i++) { + std::vector shape = domain_decomposer.subdomain_shape(i); + device_subdomain_buffer[i].resize(shape); + device_compressed_buffer[i].resize( + {domain_decomposer.subdomain_compressed_buffer_size( + domain_decomposer.largest_subdomain())}); + + HierarchyType &hierarchy = + Cache::cache.GetHierarchyCache(domain_decomposer.subdomain_shape(i)); + compressor[i].Adapt(hierarchy, config, 0); + } + + std::vector compressed_size(domain_decomposer.num_subdomains(), 0); + + DeviceRuntime::SyncDevice(); + + if (log::level & log::TIME) { + timer_series.end(); + timer_series.print("Prepare device environment"); + timer_series.clear(); + timer_series.start(); + } + + Timer timer_profile; + std::vector h2d, d2h, comp; + std::vector size; + bool profile = false; + bool profile_e2e = false; + + // For serialization + SIZE byte_offset = 0; + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.clear(); + timer_profile.start(); + } + +#pragma omp parallel for + for (SIZE curr_subdomain_id = 0; + curr_subdomain_id < domain_decomposer.num_subdomains(); + curr_subdomain_id++) { + + domain_decomposer.copy_subdomain( + device_subdomain_buffer[curr_subdomain_id], curr_subdomain_id, + subdomain_copy_direction::OriginalToSubdomain, 0); + } + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + h2d.push_back(timer_profile.get()); + timer_profile.clear(); + timer_profile.start(); + } + + if (config.cpu_mode == cpu_parallelization_mode::INTER_BLOCK) { + omp_set_nested(0); +#pragma omp parallel for + for (SIZE curr_subdomain_id = 0; + curr_subdomain_id < domain_decomposer.num_subdomains(); + curr_subdomain_id++) { + + std::stringstream ss; + for (DIM d = 0; d < D; d++) { + ss << compressor[curr_subdomain_id].hierarchy->level_shape( + compressor[curr_subdomain_id].hierarchy->l_target(), d) + << " "; + } + log::info("Compressing subdomain " + std::to_string(curr_subdomain_id) + + " with shape: " + ss.str()); + + compressor[curr_subdomain_id].Compress( + device_subdomain_buffer[curr_subdomain_id], local_ebtype, local_tol, + s, norm, device_compressed_buffer[curr_subdomain_id], 0); + + compressed_size[curr_subdomain_id] = + device_compressed_buffer[curr_subdomain_id].shape(0); + double CR = + (double)(compressor[curr_subdomain_id].hierarchy->total_num_elems() * + sizeof(T)) / + compressed_size[curr_subdomain_id]; + log::info("Subdomain CR: " + std::to_string(CR)); + if (CR < 1.0) { + log::info("Using uncompressed data instead"); + domain_decomposer.copy_subdomain( + device_subdomain_buffer[curr_subdomain_id], curr_subdomain_id, + subdomain_copy_direction::OriginalToSubdomain, 0); + SIZE linearized_width = 1; + for (DIM d = 0; d < D - 1; d++) + linearized_width *= + device_subdomain_buffer[curr_subdomain_id].shape(d); + MemoryManager::CopyND( + device_compressed_buffer[curr_subdomain_id].data(), + device_subdomain_buffer[curr_subdomain_id].shape(D - 1) * sizeof(T), + (Byte *)device_subdomain_buffer[curr_subdomain_id].data(), + device_subdomain_buffer[curr_subdomain_id].ld(D - 1) * sizeof(T), + device_subdomain_buffer[curr_subdomain_id].shape(D - 1) * sizeof(T), + linearized_width, 0); + compressed_size[curr_subdomain_id] = + compressor[curr_subdomain_id].hierarchy->total_num_elems() * + sizeof(T); + } + } + } else { + for (SIZE curr_subdomain_id = 0; + curr_subdomain_id < domain_decomposer.num_subdomains(); + curr_subdomain_id++) { + + std::stringstream ss; + for (DIM d = 0; d < D; d++) { + ss << compressor[curr_subdomain_id].hierarchy->level_shape( + compressor[curr_subdomain_id].hierarchy->l_target(), d) + << " "; + } + log::info("Compressing subdomain " + std::to_string(curr_subdomain_id) + + " with shape: " + ss.str()); + + compressor[curr_subdomain_id].Compress( + device_subdomain_buffer[curr_subdomain_id], local_ebtype, local_tol, + s, norm, device_compressed_buffer[curr_subdomain_id], 0); + + compressed_size[curr_subdomain_id] = + device_compressed_buffer[curr_subdomain_id].shape(0); + double CR = + (double)(compressor[curr_subdomain_id].hierarchy->total_num_elems() * + sizeof(T)) / + compressed_size[curr_subdomain_id]; + log::info("Subdomain CR: " + std::to_string(CR)); + if (CR < 1.0) { + log::info("Using uncompressed data instead"); + domain_decomposer.copy_subdomain( + device_subdomain_buffer[curr_subdomain_id], curr_subdomain_id, + subdomain_copy_direction::OriginalToSubdomain, 0); + SIZE linearized_width = 1; + for (DIM d = 0; d < D - 1; d++) + linearized_width *= + device_subdomain_buffer[curr_subdomain_id].shape(d); + MemoryManager::CopyND( + device_compressed_buffer[curr_subdomain_id].data(), + device_subdomain_buffer[curr_subdomain_id].shape(D - 1) * sizeof(T), + (Byte *)device_subdomain_buffer[curr_subdomain_id].data(), + device_subdomain_buffer[curr_subdomain_id].ld(D - 1) * sizeof(T), + device_subdomain_buffer[curr_subdomain_id].shape(D - 1) * sizeof(T), + linearized_width, 0); + compressed_size[curr_subdomain_id] = + compressor[curr_subdomain_id].hierarchy->total_num_elems() * + sizeof(T); + } + } + } + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + comp.push_back(timer_profile.get()); + timer_profile.clear(); + timer_profile.start(); + } + + for (SIZE curr_subdomain_id = 0; + curr_subdomain_id < domain_decomposer.num_subdomains(); + curr_subdomain_id++) { + // Check if we have enough space + if (compressed_size[curr_subdomain_id] > + compressed_subdomain_size - byte_offset - sizeof(SIZE)) { + log::err( + "Output too large (original size: " + + std::to_string((double)(compressor[curr_subdomain_id] + .hierarchy->total_num_elems()) * + sizeof(T) / 1e9) + + " GB, compressed size: " + + std::to_string((double)(compressed_size[curr_subdomain_id]) / 1e9) + + " GB, leftover buffer space: " + + std::to_string( + (double)(compressed_subdomain_size - byte_offset - sizeof(SIZE)) / + 1e9) + + " GB)"); + return compress_status_type::OutputTooLargeFailure; + } + + Serialize(compressed_subdomain_data, + &compressed_size[curr_subdomain_id], 1, + byte_offset, 0); + Serialize( + compressed_subdomain_data, + device_compressed_buffer[curr_subdomain_id].data(), + compressed_size[curr_subdomain_id], byte_offset, 0); + + size.push_back(compressor[curr_subdomain_id].hierarchy->total_num_elems() * + sizeof(T)); + } + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + d2h.push_back(timer_profile.get()); + } + + SIZE total_size = 0; + for (auto s : size) + total_size += s; + + if (profile) { + std::cout << "comp: " + << "\n"; + for (float t : comp) + std::cout << t << ", "; + std::cout << "\n"; + + std::cout << "h2d: " + << "\n"; + for (float t : h2d) + std::cout << t << ", "; + std::cout << "\n"; + + std::cout << "d2h: " + << "\n"; + for (float t : d2h) + std::cout << t << ", "; + std::cout << "\n"; + + std::cout << "size: " + << "\n"; + std::cout << total_size << "\n"; + + std::cout << "comp_speed: " + << "\n"; + std::cout << total_size / comp[0] << "\n"; + } + + compressed_subdomain_size = byte_offset; + DeviceRuntime::SyncDevice(); + if (log::level & log::TIME) { + timer_series.end(); + timer_series.print("Compress pipeline", total_size); + timer_series.clear(); + } + return compress_status_type::Success; +} + +template +enum compress_status_type decompress_pipeline_cpu( + DomainDecomposer &domain_decomposer, + T local_tol, T s, T norm, enum error_bound_type local_ebtype, + Config &config, Byte *compressed_subdomain_data) { + Timer timer_series; + if (log::level & log::TIME) + timer_series.start(); + + SIZE byte_offset = 0; + using Cache = CompressorCache; + using HierarchyType = typename CompressorType::HierarchyType; + CompressorType *compressor = Cache::cache.compressor; + Array *device_subdomain_buffer = + Cache::cache.device_subdomain_buffer; + Array<1, Byte, DeviceType> *device_compressed_buffer = + Cache::cache.device_compressed_buffer; + + if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0), + domain_decomposer.uniform, config)) { + Cache::cache.ClearHierarchyCache(); + } + + for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { + if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id), + domain_decomposer.uniform, config)) { + Cache::cache.InsertHierarchyCache( + domain_decomposer.subdomain_hierarchy(id)); + } + } + + log::info("Adjust device buffers"); + for (int i = 0; i < domain_decomposer.num_subdomains(); i++) { + std::vector shape = domain_decomposer.subdomain_shape(i); + device_subdomain_buffer[i].resize(shape); + device_compressed_buffer[i].resize( + {domain_decomposer.subdomain_compressed_buffer_size( + domain_decomposer.largest_subdomain())}); + + HierarchyType &hierarchy = + Cache::cache.GetHierarchyCache(domain_decomposer.subdomain_shape(i)); + compressor[i].Adapt(hierarchy, config, 0); + } + std::vector compressed_size(domain_decomposer.num_subdomains(), 0); + DeviceRuntime::SyncDevice(); + + if (log::level & log::TIME) { + timer_series.end(); + timer_series.print("Prepare device environment"); + timer_series.clear(); + timer_series.start(); + } + + Timer timer_profile; + std::vector h2d, d2h, comp; + std::vector size; + bool profile = false; + bool profile_e2e = false; + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.clear(); + timer_profile.start(); + } + + for (SIZE curr_subdomain_id = 0; + curr_subdomain_id < domain_decomposer.num_subdomains(); + curr_subdomain_id++) { + + // Deserialize and copy next compressed data on queue 1 + SIZE *compressed_size_ptr = &compressed_size[curr_subdomain_id]; + Byte *compressed_data; + + Deserialize(compressed_subdomain_data, + compressed_size_ptr, 1, byte_offset, false, + 0); + DeviceRuntime::SyncQueue(0); + Deserialize(compressed_subdomain_data, compressed_data, + compressed_size[curr_subdomain_id], + byte_offset, true, 0); + + device_compressed_buffer[curr_subdomain_id].resize( + {compressed_size[curr_subdomain_id]}); + + MemoryManager::Copy1D( + device_compressed_buffer[curr_subdomain_id].data(), compressed_data, + compressed_size[curr_subdomain_id], 0); + size.push_back(compressor[curr_subdomain_id].hierarchy->total_num_elems() * + sizeof(T)); + } + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + h2d.push_back(timer_profile.get()); + timer_profile.clear(); + timer_profile.start(); + } + + if (config.cpu_mode == cpu_parallelization_mode::INTER_BLOCK) { + omp_set_nested(0); +#pragma omp parallel for + for (SIZE curr_subdomain_id = 0; + curr_subdomain_id < domain_decomposer.num_subdomains(); + curr_subdomain_id++) { + double CR = + (double)(compressor[curr_subdomain_id].hierarchy->total_num_elems() * + sizeof(T)) / + compressed_size[curr_subdomain_id]; + log::info("Subdomain CR: " + std::to_string(CR)); + if (CR > 1.0) { + std::stringstream ss; + for (DIM d = 0; d < D; d++) { + ss << compressor[curr_subdomain_id].hierarchy->level_shape( + compressor[curr_subdomain_id].hierarchy->l_target(), d) + << " "; + } + log::info("Decompressing subdomain " + + std::to_string(curr_subdomain_id) + + " with shape: " + ss.str()); + compressor[curr_subdomain_id].Deserialize( + device_compressed_buffer[curr_subdomain_id], 0); + } + + if (CR > 1.0) { + compressor[curr_subdomain_id].LosslessDecompress( + device_compressed_buffer[curr_subdomain_id], 0); + compressor[curr_subdomain_id].DequantizeRecompose( + device_subdomain_buffer[curr_subdomain_id], local_ebtype, local_tol, + s, norm, 0); + } else { + log::info("Skipping decompression as original data was saved instead"); + device_subdomain_buffer[curr_subdomain_id].resize( + {compressor[curr_subdomain_id].hierarchy->level_shape( + compressor[curr_subdomain_id].hierarchy->l_target())}); + SIZE linearized_width = 1; + for (DIM d = 0; d < D - 1; d++) + linearized_width *= + device_subdomain_buffer[curr_subdomain_id].shape(d); + MemoryManager::CopyND( + device_subdomain_buffer[curr_subdomain_id].data(), + device_subdomain_buffer[curr_subdomain_id].ld(D - 1), + (T *)device_compressed_buffer[curr_subdomain_id].data(), + device_subdomain_buffer[curr_subdomain_id].shape(D - 1), + device_subdomain_buffer[curr_subdomain_id].shape(D - 1), + linearized_width, 0); + } + } + } else { + for (SIZE curr_subdomain_id = 0; + curr_subdomain_id < domain_decomposer.num_subdomains(); + curr_subdomain_id++) { + double CR = + (double)(compressor[curr_subdomain_id].hierarchy->total_num_elems() * + sizeof(T)) / + compressed_size[curr_subdomain_id]; + log::info("Subdomain CR: " + std::to_string(CR)); + if (CR > 1.0) { + std::stringstream ss; + for (DIM d = 0; d < D; d++) { + ss << compressor[curr_subdomain_id].hierarchy->level_shape( + compressor[curr_subdomain_id].hierarchy->l_target(), d) + << " "; + } + log::info("Decompressing subdomain " + + std::to_string(curr_subdomain_id) + + " with shape: " + ss.str()); + compressor[curr_subdomain_id].Deserialize( + device_compressed_buffer[curr_subdomain_id], 0); + } + + if (CR > 1.0) { + compressor[curr_subdomain_id].LosslessDecompress( + device_compressed_buffer[curr_subdomain_id], 0); + compressor[curr_subdomain_id].DequantizeRecompose( + device_subdomain_buffer[curr_subdomain_id], local_ebtype, local_tol, + s, norm, 0); + } else { + log::info("Skipping decompression as original data was saved instead"); + device_subdomain_buffer[curr_subdomain_id].resize( + {compressor[curr_subdomain_id].hierarchy->level_shape( + compressor[curr_subdomain_id].hierarchy->l_target())}); + SIZE linearized_width = 1; + for (DIM d = 0; d < D - 1; d++) + linearized_width *= + device_subdomain_buffer[curr_subdomain_id].shape(d); + MemoryManager::CopyND( + device_subdomain_buffer[curr_subdomain_id].data(), + device_subdomain_buffer[curr_subdomain_id].ld(D - 1), + (T *)device_compressed_buffer[curr_subdomain_id].data(), + device_subdomain_buffer[curr_subdomain_id].shape(D - 1), + device_subdomain_buffer[curr_subdomain_id].shape(D - 1), + linearized_width, 0); + } + } + } + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + comp.push_back(timer_profile.get()); + timer_profile.clear(); + timer_profile.start(); + } + + omp_set_nested(0); +#pragma omp parallel for + for (SIZE curr_subdomain_id = 0; + curr_subdomain_id < domain_decomposer.num_subdomains(); + curr_subdomain_id++) { + domain_decomposer.copy_subdomain( + device_subdomain_buffer[curr_subdomain_id], curr_subdomain_id, + subdomain_copy_direction::SubdomainToOriginal, 0); + } + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + d2h.push_back(timer_profile.get()); + } + + SIZE total_size = 0; + for (auto s : size) + total_size += s; + + if (profile) { + std::cout << "comp: " + << "\n"; + for (float t : comp) + std::cout << t << ", "; + std::cout << "\n"; + + std::cout << "h2d: " + << "\n"; + for (float t : h2d) + std::cout << t << ", "; + std::cout << "\n"; + + std::cout << "d2h: " + << "\n"; + for (float t : d2h) + std::cout << t << ", "; + std::cout << "\n"; + + std::cout << "size: " + << "\n"; + std::cout << total_size << "\n"; + + std::cout << "comp_speed: " + << "\n"; + std::cout << total_size / comp[0] << "\n"; + } + + DeviceRuntime::SyncDevice(); + if (log::level & log::TIME) { + timer_series.end(); + timer_series.print("Decompress pipeline", total_size); + timer_series.clear(); + } + return compress_status_type::Success; +} +} // namespace mgard_x_160 diff --git a/include/mgard-x-160/CompressionHighLevel/CompressionHighLevel.h b/include/mgard-x-160/CompressionHighLevel/CompressionHighLevel.h new file mode 100644 index 000000000..18d15c543 --- /dev/null +++ b/include/mgard-x-160/CompressionHighLevel/CompressionHighLevel.h @@ -0,0 +1,85 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include +#include +#include +#include +#include + +// #include "../Hierarchy/Hierarchy.h" +// #include "../RuntimeX/RuntimeXPublic.h" +// #include "Metadata.hpp" + +#ifndef MGARD_X_160_COMPRESSION_HIGH_LEVEL_API_H +#define MGARD_X_160_COMPRESSION_HIGH_LEVEL_API_H + +namespace mgard_x_160 { + +template +enum compress_status_type +compress(DIM D, data_type dtype, std::vector shape, double tol, double s, + enum error_bound_type mode, const void *original_data, + void *&compressed_data, size_t &compressed_size, Config config, + bool output_pre_allocated); + +template +enum compress_status_type +compress(DIM D, data_type dtype, std::vector shape, double tol, double s, + enum error_bound_type mode, const void *original_data, + void *&compressed_data, size_t &compressed_size, + bool output_pre_allocated); + +template +enum compress_status_type +compress(DIM D, data_type dtype, std::vector shape, double tol, double s, + enum error_bound_type mode, const void *original_data, + void *&compressed_data, size_t &compressed_size, + std::vector coords, Config config, + bool output_pre_allocated); + +template +enum compress_status_type +compress(DIM D, data_type dtype, std::vector shape, double tol, double s, + enum error_bound_type mode, const void *original_data, + void *&compressed_data, size_t &compressed_size, + std::vector coords, bool output_pre_allocated); + +template +enum compress_status_type +decompress(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, Config config, bool output_pre_allocated); + +template +enum compress_status_type +decompress(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, bool output_pre_allocated); + +template +enum compress_status_type decompress(const void *compressed_data, + size_t compressed_size, + void *&decompressed_data, data_type &dtype, + std::vector &shape, + Config config, bool output_pre_allocated); + +template +enum compress_status_type +decompress(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, data_type &dtype, + std::vector &shape, bool output_pre_allocated); + +template enum compress_status_type release_cache(); + +template void pin_memory(void *ptr, SIZE num_bytes); + +template bool check_memory_pinned(void *ptr); + +template void unpin_memory(void *ptr); + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/CompressionHighLevel/CompressionHighLevel.hpp b/include/mgard-x-160/CompressionHighLevel/CompressionHighLevel.hpp new file mode 100644 index 000000000..0304f48e5 --- /dev/null +++ b/include/mgard-x-160/CompressionHighLevel/CompressionHighLevel.hpp @@ -0,0 +1,1051 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include +#include +#include +#include +#include +#include + +#include "../Config/Config.h" +#include "../Hierarchy/Hierarchy.h" +#include "../RuntimeX/RuntimeX.h" + +#include "../CompressionLowLevel/Compressor.h" + +#include "../CompressionLowLevel/CompressorCache.hpp" +#include "../CompressionLowLevel/HybridHierarchyCompressor.h" +#include "../CompressionLowLevel/NormCalculator.hpp" + +#include "../DomainDecomposer/DomainDecomposer.hpp" +#include "../Metadata/Metadata.hpp" +#include "CompressionHighLevel.h" + +#ifndef MGARD_X_160_COMPRESSION_HIGH_LEVEL_API_HPP +#define MGARD_X_160_COMPRESSION_HIGH_LEVEL_API_HPP + +#if MGARD_ENABLE_OPENMP +#include "CPUPipelines.hpp" +#endif +#include "ErrorToleranceCalculator.hpp" +#include "GPUPipelines.hpp" +#include "ShapeAdjustment.hpp" + +#define OUTPUT_SAFTY_OVERHEAD 1e6 + +namespace mgard_x_160 { + +// The hybrid hierarchy can only be given a theoretical s-norm bound when the +// block-local stage is a single level and there is no global stage. +// +// With one local level every block is transformed independently, so the total +// squared error is a plain sum over blocks and the budget divides evenly in +// quadrature -- tol / sqrt(num_blocks) per block -- exactly as +// calc_local_abs_tol does across subdomains for domain decomposition. A second +// local level mixes the coarse outputs of neighbouring blocks, and a global +// stage decomposes across blocks entirely; in both cases the blocks stop being +// independent and the sum no longer bounds the error. +// +// So for s != inf we fall back to that one configuration and say so. L-inf is +// untouched: its bound composes over levels and does not need independence. +template +void restrict_hybrid_config_for_s_norm(Config &config, T s) { + if (config.decomposition != decomposition_type::Hybrid || + s == std::numeric_limits::infinity()) { + return; + } + if (config.num_local_refactoring_level != 1) { + log::warn("hybrid with an s-norm bound supports one block-local level " + "only; falling back from " + + std::to_string(config.num_local_refactoring_level) + " to 1", + true); + config.num_local_refactoring_level = 1; + } + if (config.num_global_refactoring_level != 0) { + log::warn("hybrid with an s-norm bound cannot use a global stage (it " + "decomposes across blocks, so the per-block error budget no " + "longer composes); falling back from " + + std::to_string(config.num_global_refactoring_level) + + " global levels to 0", + true); + config.num_global_refactoring_level = 0; + } +} + +template +enum compress_status_type +general_compress_pipeline(std::vector shape, T tol, T s, + enum error_bound_type ebtype, + const void *original_data, void *&compressed_data, + size_t &compressed_size, Config config, bool uniform, + std::vector coords, bool output_pre_allocated) { + + DeviceRuntime::Initialize(); + size_t total_num_elem = 1; + for (int i = 0; i < D; i++) + total_num_elem *= shape[i]; + + config.apply(); + + // Before anything reads the level counts: the DomainDecomposer sizes + // subdomains from them, and they are recorded in the file header for the + // decompressor, so the adjustment has to happen here to stay consistent. + restrict_hybrid_config_for_s_norm(config, s); + + // Resolve the BlockMGARD basis before DomainDecomposer and compressor-cache + // construction. The resolved value is also what gets written to metadata. + // The plain (non-hybrid) path resolves Auto/explicit the same way, but + // purely from s -- which round-trips through metadata on its own -- so it + // needs no equivalent pre-resolution or persistence here. + if (config.decomposition == decomposition_type::Hybrid) { + config.projection_mode = resolve_projection_mode(config.projection_mode, s); + } + + log::info("adjust_shape: " + std::to_string(config.adjust_shape)); + if (config.adjust_shape) { + adjust_shape(shape, config); + } + + Timer timer_total, timer_each; + if (log::level & log::TIME) + timer_total.start(); + + DomainDecomposer domain_decomposer; + if (uniform) { + domain_decomposer = + DomainDecomposer(shape, config); + } else { + domain_decomposer = DomainDecomposer( + shape, config, coords); + } + domain_decomposer.set_original_data((T *)original_data); + + using Cache = CompressorCache; + if (std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value) { + Cache::cache.SafeInitialize(2, 1); + } else { + Cache::cache.SafeInitialize(domain_decomposer.num_subdomains(), + domain_decomposer.num_subdomains()); + } + + T norm = 1; + T local_tol = tol; + enum error_bound_type local_ebtype; + + if (config.compressor == compressor_type::MGARD) { + log::info("Compressor type: MGARD"); + } else if (config.compressor == compressor_type::ZFP) { + log::info("Compressor type: ZFP"); + } + + if (config.decomposition == decomposition_type::MultiDim) { + log::info("Multilevel Decomposition: multi-dimensional"); + } else if (config.decomposition == decomposition_type::SingleDim) { + log::info("Multilevel Decomposition: single-dimensional"); + } else if (config.decomposition == decomposition_type::Hybrid) { + log::info("Multilevel Decomposition: hybrid"); + } + + log::info("tol: " + std::to_string(tol)); + log::info("s: " + std::to_string(s)); + log::info("coordinate normalization: " + + std::to_string(config.normalize_coordinates)); + if (!domain_decomposer.domain_decomposed()) { + local_tol = tol; + local_ebtype = ebtype; + } else { + if (log::level & log::TIME) + timer_each.start(); + if (ebtype == error_bound_type::REL) { + // norm = calc_norm_decomposed(domain_decomposer, s, + // config.normalize_coordinates, + // total_num_elem); + norm = calc_norm_decomposed_w_prefetch( + domain_decomposer, s, config.normalize_coordinates, total_num_elem); + } + local_tol = calc_local_abs_tol(ebtype, norm, tol, s, + domain_decomposer.num_subdomains()); + // Force to use ABS mode when do domain decomposition + local_ebtype = error_bound_type::ABS; + if (log::level & log::TIME) { + timer_each.end(); + timer_each.print("Calculate norm of decomposed domain", + total_num_elem * sizeof(T)); + timer_each.clear(); + } + } + + if (log::level & log::TIME) + timer_each.start(); + // Use consistance memory space between input and output data + size_t output_buffer_size; + if (!output_pre_allocated) { + output_buffer_size = total_num_elem * sizeof(T) + OUTPUT_SAFTY_OVERHEAD; + // The header stores non-uniform coordinates as doubles, which can exceed + // the safety margin (8 MB for a 1M-point 1D grid). + if (!uniform) { + for (DIM d = 0; d < D; d++) { + output_buffer_size += shape[d] * sizeof(double); + } + } + if (MemoryManager::IsDevicePointer(original_data)) { + DeviceRuntime::SelectDevice( + MemoryManager::GetPointerDevice(original_data)); + MemoryManager::Malloc1D(compressed_data, output_buffer_size); + } else { + compressed_data = (unsigned char *)malloc(output_buffer_size); + } + } else { + // compressed_size stores pre-allocated buffer size + output_buffer_size = compressed_size; + } + log::info("Output preallocated: " + std::to_string(output_pre_allocated)); + + bool input_previously_pinned = true; + if (!MemoryManager::IsDevicePointer((void *)original_data)) { + input_previously_pinned = + MemoryManager::CheckHostRegister((void *)original_data); + if (!input_previously_pinned && config.auto_pin_host_buffers) { + MemoryManager::HostRegister((void *)original_data, + total_num_elem * sizeof(T)); + } + log::info("Input previously pinned: " + + std::to_string(input_previously_pinned)); + } else { + log::info("Input on device"); + } + + bool output_previously_pinned = true; + if (!MemoryManager::IsDevicePointer((void *)compressed_data)) { + output_previously_pinned = + MemoryManager::CheckHostRegister((void *)compressed_data); + if (!output_previously_pinned && config.auto_pin_host_buffers) { + MemoryManager::HostRegister((void *)compressed_data, + output_buffer_size); + } + log::info("Output previously pinned: " + + std::to_string(output_previously_pinned)); + } else { + log::info("Output on device"); + } + + // Estimate metadata size + Metadata m; + // FillForCompression does not carry the BlockDelta block size; set it here so + // it is persisted in (and restored from) the metadata header. + m.block_delta_block_size = config.block_delta_block_size; + // The decompressor rebuilds the hierarchy, so it needs the level cap. + m.has_num_global_refactoring_level = true; + m.num_global_refactoring_level = config.num_global_refactoring_level; + // Same for the hybrid (BlockMGARD) parameters: the decompressor cannot + // reconstruct the buffer layout or the per-block quantization steps without + // them. + if (config.decomposition == decomposition_type::Hybrid) { + m.hybrid_num_local_levels = config.num_local_refactoring_level; + m.hybrid_num_global_levels = config.num_global_refactoring_level; + m.hybrid_local_block_size = MGARDX_HYBRID_LOCAL_BLOCK_SIZE; + m.hybrid_projection_mode = config.projection_mode; + m.hybrid_enable_roi = config.enable_roi; + if (config.enable_roi) { + // Level-0 block grid: the local refactor pads each dimension up to a + // multiple of the block size, so the block count per dimension is just + // the ceiling division of the original shape. + m.hybrid_roi_block_dimensions.resize(shape.size()); + size_t expected_blocks = 1; + for (DIM d = 0; d < shape.size(); d++) { + m.hybrid_roi_block_dimensions[d] = + (shape[d] + MGARDX_HYBRID_LOCAL_BLOCK_SIZE - 1) / + MGARDX_HYBRID_LOCAL_BLOCK_SIZE; + expected_blocks *= m.hybrid_roi_block_dimensions[d]; + } + if (config.roi_tolerance_map.size() != expected_blocks) { + throw ProcessingException( + "ROI tolerance map holds " + + std::to_string(config.roi_tolerance_map.size()) + + " entries but the level-0 block grid of this shape needs " + + std::to_string(expected_blocks) + "."); + } + m.hybrid_roi_tolerance_map = config.roi_tolerance_map; + // Pre-existing limitation, surfaced here rather than left silent: the + // tolerance map is a single flat array indexed by each subdomain's own + // block id, so with more than one subdomain every subdomain re-reads the + // map from offset 0 and blocks get the wrong tolerances. The map is + // still recorded verbatim, so this warns rather than refuses -- the file + // round-trips exactly as it was compressed. + if (domain_decomposer.domain_decomposed()) { + log::warn("ROI tolerance map is applied per subdomain from offset 0, " + "so ROI results are not correct under domain decomposition; " + "compress without domain decomposition for correct ROI."); + } + } + } + if (uniform) { + m.FillForCompression( + ebtype, tol, s, norm, config.decomposition, config.lossless, + config.huff_dict_size, config.huff_block_size, shape, + domain_decomposer.domain_decomposed(), config.domain_decomposition, + domain_decomposer.domain_decomposed_dim(), + domain_decomposer.domain_decomposed_size()); + } else { + m.FillForCompression( + ebtype, tol, s, norm, config.decomposition, config.lossless, + config.huff_dict_size, config.huff_block_size, shape, + domain_decomposer.domain_decomposed(), config.domain_decomposition, + domain_decomposer.domain_decomposed_dim(), + domain_decomposer.domain_decomposed_size(), coords); + } + + uint32_t metadata_size; + SERIALIZED_TYPE *serizalied_meta = m.Serialize(metadata_size); + MemoryManager::Free(serizalied_meta); + + if (metadata_size > output_buffer_size) { + log::err("output buffer of " + std::to_string(output_buffer_size) + + " bytes cannot hold the " + std::to_string(metadata_size) + + "-byte header."); + if (!input_previously_pinned && config.auto_pin_host_buffers) { + MemoryManager::HostUnregister((void *)original_data); + } + if (!output_previously_pinned && config.auto_pin_host_buffers) { + MemoryManager::HostUnregister((void *)compressed_data); + } + return compress_status_type::OutputTooLargeFailure; + } + + SIZE byte_offset = metadata_size; + Byte *compressed_subdomain_data = (Byte *)compressed_data + byte_offset; + SIZE compressed_subdomain_size = output_buffer_size - metadata_size; + + if (log::level & log::TIME) { + timer_each.end(); + timer_each.print("Prepare input and output buffer"); + timer_each.clear(); + } + + enum compress_status_type compress_status; + DeviceRuntime::SelectDevice(config.dev_id); + if constexpr (std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value) { + compress_status = compress_pipeline_gpu( + domain_decomposer, local_tol, s, norm, local_ebtype, config, + compressed_subdomain_data, compressed_subdomain_size); + } else { + compress_status = compress_pipeline_cpu( + domain_decomposer, local_tol, s, norm, local_ebtype, config, + compressed_subdomain_data, compressed_subdomain_size); + } + + if (log::level & log::TIME) + timer_each.start(); + if (uniform) { + m.FillForCompression( + ebtype, tol, s, norm, config.decomposition, config.lossless, + config.huff_dict_size, config.huff_block_size, shape, + domain_decomposer.domain_decomposed(), config.domain_decomposition, + domain_decomposer.domain_decomposed_dim(), + domain_decomposer.domain_decomposed_size()); + } else { + m.FillForCompression( + ebtype, tol, s, norm, config.decomposition, config.lossless, + config.huff_dict_size, config.huff_block_size, shape, + domain_decomposer.domain_decomposed(), config.domain_decomposition, + domain_decomposer.domain_decomposed_dim(), + domain_decomposer.domain_decomposed_size(), coords); + } + + serizalied_meta = m.Serialize(metadata_size); + + // Serialize + byte_offset = 0; + Serialize( + (Byte *)compressed_data, serizalied_meta, metadata_size, byte_offset, 0); + DeviceRuntime::SyncQueue(0); + MemoryManager::Free(serizalied_meta); + byte_offset += compressed_subdomain_size; + DeviceRuntime::SyncQueue(0); + compressed_size = byte_offset; + + if (!input_previously_pinned && config.auto_pin_host_buffers) { + MemoryManager::HostUnregister((void *)original_data); + } + if (!output_previously_pinned && config.auto_pin_host_buffers) { + MemoryManager::HostUnregister((void *)compressed_data); + } + + if (config.auto_cache_release) + Cache::cache.SafeRelease(); + DeviceRuntime::Finalize(); + + if (log::level & log::TIME) { + timer_each.end(); + timer_each.print("Serialization"); + timer_each.clear(); + timer_total.end(); + timer_total.print("High-level compression", total_num_elem * sizeof(T)); + timer_total.clear(); + } + + return compress_status; +} + +// Wraps the compression pipeline and translates any exception thrown by the +// internal (de)compression steps into a compress_status_type failure code, so +// that the library never terminates the calling application via exit(). +template +enum compress_status_type +general_compress(std::vector shape, T tol, T s, + enum error_bound_type ebtype, const void *original_data, + void *&compressed_data, size_t &compressed_size, Config config, + bool uniform, std::vector coords, + bool output_pre_allocated) { + try { + return general_compress_pipeline( + shape, tol, s, ebtype, original_data, compressed_data, compressed_size, + config, uniform, coords, output_pre_allocated); + } catch (const Exception &e) { + log::err(std::string("compression failed: ") + e.what()); + return e.status(); + } catch (const std::exception &e) { + log::err(std::string("compression failed: ") + e.what()); + return compress_status_type::Failure; + } +} + +template +enum compress_status_type +compress(std::vector shape, T tol, T s, enum error_bound_type ebtype, + const void *original_data, void *&compressed_data, + size_t &compressed_size, Config config, bool output_pre_allocated) { + if (config.compressor == compressor_type::MGARD) { + if (config.decomposition != decomposition_type::Hybrid) { + return general_compress>( + shape, tol, s, ebtype, original_data, compressed_data, + compressed_size, config, true, std::vector(0), + output_pre_allocated); + } else { + return general_compress>( + shape, tol, s, ebtype, original_data, compressed_data, + compressed_size, config, true, std::vector(0), + output_pre_allocated); + } + } else if (config.compressor == compressor_type::ZFP) { + log::err("MGARD not built with external compressor ZFP"); + return compress_status_type::Failure; + } +} + +template +enum compress_status_type +compress(std::vector shape, T tol, T s, enum error_bound_type ebtype, + const void *original_data, void *&compressed_data, + size_t &compressed_size, Config config, std::vector coords, + bool output_pre_allocated) { + if (config.compressor == compressor_type::MGARD) { + if (config.decomposition != decomposition_type::Hybrid) { + return general_compress>( + shape, tol, s, ebtype, original_data, compressed_data, + compressed_size, config, false, coords, output_pre_allocated); + } else { + return general_compress>( + shape, tol, s, ebtype, original_data, compressed_data, + compressed_size, config, false, coords, output_pre_allocated); + } + } else if (config.compressor == compressor_type::ZFP) { + log::err("MGARD not built with external compressor ZFP"); + return compress_status_type::Failure; + } +} + +template +enum compress_status_type +general_decompress_pipeline(std::vector shape, + const void *compressed_data, size_t compressed_size, + void *&decompressed_data, Config config, + bool output_pre_allocated) { + DeviceRuntime::Initialize(); + size_t total_num_elem = 1; + for (int i = 0; i < D; i++) + total_num_elem *= shape[i]; + + config.apply(); + + Timer timer_total, timer_each; + if (log::level & log::TIME) + timer_total.start(); + if (log::level & log::TIME) + timer_each.start(); + + // Use consistance memory space between input and output data + if (!output_pre_allocated) { + if (MemoryManager::IsDevicePointer(compressed_data)) { + DeviceRuntime::SelectDevice( + MemoryManager::GetPointerDevice(compressed_data)); + MemoryManager::Malloc1D(decompressed_data, + total_num_elem * sizeof(T)); + } else { + decompressed_data = (void *)malloc(total_num_elem * sizeof(T)); + } + } + log::info("Output preallocated: " + std::to_string(output_pre_allocated)); + + bool input_previously_pinned = true; + if (!MemoryManager::IsDevicePointer((void *)compressed_data)) { + input_previously_pinned = + MemoryManager::CheckHostRegister((void *)compressed_data); + if (!input_previously_pinned && config.auto_pin_host_buffers) { + MemoryManager::HostRegister((void *)compressed_data, + compressed_size); + } + log::info("Input previously pinned: " + + std::to_string(input_previously_pinned)); + } else { + log::info("Input on device"); + } + bool output_previously_pinned = true; + if (!MemoryManager::IsDevicePointer((void *)decompressed_data)) { + output_previously_pinned = + MemoryManager::CheckHostRegister((void *)decompressed_data); + if (!output_previously_pinned && config.auto_pin_host_buffers) { + MemoryManager::HostRegister((void *)decompressed_data, + total_num_elem * sizeof(T)); + } + log::info("Output previously pinned: " + + std::to_string(output_previously_pinned)); + } else { + log::info("Output on device"); + } + + if (log::level & log::TIME) { + timer_each.end(); + timer_each.print("Prepare input and output buffer"); + timer_each.clear(); + } + + if (log::level & log::TIME) + timer_each.start(); + + Metadata m; + m.Deserialize((SERIALIZED_TYPE *)compressed_data); + m.InitializeConfig(config); + + std::vector coords(D); + if (m.dstype == data_structure_type::Cartesian_Grid_Non_Uniform) { + for (DIM d = 0; d < D; d++) { + coords[d] = new T[shape[d]]; + for (SIZE i = 0; i < shape[d]; i++) { + coords[d][i] = (float)m.coords[d][i]; + } + } + } + + if (config.compressor == compressor_type::MGARD) { + log::info("Compressor type: MGARD"); + } else if (config.compressor == compressor_type::ZFP) { + log::info("Compressor type: ZFP"); + } + + if (config.decomposition == decomposition_type::MultiDim) { + log::info("Multilevel Decomposition: multi-dimensional"); + } else if (config.decomposition == decomposition_type::SingleDim) { + log::info("Multilevel Decomposition: single-dimensional"); + } else if (config.decomposition == decomposition_type::Hybrid) { + log::info("Multilevel Decomposition: hybrid"); + } + log::info("tol: " + std::to_string(m.tol)); + log::info("s: " + std::to_string(m.s)); + + // Initialize DomainDecomposer + DomainDecomposer domain_decomposer; + + if (m.dstype == data_structure_type::Cartesian_Grid_Uniform) { + domain_decomposer = DomainDecomposer( + shape, m.domain_decomposed, m.domain_decomposed_dim, + m.domain_decomposed_size, config); + } else { + domain_decomposer = DomainDecomposer( + shape, m.domain_decomposed, m.domain_decomposed_dim, + m.domain_decomposed_size, config, coords); + } + domain_decomposer.set_original_data((T *)decompressed_data); + + using Cache = CompressorCache; + if (std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value) { + Cache::cache.SafeInitialize(2, 1); + } else { + Cache::cache.SafeInitialize(domain_decomposer.num_subdomains(), + domain_decomposer.num_subdomains()); + } + + // Preparing decompression parameters + T local_tol; + enum error_bound_type local_ebtype; + + if (log::level & log::TIME) + timer_each.start(); + + if (!m.domain_decomposed) { + local_tol = m.tol; + local_ebtype = m.ebtype; + } else { + local_tol = calc_local_abs_tol(m.ebtype, m.norm, m.tol, m.s, + domain_decomposer.num_subdomains()); + // Force to use ABS mode when do domain decomposition + local_ebtype = error_bound_type::ABS; + } + + // Deserialize compressed data + Byte *compressed_subdomain_data = (Byte *)compressed_data + m.metadata_size; + + if (log::level & log::TIME) { + timer_each.end(); + timer_each.print("Deserialization"); + timer_each.clear(); + } + enum compress_status_type decompress_status; + DeviceRuntime::SelectDevice(config.dev_id); + + if constexpr (std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value) { + decompress_status = decompress_pipeline_gpu( + domain_decomposer, local_tol, (T)m.s, (T)m.norm, local_ebtype, config, + compressed_subdomain_data); + } else { + decompress_status = decompress_pipeline_cpu( + domain_decomposer, local_tol, (T)m.s, (T)m.norm, local_ebtype, config, + compressed_subdomain_data); + } + + if (!input_previously_pinned && config.auto_pin_host_buffers) { + MemoryManager::HostUnregister((void *)compressed_data); + } + if (!output_previously_pinned && config.auto_pin_host_buffers) { + MemoryManager::HostUnregister((void *)decompressed_data); + } + + if (m.dstype == data_structure_type::Cartesian_Grid_Non_Uniform) { + for (DIM d = 0; d < D; d++) + delete[] coords[d]; + } + + if (config.auto_cache_release) + Cache::cache.SafeRelease(); + DeviceRuntime::Finalize(); + + if (log::level & log::TIME) { + timer_total.end(); + timer_total.print("High-level decompression", total_num_elem * sizeof(T)); + timer_total.clear(); + } + + return decompress_status; +} + +// Wraps the decompression pipeline and translates any exception thrown by the +// internal steps (e.g. malformed metadata, lossless backend errors) into a +// compress_status_type failure code instead of terminating via exit(). +template +enum compress_status_type +general_decompress(std::vector shape, const void *compressed_data, + size_t compressed_size, void *&decompressed_data, + Config config, bool output_pre_allocated) { + try { + return general_decompress_pipeline( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } catch (const Exception &e) { + log::err(std::string("decompression failed: ") + e.what()); + return e.status(); + } catch (const std::exception &e) { + log::err(std::string("decompression failed: ") + e.what()); + return compress_status_type::Failure; + } +} + +template +enum compress_status_type +decompress(std::vector shape, const void *compressed_data, + size_t compressed_size, void *&decompressed_data, Config config, + bool output_pre_allocated) { + // MGARD 1.6.0 wrote Huffman+LZ4 with NVIDIA nvCOMP's LZ4 format, which this + // build does not depend on (its LZ4 backend is a different, portable format). + if (config.lossless == lossless_type::Huffman_LZ4) { + log::err("Huffman+LZ4 data compressed by MGARD 1.6.0 (nvCOMP LZ4) is not " + "supported."); + return compress_status_type::Failure; + } + if (config.compressor == compressor_type::MGARD) { + if (config.decomposition != decomposition_type::Hybrid) { + return general_decompress>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else { + // MGARD 1.6.0 did not record the hybrid hierarchy parameters in its + // header, so its hybrid streams cannot be reconstructed. + log::err("hybrid-hierarchy data compressed by MGARD 1.6.0 is not " + "supported."); + return compress_status_type::Failure; + } + } else if (config.compressor == compressor_type::ZFP) { + log::err("MGARD not built with external compressor ZFP"); + return compress_status_type::Failure; + } +} + +template +enum compress_status_type +compress(DIM D, data_type dtype, std::vector shape, double tol, double s, + enum error_bound_type mode, const void *original_data, + void *&compressed_data, size_t &compressed_size, Config config, + bool output_pre_allocated) { + if (dtype == data_type::Float) { + if (D == 1) { + return compress<1, float, DeviceType>(shape, tol, s, mode, original_data, + compressed_data, compressed_size, + config, output_pre_allocated); + } else if (D == 2) { + return compress<2, float, DeviceType>(shape, tol, s, mode, original_data, + compressed_data, compressed_size, + config, output_pre_allocated); + } else if (D == 3) { + return compress<3, float, DeviceType>(shape, tol, s, mode, original_data, + compressed_data, compressed_size, + config, output_pre_allocated); + } else if (D == 4) { + return compress<4, float, DeviceType>(shape, tol, s, mode, original_data, + compressed_data, compressed_size, + config, output_pre_allocated); + } else if (D == 5) { + return compress<5, float, DeviceType>(shape, tol, s, mode, original_data, + compressed_data, compressed_size, + config, output_pre_allocated); + } else { + return compress_status_type::NotSupportHigherNumberOfDimensionsFailure; + } + } else if (dtype == data_type::Double) { + if (D == 1) { + return compress<1, double, DeviceType>(shape, tol, s, mode, original_data, + compressed_data, compressed_size, + config, output_pre_allocated); + } else if (D == 2) { + return compress<2, double, DeviceType>(shape, tol, s, mode, original_data, + compressed_data, compressed_size, + config, output_pre_allocated); + } else if (D == 3) { + return compress<3, double, DeviceType>(shape, tol, s, mode, original_data, + compressed_data, compressed_size, + config, output_pre_allocated); + } else if (D == 4) { + return compress<4, double, DeviceType>(shape, tol, s, mode, original_data, + compressed_data, compressed_size, + config, output_pre_allocated); + } else if (D == 5) { + return compress<5, double, DeviceType>(shape, tol, s, mode, original_data, + compressed_data, compressed_size, + config, output_pre_allocated); + } else { + return compress_status_type::NotSupportHigherNumberOfDimensionsFailure; + } + } else { + return compress_status_type::NotSupportDataTypeFailure; + } +} + +template +enum compress_status_type +compress(DIM D, data_type dtype, std::vector shape, double tol, double s, + enum error_bound_type mode, const void *original_data, + void *&compressed_data, size_t &compressed_size, + bool output_pre_allocated) { + + Config config; + return compress(D, dtype, shape, tol, s, mode, original_data, + compressed_data, compressed_size, config, + output_pre_allocated); +} + +template +enum compress_status_type +compress(DIM D, data_type dtype, std::vector shape, double tol, double s, + enum error_bound_type mode, const void *original_data, + void *&compressed_data, size_t &compressed_size, + std::vector coords, Config config, + bool output_pre_allocated) { + + if (dtype == data_type::Float) { + std::vector float_coords; + for (auto &coord : coords) + float_coords.push_back((float *)coord); + if (D == 1) { + return compress<1, float, DeviceType>( + shape, tol, s, mode, original_data, compressed_data, compressed_size, + config, float_coords, output_pre_allocated); + } else if (D == 2) { + return compress<2, float, DeviceType>( + shape, tol, s, mode, original_data, compressed_data, compressed_size, + config, float_coords, output_pre_allocated); + } else if (D == 3) { + return compress<3, float, DeviceType>( + shape, tol, s, mode, original_data, compressed_data, compressed_size, + config, float_coords, output_pre_allocated); + } else if (D == 4) { + return compress<4, float, DeviceType>( + shape, tol, s, mode, original_data, compressed_data, compressed_size, + config, float_coords, output_pre_allocated); + } else if (D == 5) { + return compress<5, float, DeviceType>( + shape, tol, s, mode, original_data, compressed_data, compressed_size, + config, float_coords, output_pre_allocated); + } else { + return compress_status_type::NotSupportHigherNumberOfDimensionsFailure; + } + } else if (dtype == data_type::Double) { + std::vector double_coords; + for (auto &coord : coords) + double_coords.push_back((double *)coord); + if (D == 1) { + return compress<1, double, DeviceType>( + shape, tol, s, mode, original_data, compressed_data, compressed_size, + config, double_coords, output_pre_allocated); + } else if (D == 2) { + return compress<2, double, DeviceType>( + shape, tol, s, mode, original_data, compressed_data, compressed_size, + config, double_coords, output_pre_allocated); + } else if (D == 3) { + return compress<3, double, DeviceType>( + shape, tol, s, mode, original_data, compressed_data, compressed_size, + config, double_coords, output_pre_allocated); + } else if (D == 4) { + return compress<4, double, DeviceType>( + shape, tol, s, mode, original_data, compressed_data, compressed_size, + config, double_coords, output_pre_allocated); + } else if (D == 5) { + return compress<5, double, DeviceType>( + shape, tol, s, mode, original_data, compressed_data, compressed_size, + config, double_coords, output_pre_allocated); + } else { + return compress_status_type::NotSupportHigherNumberOfDimensionsFailure; + } + } else { + return compress_status_type::NotSupportDataTypeFailure; + } +} + +template +enum compress_status_type +compress(DIM D, data_type dtype, std::vector shape, double tol, double s, + enum error_bound_type mode, const void *original_data, + void *&compressed_data, size_t &compressed_size, + std::vector coords, bool output_pre_allocated) { + Config config; + return compress(D, dtype, shape, tol, s, mode, original_data, + compressed_data, compressed_size, coords, config, + output_pre_allocated); +} + +template +enum compress_status_type +decompress(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, Config config, bool output_pre_allocated) { + Metadata meta; + meta.Deserialize((SERIALIZED_TYPE *)compressed_data); + meta.InitializeConfig(config); + + std::vector shape(meta.total_dims); + for (DIM d = 0; d < shape.size(); d++) + shape[d] = (SIZE)meta.shape[d]; + data_type dtype = meta.dtype; + if (dtype == data_type::Float) { + if (shape.size() == 1) { + return decompress<1, float, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else if (shape.size() == 2) { + return decompress<2, float, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else if (shape.size() == 3) { + return decompress<3, float, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else if (shape.size() == 4) { + return decompress<4, float, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else if (shape.size() == 5) { + return decompress<5, float, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else { + return compress_status_type::NotSupportHigherNumberOfDimensionsFailure; + } + } else if (dtype == data_type::Double) { + if (shape.size() == 1) { + enum compress_status_type s = decompress<1, double, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + return s; + } else if (shape.size() == 2) { + return decompress<2, double, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else if (shape.size() == 3) { + return decompress<3, double, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else if (shape.size() == 4) { + return decompress<4, double, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else if (shape.size() == 5) { + return decompress<5, double, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else { + return compress_status_type::NotSupportHigherNumberOfDimensionsFailure; + } + } else { + return compress_status_type::NotSupportDataTypeFailure; + } +} + +template +enum compress_status_type +decompress(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, bool output_pre_allocated) { + Config config; + return decompress(compressed_data, compressed_size, + decompressed_data, config, + output_pre_allocated); +} + +template +enum compress_status_type decompress(const void *compressed_data, + size_t compressed_size, + void *&decompressed_data, data_type &dtype, + std::vector &shape, + Config config, bool output_pre_allocated) { + Metadata meta; + meta.Deserialize((SERIALIZED_TYPE *)compressed_data); + meta.InitializeConfig(config); + + shape = std::vector(meta.total_dims); + for (DIM d = 0; d < shape.size(); d++) + shape[d] = (SIZE)meta.shape[d]; + dtype = meta.dtype; + + if (dtype == data_type::Float) { + if (shape.size() == 1) { + return decompress<1, float, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else if (shape.size() == 2) { + return decompress<2, float, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else if (shape.size() == 3) { + return decompress<3, float, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else if (shape.size() == 4) { + return decompress<4, float, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else if (shape.size() == 5) { + return decompress<5, float, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else { + return compress_status_type::NotSupportHigherNumberOfDimensionsFailure; + } + } else if (dtype == data_type::Double) { + if (shape.size() == 1) { + return decompress<1, double, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else if (shape.size() == 2) { + return decompress<2, double, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else if (shape.size() == 3) { + return decompress<3, double, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else if (shape.size() == 4) { + return decompress<4, double, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else if (shape.size() == 5) { + return decompress<5, double, DeviceType>( + shape, compressed_data, compressed_size, decompressed_data, config, + output_pre_allocated); + } else { + return compress_status_type::NotSupportHigherNumberOfDimensionsFailure; + } + } else { + return compress_status_type::NotSupportDataTypeFailure; + } +} + +template +enum compress_status_type +decompress(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, data_type &dtype, + std::vector &shape, bool output_pre_allocated) { + Config config; + return decompress(compressed_data, compressed_size, + decompressed_data, dtype, shape, config, + output_pre_allocated); +} + +template void release_cache() { + using Cache1 = + CompressorCache>; + Cache1::cache.SafeRelease(); +} + +template void release_cache() { + release_cache<1, T, DeviceType>(); + release_cache<2, T, DeviceType>(); + release_cache<3, T, DeviceType>(); + release_cache<4, T, DeviceType>(); + release_cache<5, T, DeviceType>(); +} + +template enum compress_status_type release_cache() { + release_cache(); + release_cache(); + return compress_status_type::Success; +} + +template void pin_memory(void *ptr, SIZE num_bytes) { + MemoryManager::HostRegister((Byte *)ptr, num_bytes); +} + +template bool check_memory_pinned(void *ptr) { + return MemoryManager::CheckHostRegister((Byte *)ptr); +} + +template void unpin_memory(void *ptr) { + MemoryManager::HostUnregister((Byte *)ptr); +} + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/CompressionHighLevel/ErrorToleranceCalculator.hpp b/include/mgard-x-160/CompressionHighLevel/ErrorToleranceCalculator.hpp new file mode 100644 index 000000000..ea9a56824 --- /dev/null +++ b/include/mgard-x-160/CompressionHighLevel/ErrorToleranceCalculator.hpp @@ -0,0 +1,148 @@ +namespace mgard_x_160 { +template +T calc_subdomain_norm_series_w_prefetch( + DomainDecomposer &domain_decomposer, + T s) { + + DeviceRuntime::SyncQueue(0); + Array<1, T, DeviceType> norm_array({1}); + SubArray<1, T, DeviceType> norm_subarray(norm_array); + T norm = 0; + + // Two buffers one for current and one for next + Array device_subdomain_buffer[2]; + // Pre-allocate to the size of the first subdomain + // Following subdomains should be no bigger than the first one + // We shouldn't need to reallocate in the future + device_subdomain_buffer[0].resize(domain_decomposer.subdomain_shape(0)); + device_subdomain_buffer[1].resize(domain_decomposer.subdomain_shape(0)); + + // Pre-fetch the first subdomain to one buffer + int current_buffer = 0; + domain_decomposer.copy_subdomain( + device_subdomain_buffer[current_buffer], 0, + subdomain_copy_direction::OriginalToSubdomain, 0); + + for (SIZE curr_subdomain_id = 0; + curr_subdomain_id < domain_decomposer.num_subdomains(); + curr_subdomain_id++) { + SIZE next_subdomain_id; + int next_buffer = (current_buffer + 1) % 2; + if (curr_subdomain_id + 1 < domain_decomposer.num_subdomains()) { + // Prefetch the next subdomain + next_subdomain_id = curr_subdomain_id + 1; + // Copy data + domain_decomposer.copy_subdomain( + device_subdomain_buffer[next_buffer], next_subdomain_id, + subdomain_copy_direction::OriginalToSubdomain, 1); + } + + // device_input_buffer has to be not pitched to avoid copy for + // linearization + assert(!device_subdomain_buffer[current_buffer].isPitched()); + // Disable normalize_coordinate since we do not want to void dividing + // total_elems + T local_norm = + norm_calculator(device_subdomain_buffer[current_buffer], + SubArray<1, T, DeviceType>(), norm_subarray, s, false); + if (s == std::numeric_limits::infinity()) { + norm = std::max(norm, local_norm); + } else { + norm += local_norm * local_norm; + } + current_buffer = next_buffer; + DeviceRuntime::SyncQueue(1); + } + + DeviceRuntime::SyncDevice(); + return norm; +} + +template +T calc_norm_decomposed_w_prefetch( + DomainDecomposer &domain_decomposer, T s, + bool normalize_coordinates, SIZE total_num_elem) { + + // Process a series of subdomains according to the subdomain id list + T norm = calc_subdomain_norm_series_w_prefetch(domain_decomposer, s); + if (s != std::numeric_limits::infinity()) { + if (!normalize_coordinates) { + norm = std::sqrt(norm); + } else { + norm = std::sqrt(norm / total_num_elem); + } + } + if (s == std::numeric_limits::infinity()) { + log::info("L_inf norm: " + std::to_string(norm)); + } else { + log::info("L_2 norm: " + std::to_string(norm)); + } + return norm; +} + +template +T calc_norm_decomposed( + DomainDecomposer &domain_decomposer, T s, + bool normalize_coordinates, SIZE total_num_elem) { + Array device_input_buffer; + T norm = 0; + for (int subdomain_id = 0; subdomain_id < domain_decomposer.num_subdomains(); + subdomain_id++) { + domain_decomposer.copy_subdomain( + device_input_buffer, subdomain_id, + subdomain_copy_direction::OriginalToSubdomain, 0); + DeviceRuntime::SyncQueue(0); + Array<1, T, DeviceType> norm_array({1}); + SubArray<1, T, DeviceType> norm_subarray(norm_array); + // device_input_buffer has to be not pitched to avoid copy for + // linearization + assert(!device_input_buffer.isPitched()); + // Disable normalize_coordinate since we do not want to void dividing + // total_elems + T local_norm = + norm_calculator(device_input_buffer, SubArray<1, T, DeviceType>(), + norm_subarray, s, false); + if (s == std::numeric_limits::infinity()) { + norm = std::max(norm, local_norm); + } else { + norm += local_norm * local_norm; + } + } + if (s != std::numeric_limits::infinity()) { + if (!normalize_coordinates) { + norm = std::sqrt(norm); + } else { + norm = std::sqrt(norm / total_num_elem); + } + } + if (s == std::numeric_limits::infinity()) { + log::info("L_inf norm: " + std::to_string(norm)); + } else { + log::info("L_2 norm: " + std::to_string(norm)); + } + return norm; +} + +template +T calc_local_abs_tol(enum error_bound_type ebtype, T norm, T tol, T s, + SIZE num_subdomain) { + T local_abs_tol; + if (ebtype == error_bound_type::REL) { + if (s == std::numeric_limits::infinity()) { + log::info("L_inf norm: " + std::to_string(norm)); + local_abs_tol = tol * norm; + } else { + log::info("L_2 norm: " + std::to_string(norm)); + local_abs_tol = std::sqrt((tol * norm) * (tol * norm) / num_subdomain); + } + } else { + if (s == std::numeric_limits::infinity()) { + local_abs_tol = tol; + } else { + local_abs_tol = std::sqrt((tol * tol) / num_subdomain); + } + } + log::info("local abs tol: " + std::to_string(local_abs_tol)); + return local_abs_tol; +} +} // namespace mgard_x_160 \ No newline at end of file diff --git a/include/mgard-x-160/CompressionHighLevel/GPUPipelines.hpp b/include/mgard-x-160/CompressionHighLevel/GPUPipelines.hpp new file mode 100644 index 000000000..e46c87aa4 --- /dev/null +++ b/include/mgard-x-160/CompressionHighLevel/GPUPipelines.hpp @@ -0,0 +1,590 @@ +namespace mgard_x_160 { +template +enum compress_status_type compress_pipeline_gpu( + DomainDecomposer &domain_decomposer, + T local_tol, T s, T &norm, enum error_bound_type local_ebtype, + Config &config, Byte *compressed_subdomain_data, + SIZE &compressed_subdomain_size) { + Timer timer_series; + if (log::level & log::TIME) + timer_series.start(); + + using Cache = CompressorCache; + using HierarchyType = typename CompressorType::HierarchyType; + CompressorType &compressor = Cache::cache.compressor[0]; + Array *device_subdomain_buffer = + Cache::cache.device_subdomain_buffer; + Array<1, Byte, DeviceType> *device_compressed_buffer = + Cache::cache.device_compressed_buffer; + + if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0), + domain_decomposer.uniform, config)) { + Cache::cache.ClearHierarchyCache(); + } + + for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { + if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id), + domain_decomposer.uniform, config)) { + Cache::cache.InsertHierarchyCache( + domain_decomposer.subdomain_hierarchy(id)); + } + } + log::info("Adjust device buffers"); + std::vector shape = + domain_decomposer.subdomain_shape(domain_decomposer.largest_subdomain()); + SIZE num_elements = 1; + for (int i = 0; i < shape.size(); i++) + num_elements *= shape[i]; + device_subdomain_buffer[0].resize(shape); + device_subdomain_buffer[1].resize(shape); + device_compressed_buffer[0].resize( + {domain_decomposer.subdomain_compressed_buffer_size( + domain_decomposer.largest_subdomain())}); + device_compressed_buffer[1].resize( + {domain_decomposer.subdomain_compressed_buffer_size( + domain_decomposer.largest_subdomain())}); + + HierarchyType &hierarchy = Cache::cache.GetHierarchyCache( + domain_decomposer.subdomain_shape(domain_decomposer.largest_subdomain())); + log::info("Adapt Compressor to hierarchy"); + compressor.Adapt(hierarchy, config, 0); + + DeviceRuntime::SyncDevice(); + + if (log::level & log::TIME) { + timer_series.end(); + timer_series.print("Prepare device environment"); + timer_series.clear(); + timer_series.start(); + } + + Timer timer_profile; + std::vector h2d, d2h, comp; + std::vector size; + bool profile = false; + bool profile_e2e = false; + + // For serialization + SIZE byte_offset = 0; + + // Pre-fetch the first subdomain to one buffer + int current_buffer = 0; + int current_queue = 0; + + if (profile || profile_e2e) { + DeviceRuntime::SyncDevice(); + timer_profile.clear(); + timer_profile.start(); + } + domain_decomposer.copy_subdomain( + device_subdomain_buffer[current_buffer], 0, + subdomain_copy_direction::OriginalToSubdomain, current_queue); + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + h2d.push_back(timer_profile.get()); + } + + for (SIZE curr_subdomain_id = 0; + curr_subdomain_id < domain_decomposer.num_subdomains(); + curr_subdomain_id++) { + SIZE next_subdomain_id; + int next_buffer = (current_buffer + 1) % 2; + int next_queue = (current_queue + 1) % 3; + HierarchyType &hierarchy = Cache::cache.GetHierarchyCache( + domain_decomposer.subdomain_shape(curr_subdomain_id)); + log::info("Adapt Compressor to hierarchy"); + compressor.Adapt(hierarchy, config, current_queue); + + if (curr_subdomain_id + 1 < domain_decomposer.num_subdomains()) { + // Prefetch the next subdomain + next_subdomain_id = curr_subdomain_id + 1; + // Copy data + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.clear(); + timer_profile.start(); + } + domain_decomposer.copy_subdomain( + device_subdomain_buffer[next_buffer], next_subdomain_id, + subdomain_copy_direction::OriginalToSubdomain, next_queue); + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + h2d.push_back(timer_profile.get()); + } + } + + std::stringstream ss; + for (DIM d = 0; d < D; d++) { + ss << compressor.hierarchy->level_shape(compressor.hierarchy->l_target(), + d) + << " "; + } + log::info("Compressing subdomain " + std::to_string(curr_subdomain_id) + + " with shape: " + ss.str()); + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.clear(); + timer_profile.start(); + } + compressor.Compress( + device_subdomain_buffer[current_buffer], local_ebtype, local_tol, s, + norm, device_compressed_buffer[current_buffer], current_queue); + + SIZE compressed_size = device_compressed_buffer[current_buffer].shape(0); + double CR = (double)compressor.hierarchy->total_num_elems() * sizeof(T) / + compressed_size; + log::info("Subdomain CR: " + std::to_string(CR)); + if (CR < 1.0) { + log::info("Using uncompressed data instead"); + domain_decomposer.copy_subdomain( + device_subdomain_buffer[current_buffer], curr_subdomain_id, + subdomain_copy_direction::OriginalToSubdomain, current_queue); + SIZE linearized_width = 1; + for (DIM d = 0; d < D - 1; d++) + linearized_width *= device_subdomain_buffer[current_buffer].shape(d); + MemoryManager::CopyND( + device_compressed_buffer[current_buffer].data(), + device_subdomain_buffer[current_buffer].shape(D - 1) * sizeof(T), + (Byte *)device_subdomain_buffer[current_buffer].data(), + device_subdomain_buffer[current_buffer].ld(D - 1) * sizeof(T), + device_subdomain_buffer[current_buffer].shape(D - 1) * sizeof(T), + linearized_width, current_queue); + compressed_size = compressor.hierarchy->total_num_elems() * sizeof(T); + } + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + comp.push_back(timer_profile.get()); + } + + size.push_back(compressor.hierarchy->total_num_elems() * sizeof(T)); + + // Check if we have enough space + if (compressed_size > + compressed_subdomain_size - byte_offset - sizeof(SIZE)) { + log::err("Output too large (original size: " + + std::to_string((double)compressor.hierarchy->total_num_elems() * + sizeof(T) / 1e9) + + " GB, compressed size: " + + std::to_string((double)compressed_size / 1e9) + + " GB, leftover buffer space: " + + std::to_string((double)(compressed_subdomain_size - byte_offset - + sizeof(SIZE)) / + 1e9) + + " GB)"); + return compress_status_type::OutputTooLargeFailure; + } + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.clear(); + timer_profile.start(); + } + Serialize(compressed_subdomain_data, &compressed_size, 1, + byte_offset, current_queue); + Serialize(compressed_subdomain_data, + device_compressed_buffer[current_buffer].data(), + compressed_size, byte_offset, current_queue); + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + d2h.push_back(timer_profile.get()); + } + + if (config.compress_with_dryrun) { + domain_decomposer.copy_subdomain( + device_subdomain_buffer[current_buffer], curr_subdomain_id, + subdomain_copy_direction::SubdomainToOriginal, current_queue); + } + current_buffer = next_buffer; + current_queue = next_queue; + } + + SIZE total_size = 0; + for (auto t : size) + total_size += t; + + if (profile_e2e) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + timer_profile.print("end to end"); + + timer_profile.print_throughput("end to end", total_size * 1e9); + } + + if (profile) { + std::cout << "comp: " + << "\n"; + for (float t : comp) + std::cout << t << ", "; + std::cout << "\n"; + + std::cout << "h2d: " + << "\n"; + for (float t : h2d) + std::cout << t << ", "; + std::cout << "\n"; + + std::cout << "d2h: " + << "\n"; + for (float t : d2h) + std::cout << t << ", "; + std::cout << "\n"; + + std::cout << "size: " + << "\n"; + for (SIZE t : size) + std::cout << t << ", "; + std::cout << "\n"; + + std::cout << "comp_speed: " + << "\n"; + for (int i = 0; i < comp.size(); i++) + std::cout << size[i] / comp[i] << ", "; + std::cout << "\n"; + } + + compressed_subdomain_size = byte_offset; + DeviceRuntime::SyncDevice(); + if (log::level & log::TIME) { + timer_series.end(); + timer_series.print("Compress pipeline", total_size); + timer_series.clear(); + } + return compress_status_type::Success; +} + +template +enum compress_status_type decompress_pipeline_gpu( + DomainDecomposer &domain_decomposer, + T local_tol, T s, T norm, enum error_bound_type local_ebtype, + Config &config, Byte *compressed_subdomain_data) { + Timer timer_series; + if (log::level & log::TIME) + timer_series.start(); + + SIZE byte_offset = 0; + using Cache = CompressorCache; + using HierarchyType = typename CompressorType::HierarchyType; + CompressorType &compressor = Cache::cache.compressor[0]; + Array *device_subdomain_buffer = + Cache::cache.device_subdomain_buffer; + Array<1, Byte, DeviceType> *device_compressed_buffer = + Cache::cache.device_compressed_buffer; + + if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0), + domain_decomposer.uniform, config)) { + Cache::cache.ClearHierarchyCache(); + } + + for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) { + if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id), + domain_decomposer.uniform, config)) { + Cache::cache.InsertHierarchyCache( + domain_decomposer.subdomain_hierarchy(id)); + } + } + + log::info("Adjust device buffers"); + std::vector shape = + domain_decomposer.subdomain_shape(domain_decomposer.largest_subdomain()); + SIZE num_elements = 1; + for (int i = 0; i < shape.size(); i++) + num_elements *= shape[i]; + device_subdomain_buffer[0].resize(shape); + device_subdomain_buffer[1].resize(shape); + device_compressed_buffer[0].resize( + {domain_decomposer.subdomain_compressed_buffer_size( + domain_decomposer.largest_subdomain())}); + device_compressed_buffer[1].resize( + {domain_decomposer.subdomain_compressed_buffer_size( + domain_decomposer.largest_subdomain())}); + + HierarchyType &hierarchy = Cache::cache.GetHierarchyCache( + domain_decomposer.subdomain_shape(domain_decomposer.largest_subdomain())); + log::info("Adapt Compressor to hierarchy"); + compressor.Adapt(hierarchy, config, 0); + + DeviceRuntime::SyncDevice(); + + if (log::level & log::TIME) { + timer_series.end(); + timer_series.print("Prepare device environment"); + timer_series.clear(); + timer_series.start(); + } + + Timer timer_profile; + Timer timer_decompress_kernel; + std::vector h2d, d2h, comp; + std::vector size; + bool profile = false; + bool profile_e2e = false; + + // Pre-fetch the first subdomain on queue 0 + int current_buffer = 0; + int current_queue = 0; + + if (profile || profile_e2e) { + DeviceRuntime::SyncDevice(); + timer_profile.clear(); + timer_profile.start(); + } + + SIZE compressed_size; + SIZE *compressed_size_ptr = &compressed_size; + Byte *compressed_data; + + Deserialize(compressed_subdomain_data, compressed_size_ptr, + 1, byte_offset, false, current_queue); + DeviceRuntime::SyncQueue(current_queue); + Deserialize(compressed_subdomain_data, compressed_data, + compressed_size, byte_offset, true, + current_queue); + + device_compressed_buffer[current_buffer].resize({compressed_size}); + + MemoryManager::Copy1D( + device_compressed_buffer[current_buffer].data(), compressed_data, + compressed_size, current_queue); + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + h2d.push_back(timer_profile.get()); + } + + for (SIZE curr_subdomain_id = 0; + curr_subdomain_id < domain_decomposer.num_subdomains(); + curr_subdomain_id++) { + SIZE next_subdomain_id; + int next_buffer = (current_buffer + 1) % 2; + int next_queue = (current_queue + 1) % 3; + HierarchyType &hierarchy = Cache::cache.GetHierarchyCache( + domain_decomposer.subdomain_shape(curr_subdomain_id)); + log::info("Adapt Compressor to hierarchy"); + compressor.Adapt(hierarchy, config, current_queue); + + if (curr_subdomain_id + 1 < domain_decomposer.num_subdomains()) { + // Prefetch the next subdomain + next_subdomain_id = curr_subdomain_id + 1; + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.clear(); + timer_profile.start(); + } + + // Deserialize and copy next compressed data on queue 1 + SIZE compressed_size; + SIZE *compressed_size_ptr = &compressed_size; + Byte *compressed_data; + + Deserialize(compressed_subdomain_data, + compressed_size_ptr, 1, byte_offset, false, + next_queue); + DeviceRuntime::SyncQueue(next_queue); + Deserialize(compressed_subdomain_data, compressed_data, + compressed_size, byte_offset, true, + next_queue); + + device_compressed_buffer[next_buffer].resize({compressed_size}); + + MemoryManager::Copy1D( + device_compressed_buffer[next_buffer].data(), compressed_data, + compressed_size, next_queue); + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + h2d.push_back(timer_profile.get()); + } + } + + double CR = (double)compressor.hierarchy->total_num_elems() * sizeof(T) / + device_compressed_buffer[current_buffer].shape(0); + log::info("Subdomain CR: " + std::to_string(CR)); + if (CR > 1.0) { + std::stringstream ss; + for (DIM d = 0; d < D; d++) { + ss << compressor.hierarchy->level_shape( + compressor.hierarchy->l_target(), d) + << " "; + } + log::info("Decompressing subdomain " + std::to_string(curr_subdomain_id) + + " with shape: " + ss.str()); + compressor.Deserialize(device_compressed_buffer[current_buffer], + current_queue); + } + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.clear(); + timer_profile.start(); + } + + if (curr_subdomain_id > 0) { + // We delay D2H since since it can delay the D2H in lossless decompession + // and dequantization + int previous_buffer = std::abs((current_buffer - 1) % 2); + int previous_queue = std::abs((current_queue - 1) % 3); + SIZE prev_subdomain_id = curr_subdomain_id - 1; + domain_decomposer.copy_subdomain( + device_subdomain_buffer[previous_buffer], prev_subdomain_id, + subdomain_copy_direction::SubdomainToOriginal, previous_queue); + } + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + d2h.push_back(timer_profile.get()); + } + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.clear(); + timer_profile.start(); + } + if (CR > 1.0) { + if (log::level & log::TIME) { + DeviceRuntime::SyncDevice(); + timer_decompress_kernel.clear(); + timer_decompress_kernel.start(); + } + compressor.LosslessDecompress(device_compressed_buffer[current_buffer], + current_queue); + compressor.DequantizeRecompose(device_subdomain_buffer[current_buffer], + local_ebtype, local_tol, s, norm, + current_queue); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(0); + timer_decompress_kernel.end(); + timer_decompress_kernel.print("Decompression Kernel"); + timer_decompress_kernel.clear(); + } + } else { + log::info("Skipping decompression as original data was saved instead"); + // Print info here + // log::info("The compression ration is "+std::to_string(CR)); + device_subdomain_buffer[current_buffer].resize( + {compressor.hierarchy->level_shape( + compressor.hierarchy->l_target())}); + SIZE linearized_width = 1; + for (DIM d = 0; d < D - 1; d++) + linearized_width *= device_subdomain_buffer[current_buffer].shape(d); + MemoryManager::CopyND( + device_subdomain_buffer[current_buffer].data(), + device_subdomain_buffer[current_buffer].ld(D - 1), + (T *)device_compressed_buffer[current_buffer].data(), + device_subdomain_buffer[current_buffer].shape(D - 1), + device_subdomain_buffer[current_buffer].shape(D - 1), + linearized_width, current_queue); + } + + // compressor.Dequantize(device_subdomain_buffer[current_buffer], + // local_ebtype, local_tol, s, norm, current_queue); + + // compressor.Recompose(device_subdomain_buffer[current_buffer], + // current_queue); + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + comp.push_back(timer_profile.get()); + } + + size.push_back(compressor.hierarchy->total_num_elems() * sizeof(T)); + + // Need to ensure decompession is complete without blocking other operations + DeviceRuntime::SyncQueue(current_queue); + current_buffer = next_buffer; + current_queue = next_queue; + } + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.clear(); + timer_profile.start(); + } + + // Copy the last subdomain + int previous_buffer = std::abs((current_buffer - 1) % 2); + int previous_queue = previous_buffer; + SIZE prev_subdomain_id = domain_decomposer.num_subdomains() - 1; + // Add resizing for device_subdomain_buffer + device_subdomain_buffer[previous_buffer].resize( + {compressor.hierarchy->level_shape(compressor.hierarchy->l_target())}); + domain_decomposer.copy_subdomain( + device_subdomain_buffer[previous_buffer], prev_subdomain_id, + subdomain_copy_direction::SubdomainToOriginal, previous_queue); + + SIZE total_size = 0; + for (auto t : size) + total_size += t; + + if (profile) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + d2h.push_back(timer_profile.get()); + } + + if (profile_e2e) { + DeviceRuntime::SyncDevice(); + timer_profile.end(); + timer_profile.print("end to end"); + timer_profile.print_throughput("end to end", total_size * 1e9); + } + + if (profile) { + // double total_size = domain_decomposer.shape[0] * + // domain_decomposer.shape[1] * domain_decomposer.shape[2] * sizeof(T) / + // 1e9; std::cout << "comp: " << comp / domain_decomposer.num_subdomains() + // << "(" << total_size / comp << " GB/s)"<< "\n"; std::cout << "h2d: " << + // h2d / domain_decomposer.num_subdomains() << "(" << total_size / h2d << " + // GB/s)"<< "\n"; std::cout << "d2h: " << d2h / + // domain_decomposer.num_subdomains() << "(" << byte_offset/ 1e9 / d2h << " + // GB/s)"<< "\n"; + std::cout << "comp: " + << "\n"; + for (float t : comp) + std::cout << t << ", "; + std::cout << "\n"; + + std::cout << "h2d: " + << "\n"; + for (float t : h2d) + std::cout << t << ", "; + std::cout << "\n"; + + std::cout << "d2h: " + << "\n"; + for (float t : d2h) + std::cout << t << ", "; + std::cout << "\n"; + + std::cout << "size: " + << "\n"; + for (SIZE t : size) + std::cout << t << ", "; + std::cout << "\n"; + + std::cout << "comp_speed: " + << "\n"; + for (int i = 0; i < comp.size(); i++) + std::cout << size[i] / comp[i] << ", "; + std::cout << "\n"; + } + + DeviceRuntime::SyncDevice(); + if (log::level & log::TIME) { + timer_series.end(); + timer_series.print("Decompress pipeline", total_size); + timer_series.clear(); + } + return compress_status_type::Success; +} +} // namespace mgard_x_160 \ No newline at end of file diff --git a/include/mgard-x-160/CompressionHighLevel/ShapeAdjustment.hpp b/include/mgard-x-160/CompressionHighLevel/ShapeAdjustment.hpp new file mode 100644 index 000000000..17131b5b8 --- /dev/null +++ b/include/mgard-x-160/CompressionHighLevel/ShapeAdjustment.hpp @@ -0,0 +1,79 @@ +namespace mgard_x_160 { +template int max_dim(std::vector &shape) { + int max_d = 0; + T max_n = 0; + for (int i = 0; i < shape.size(); i++) { + if (max_n < shape[i]) { + max_d = i; + max_n = shape[i]; + } + } + return max_d; +} + +template int min_dim(std::vector &shape) { + int min_d = 0; + T min_n = SIZE_MAX; + for (int i = 0; i < shape.size(); i++) { + if (min_n > shape[i]) { + min_d = i; + min_n = shape[i]; + } + } + return min_d; +} + +template std::vector find_refactors(T n) { + std::vector factors; + T z = 2; + while (z * z <= n) { + if (n % z == 0) { + factors.push_back(z); + n /= z; + } else { + z++; + } + } + if (n > 1) { + factors.push_back(n); + } + return factors; +} + +template void adjust_shape(std::vector &shape, Config config) { + log::info("Using shape adjustment"); + int num_timesteps; + if (config.domain_decomposition == domain_decomposition_type::Variable) { + // If do shape adjustment with variable domain decomposition + // the temporal dim has to be the first dim + assert(config.domain_decomposition_dim == 0); + num_timesteps = shape[0] / config.domain_decomposition_sizes[0]; + shape[0] = config.domain_decomposition_sizes[0]; + } + int max_d = max_dim(shape); + SIZE max_n = shape[max_d]; + std::vector factors = find_refactors(max_n); + // std::cout << "factors: "; + // for (SIZE f : factors) std::cout << f << " "; + // std::cout << "\n"; + shape[max_d] = 1; + for (int i = factors.size() - 1; i >= 0; i--) { + int min_d = min_dim(shape); + shape[min_d] *= factors[i]; + // std::cout << "multiple " << factors[i] << + // " to dim " << min_d << ": " << shape[min_d] << "\n"; + } + if (config.domain_decomposition == domain_decomposition_type::Variable) { + shape[0] *= num_timesteps; + } + // std::cout << "shape: "; + // for (SIZE n : shape) { + // std::cout << n << "\n"; + // } + std::stringstream ss; + for (DIM d = 0; d < shape.size(); d++) { + ss << shape[d] << " "; + } + log::info("Shape adjusted to " + ss.str()); +} +} // namespace mgard_x_160 \ No newline at end of file diff --git a/include/mgard-x-160/CompressionLowLevel/Compressor.h b/include/mgard-x-160/CompressionLowLevel/Compressor.h new file mode 100644 index 000000000..842416b70 --- /dev/null +++ b/include/mgard-x-160/CompressionLowLevel/Compressor.h @@ -0,0 +1,123 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_COMPRESSOR_H +#define MGARD_X_160_COMPRESSOR_H + +#include + +#include "../RuntimeX/RuntimeXPublic.h" + +#include "../DataRefactoring/DataRefactor.hpp" + +// #include "CompressionLowLevelWorkspace.hpp" + +#include "NormCalculator.hpp" + +#include "../Hierarchy/Hierarchy.h" + +#include "../Lossless/Lossless.hpp" +#include "../Quantization/LinearQuantization.hpp" +#include "../Utilities/ProjectionMode.h" + +#include "LossyCompressorInterface.hpp" + +namespace mgard_x_160 { + +// D-aware counterpart of infer_orthogonal_projection in +// Utilities/ProjectionMode.h, kept so the shared call sites compile unchanged. +template +inline bool infer_orthogonal_projection(compression_projection_mode_type mode, + T s) { + // MGARD 1.6.0 data: always the orthogonal basis (see Utilities/ + // ProjectionMode.h). + return true; +} + +template +class Compressor : public LossyCompressorInterface { +public: + using HierarchyType = Hierarchy; + using DataRefactorType = data_refactoring::DataRefactor; + using LosslessCompressorType = + ComposedLosslessCompressor; + using LinearQuantizerType = LinearQuantizer; + +public: + Compressor(); + + Compressor(Hierarchy &hierarchy, Config config); + + void Adapt(Hierarchy &hierarchy, Config config, + int queue_idx); + + static size_t EstimateMemoryFootprint(std::vector shape, Config config); + + void CalculateNorm(Array &original_data, + enum error_bound_type ebtype, T s, T &norm, int queue_idx); + + void Decompose(Array &original_data, + bool orthogonal_projection, int queue_idx); + + void Quantize(Array &original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + int queue_idx); + + void LosslessCompress(Array<1, Byte, DeviceType> &compressed_data, + int queue_idx); + + void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx); + + void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx); + + void Recompose(Array &decompressed_data, + bool orthogonal_projection, int queue_idx); + + void Dequantize(Array &decompressed_data, + enum error_bound_type ebtype, T tol, T s, T norm, + int queue_idx); + + // Dequantize + recompose as one step of the decompression pipelines. Here it + // is simply the two calls in sequence; a compressor that can do better (see + // HybridHierarchyCompressor, which fuses them into one pass over the local + // levels) overrides this and decides for itself. The pipelines call this + // rather than the two methods so they do not have to know which compressor + // they are driving. + void DequantizeRecompose(Array &decompressed_data, + enum error_bound_type ebtype, T tol, T s, T norm, + int queue_idx); + + void LosslessDecompress(Array<1, Byte, DeviceType> &compressed_data, + int queue_idx); + + void Compress(Array &original_data, + enum error_bound_type ebtype, T tol, T s, T &norm, + Array<1, Byte, DeviceType> &compressed_data, int queue_idx); + void Decompress(Array<1, Byte, DeviceType> &compressed_data, + enum error_bound_type ebtype, T tol, T s, T &norm, + Array &decompressed_data, int queue_idx); + + bool initialized; + Hierarchy *hierarchy; + Config config; + // Whether the last (de)compose should use orthogonal projection. Derived + // from config.projection_mode and s (see infer_orthogonal_projection) + // during Compress/Decompress/(De)quantize and consumed by Recompose, which + // does not receive s. Defaults to true so the orthogonal path is used + // unless the config/s combination resolves to the hierarchical fast path. + bool orthogonal_projection = true; + Array<1, T, DeviceType> norm_tmp_array; + Array<1, T, DeviceType> norm_array; + Array quantized_array; + DataRefactorType refactor; + LinearQuantizerType quantizer; + LosslessCompressorType lossless_compressor; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/CompressionLowLevel/Compressor.hpp b/include/mgard-x-160/CompressionLowLevel/Compressor.hpp new file mode 100644 index 000000000..ee4f592a0 --- /dev/null +++ b/include/mgard-x-160/CompressionLowLevel/Compressor.hpp @@ -0,0 +1,300 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include +#include +#include +#include +#include +#include + +#include "../Utilities/Types.h" + +#include "../Config/Config.h" +#include "../Hierarchy/Hierarchy.h" +#include "../RuntimeX/RuntimeX.h" +#include "Compressor.h" +#include "CompressorCache.hpp" + +#ifndef MGARD_X_160_COMPRESSOR_HPP +#define MGARD_X_160_COMPRESSOR_HPP + +namespace mgard_x_160 { + +static bool debug_print_compression = true; + +template +Compressor::Compressor() : initialized(false) {} + +template +Compressor::Compressor(Hierarchy &hierarchy, + Config config) + : initialized(true), hierarchy(&hierarchy), config(config), + refactor(hierarchy, config), + lossless_compressor(hierarchy.total_num_elems(), config), + quantizer(hierarchy, config) { + + norm_array = Array<1, T, DeviceType>({1}); + norm_tmp_array = Array<1, T, DeviceType>({hierarchy.total_num_elems()}, + (T *)refactor.w_array.data()); + // Reuse workspace. Warning: + if (sizeof(QUANTIZED_INT) <= sizeof(T)) { + // Reuse workspace if possible + // norm_tmp_array = Array<1, T, DeviceType>({hierarchy.total_num_elems()}, + // (T *)refactor.w_array.data()); + quantized_array = Array( + hierarchy.level_shape(hierarchy.l_target()), + (QUANTIZED_INT *)refactor.w_array.data()); + } else { + // norm_tmp_array = Array<1, T, DeviceType>({hierarchy.total_num_elems()}); + quantized_array = Array( + hierarchy.level_shape(hierarchy.l_target())); + } +} + +template +void Compressor::Adapt(Hierarchy &hierarchy, + Config config, int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + this->config = config; + refactor.Adapt(hierarchy, config, queue_idx); + quantizer.Adapt(hierarchy, config, queue_idx); + lossless_compressor.Adapt(hierarchy.total_num_elems(), config, queue_idx); + norm_array.resize({1}, queue_idx); + // Reuse workspace. + norm_tmp_array = Array<1, T, DeviceType>({hierarchy.total_num_elems()}, + (T *)refactor.w_array.data()); + if (sizeof(QUANTIZED_INT) <= sizeof(T)) { + // Reuse workspace if possible + // norm_tmp_array = Array<1, T, DeviceType>({hierarchy.total_num_elems()}, + // (T *)refactor.w_array.data()); + quantized_array = Array( + hierarchy.level_shape(hierarchy.l_target()), + (QUANTIZED_INT *)refactor.w_array.data()); + } else { + // norm_tmp_array.resize({hierarchy.total_num_elems()}, queue_idx); + quantized_array.resize(hierarchy.level_shape(hierarchy.l_target()), + queue_idx); + } +} + +template +size_t +Compressor::EstimateMemoryFootprint(std::vector shape, + Config config) { + Hierarchy hierarchy; + hierarchy.EstimateMemoryFootprint(shape); + size_t size = 0; + size += DataRefactorType::EstimateMemoryFootprint(shape); + log::dbg( + "Data refactor space: " + + std::to_string( + (double)(DataRefactorType::EstimateMemoryFootprint(shape)) / 1e9) + + " GB"); + size += LinearQuantizerType::EstimateMemoryFootprint(shape); + log::dbg( + "Quantizer space: " + + std::to_string( + (double)(LinearQuantizerType::EstimateMemoryFootprint(shape)) / 1e9) + + " GB"); + size += LosslessCompressorType::EstimateMemoryFootprint( + hierarchy.total_num_elems(), config); + log::dbg( + "Lossless space: " + + std::to_string((double)(LosslessCompressorType::EstimateMemoryFootprint( + hierarchy.total_num_elems(), config)) / + 1e9) + + " GB"); + size += sizeof(T); + if (sizeof(QUANTIZED_INT) > sizeof(T)) { + // size += sizeof(T) * hierarchy.total_num_elems(); + size += sizeof(QUANTIZED_INT) * hierarchy.total_num_elems(); + } + return size; +} + +template +void Compressor::CalculateNorm( + Array &original_data, enum error_bound_type ebtype, T s, + T &norm, int queue_idx) { + if (ebtype == error_bound_type::REL) { + norm = + norm_calculator(original_data, SubArray(norm_tmp_array), + SubArray(norm_array), s, config.normalize_coordinates); + } +} + +template +void Compressor::Decompose( + Array &original_data, bool orthogonal_projection, + int queue_idx) { + refactor.Decompose(SubArray(original_data), orthogonal_projection, queue_idx); +} + +template +void Compressor::Quantize( + Array &original_data, enum error_bound_type ebtype, T tol, + T s, T norm, int queue_idx) { + orthogonal_projection = + infer_orthogonal_projection(config.projection_mode, s); + quantizer.Quantize(original_data, ebtype, tol, s, norm, quantized_array, + lossless_compressor, queue_idx, orthogonal_projection); +} + +template +void Compressor::LosslessCompress( + Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + Array<1, QUANTIZED_INT, DeviceType> quantized_liearized_array( + {hierarchy->total_num_elems()}, (QUANTIZED_INT *)quantized_array.data()); + lossless_compressor.Compress(quantized_liearized_array, compressed_data, + queue_idx); +} + +template +void Compressor::Serialize( + Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + lossless_compressor.Serialize(compressed_data, queue_idx); +} + +template +void Compressor::Deserialize( + Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + lossless_compressor.Deserialize(compressed_data, queue_idx); +} + +template +void Compressor::Recompose( + Array &decompressed_data, bool orthogonal_projection, + int queue_idx) { + // The generic decompress pipeline always passes true here, but the basis was + // decided from s during Dequantize/Decompress and stored in + // this->orthogonal_projection. Prefer the stored decision so the hierarchical + // L-infinity fast path stays consistent between decompose and recompose; the + // caller's argument is only honored when it requests the (default) orthogonal + // path, so explicit orthogonal callers are never overridden. + refactor.Recompose(SubArray(decompressed_data), + orthogonal_projection && this->orthogonal_projection, + queue_idx); +} + +template +void Compressor::DequantizeRecompose( + Array &decompressed_data, enum error_bound_type ebtype, + T tol, T s, T norm, int queue_idx) { + Dequantize(decompressed_data, ebtype, tol, s, norm, queue_idx); + Recompose(decompressed_data, true, queue_idx); +} + +template +void Compressor::Dequantize( + Array &decompressed_data, enum error_bound_type ebtype, + T tol, T s, T norm, int queue_idx) { + orthogonal_projection = + infer_orthogonal_projection(config.projection_mode, s); + decompressed_data.resize(hierarchy->level_shape(hierarchy->l_target())); + quantizer.Dequantize(decompressed_data, ebtype, tol, s, norm, quantized_array, + lossless_compressor, queue_idx, orthogonal_projection); +} + +template +void Compressor::LosslessDecompress( + Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + Array<1, QUANTIZED_INT, DeviceType> quantized_liearized_data( + {hierarchy->total_num_elems()}, (QUANTIZED_INT *)quantized_array.data()); + lossless_compressor.Decompress(compressed_data, quantized_liearized_data, + queue_idx); +} + +template +void Compressor::Compress( + Array &original_data, enum error_bound_type ebtype, T tol, + T s, T &norm, Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + + config.apply(); + + DeviceRuntime::SelectDevice(config.dev_id); + log::info("Select device: " + DeviceRuntime::GetDeviceName()); + Timer timer_total; + for (int d = D - 1; d >= 0; d--) { + if (hierarchy->level_shape(hierarchy->l_target(), d) != + original_data.shape(d)) { + log::err("The shape of input array does not match the shape initialized " + "in hierarchy!"); + return; + } + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer_total.start(); + } + + // Resolve config.projection_mode against s (Auto defaults to the cheap + // hierarchical basis under L-infinity, orthogonal otherwise). Remember the + // decision so Recompose (which has no s) reconstructs consistently. + orthogonal_projection = + infer_orthogonal_projection(config.projection_mode, s); + + CalculateNorm(original_data, ebtype, s, norm, queue_idx); + Decompose(original_data, orthogonal_projection, queue_idx); + Quantize(original_data, ebtype, tol, s, norm, queue_idx); + LosslessCompress(compressed_data, queue_idx); + Serialize(compressed_data, queue_idx); + if (config.compress_with_dryrun) { + Dequantize(original_data, ebtype, tol, s, norm, queue_idx); + Recompose(original_data, orthogonal_projection, queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer_total.end(); + timer_total.print("Low-level compression", + hierarchy->total_num_elems() * sizeof(T)); + timer_total.clear(); + } +} + +template +void Compressor::Decompress( + Array<1, Byte, DeviceType> &compressed_data, enum error_bound_type ebtype, + T tol, T s, T &norm, Array &decompressed_data, + int queue_idx) { + config.apply(); + + DeviceRuntime::SelectDevice(config.dev_id); + log::info("Select device: " + DeviceRuntime::GetDeviceName()); + Timer timer_total, timer_each; + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer_total.start(); + } + + // Must mirror the orthogonal-projection decision made during Compress so the + // hierarchical fast path (s == inf) reconstructs consistently. + orthogonal_projection = + infer_orthogonal_projection(config.projection_mode, s); + + decompressed_data.resize(hierarchy->level_shape(hierarchy->l_target())); + Deserialize(compressed_data, queue_idx); + LosslessDecompress(compressed_data, queue_idx); + Dequantize(decompressed_data, ebtype, tol, s, norm, queue_idx); + Recompose(decompressed_data, orthogonal_projection, queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer_total.end(); + timer_total.print("Low-level decompression", + hierarchy->total_num_elems() * sizeof(T)); + timer_total.clear(); + } +} + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/CompressionLowLevel/CompressorCache.hpp b/include/mgard-x-160/CompressionLowLevel/CompressorCache.hpp new file mode 100644 index 000000000..1385a1e56 --- /dev/null +++ b/include/mgard-x-160/CompressionLowLevel/CompressorCache.hpp @@ -0,0 +1,153 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "../Utilities/Types.h" + +#include "../Config/Config.h" +#include "../Hierarchy/Hierarchy.h" +#include "../RuntimeX/RuntimeX.h" + +#ifndef MGARD_X_160_COMPRESSOR_CACHE_HPP +#define MGARD_X_160_COMPRESSOR_CACHE_HPP + +namespace mgard_x_160 { + +template +class CompressorBundle { +public: + using HierarchyType = typename CompressorType::HierarchyType; + std::unordered_map *hierarchy_cache; + CompressorType *compressor = nullptr; + Array *device_subdomain_buffer = nullptr; + Array<1, Byte, DeviceType> *device_compressed_buffer = nullptr; + + bool initialized = false; + + std::string GenHierarchyKey(std::vector shape) { + std::stringstream ss; + for (int i = 0; i < shape.size(); i++) { + ss << std::setfill('0') << std::setw(20) << shape[i] << std::fixed; + } + return ss.str(); + } + + // `config` matters for hierarchies that depend on more than the shape (a + // HybridHierarchy is built for given local/global level counts). + bool InHierarchyCache(std::vector shape, bool uniform, Config config) { + if (!initialized) { + return false; + } else if (!uniform) { + return false; + } else if (hierarchy_cache->find(GenHierarchyKey(shape)) == + hierarchy_cache->end()) { + return false; + } else if ((*hierarchy_cache)[GenHierarchyKey(shape)].data_structure() == + data_structure_type::Cartesian_Grid_Non_Uniform) { + return false; + } else if (!(*hierarchy_cache)[GenHierarchyKey(shape)].matches_config( + config)) { + return false; + } else { + return true; + } + } + + void InsertHierarchyCache(HierarchyType hierarchy) { + std::vector shape = hierarchy.level_shape(hierarchy.l_target()); + (*hierarchy_cache)[GenHierarchyKey(shape)] = std::move(hierarchy); + std::stringstream ss; + for (int i = 0; i < shape.size(); i++) { + ss << shape[i] << " "; + } + log::info("Add hierarchy with shape ( " + ss.str() + ") into cache"); + } + + HierarchyType &GetHierarchyCache(std::vector shape) { + std::stringstream ss; + for (int i = 0; i < shape.size(); i++) { + ss << shape[i] << " "; + } + log::info("Get hierarchy with shape ( " + ss.str() + ") from cache"); + return (*hierarchy_cache)[GenHierarchyKey(shape)]; + } + + void Release() { + log::info("Releasing compressor cache"); + delete hierarchy_cache; + delete[] compressor; + delete[] device_subdomain_buffer; + delete[] device_compressed_buffer; + initialized = false; + } + + void ClearHierarchyCache() { + log::info("Clear hierarchy cache"); + if (hierarchy_cache) { + delete hierarchy_cache; + } + hierarchy_cache = new std::unordered_map(); + } + + void Initialize(int num_buffers = 2, int num_compressors = 1) { + log::info("Initializing compressor cache"); + hierarchy_cache = new std::unordered_map(); + compressor = new CompressorType[num_compressors]; + device_subdomain_buffer = new Array[num_buffers]; + device_compressed_buffer = new Array<1, Byte, DeviceType>[num_buffers]; + initialized = true; + } + + void SafeInitialize(int num_buffers = 2, int num_compressors = 1) { + if (!initialized) { + Initialize(num_buffers, num_compressors); + } + } + + void SafeRelease() { + if (initialized) { + Release(); + } + } + + size_t CacheSize() { + size_t size = 0; + if (initialized) { + for (auto &item : *hierarchy_cache) { + HierarchyType &_hierarchy = item.second; + size += _hierarchy.EstimateMemoryFootprint( + _hierarchy.level_shape(_hierarchy.l_target())); + } + if (compressor->initialized) { + size += CompressorType::EstimateMemoryFootprint( + compressor->hierarchy->level_shape( + compressor->hierarchy->l_target()), + compressor->config); + size += compressor->hierarchy->total_num_elems() * sizeof(T) * 4; + } + } + return size; + } +}; + +template +class CompressorCache { +public: + static inline thread_local CompressorBundle + cache; +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/CompressionLowLevel/HybridHierarchyCompressor.h b/include/mgard-x-160/CompressionLowLevel/HybridHierarchyCompressor.h new file mode 100644 index 000000000..578f31a2b --- /dev/null +++ b/include/mgard-x-160/CompressionLowLevel/HybridHierarchyCompressor.h @@ -0,0 +1,133 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_HYBRID_HIERARCHY_COMPRESSOR_H +#define MGARD_X_160_HYBRID_HIERARCHY_COMPRESSOR_H + +#include "../DataRefactoring/BlockLocalHierarchyDataRefactor.hpp" +#include "../DataRefactoring/HybridHierarchyDataRefactor.hpp" +#include "../RuntimeX/RuntimeXPublic.h" + +// #include "CompressionLowLevelWorkspace.hpp" + +#include "../Hierarchy/HybridHierarchy.h" +#include "../Lossless/Lossless.hpp" +#include "../Quantization/HybridHierarchyLinearQuantization.hpp" +#include "../Quantization/LocalQuantization.hpp" +#include "../Utilities/ProjectionMode.h" +#include "LossyCompressorInterface.hpp" +#include "NormCalculator.hpp" + +namespace mgard_x_160 { + +template +class HybridHierarchyCompressor + : public LossyCompressorInterface { +public: + using HierarchyType = HybridHierarchy; + // using BlockLocalHierarchyDataRefactorType = + // data_refactoring::BlockLocalHierarchyDataRefactor; + using HybridHierarchyDataRefactorType = + data_refactoring::HybridHierarchyDataRefactor; + using LosslessCompressorType = + ComposedLosslessCompressor; + // using LocalQuantizerType = LocalQuantizer; + using HybridQuantizerType = + HybridHierarchyQuantizer; + + HybridHierarchyCompressor(); + + HybridHierarchyCompressor(HybridHierarchy &hierarchy, + Config config); + + void Adapt(HybridHierarchy &hierarchy, Config config, + int queue_idx); + + static size_t EstimateMemoryFootprint(std::vector shape, Config config); + + void CalculateNorm(Array &original_data, + enum error_bound_type ebtype, T s, T &norm, int queue_idx); + + void Decompose(Array &original_data, int queue_idx); + + void Quantize(Array &original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + int queue_idx); + + // Fused Decompose+Quantize (single pass over the local levels); used by + // Compress() instead of Decompose()+Quantize() when + // hybrid_quantizer.CanFuseQuantize(s) holds. + void DecomposeQuantize(Array &original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + int queue_idx); + + void LosslessCompress(Array<1, Byte, DeviceType> &compressed_data, + int queue_idx); + + void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx); + + void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx); + + void Recompose(Array &decompressed_data, + bool orthogonal_projection, int queue_idx); + + void Dequantize(Array &decompressed_data, + enum error_bound_type ebtype, T tol, T s, T norm, + int queue_idx); + + // Dequantize + recompose. Overrides the plain two-call version: picks the + // fused single-pass implementation below when the configuration allows it, + // and falls back to Dequantize() + Recompose() otherwise. This is the only + // place that choice is made, for both the pipelines and Decompress(). + void DequantizeRecompose(Array &decompressed_data, + enum error_bound_type ebtype, T tol, T s, T norm, + int queue_idx); + + // Fused Dequantize+Recompose: one pass over the local levels, coefficients + // never round-trip through global memory as T. Requires + // hybrid_quantizer.CanFuseQuantize(s); throws otherwise. Call + // DequantizeRecompose() instead unless you specifically want the fused path. + void DequantizeRecomposeFused(Array &decompressed_data, + enum error_bound_type ebtype, T tol, T s, + T norm, int queue_idx); + + void LosslessDecompress(Array<1, Byte, DeviceType> &compressed_data, + int queue_idx); + + void Compress(Array &original_data, + enum error_bound_type ebtype, T tol, T s, T &norm, + Array<1, Byte, DeviceType> &compressed_data, int queue_idx); + void Decompress(Array<1, Byte, DeviceType> &compressed_data, + enum error_bound_type ebtype, T tol, T s, T &norm, + Array &decompressed_data, int queue_idx); + + static SIZE calculate_padded_size(std::vector shape, Config config); + static SIZE + calculate_padded_size(HybridHierarchy &hierarchy, + Config config); + + bool initialized; + HybridHierarchy *hierarchy; + Config config; + bool orthogonal_projection = true; + Array<1, T, DeviceType> norm_tmp_array; + Array<1, T, DeviceType> norm_array; + // Array<1, T, DeviceType> local_decomposed_array; + // Array<1, QUANTIZED_INT, DeviceType> local_quantized_array; + Array<1, T, DeviceType> hybrid_decomposed_array; + Array<1, QUANTIZED_INT, DeviceType> hybrid_quantized_array; + // BlockLocalHierarchyDataRefactorType local_refactor; + // LocalQuantizerType local_quantizer; + HybridHierarchyDataRefactorType hybrid_refactor; + HybridQuantizerType hybrid_quantizer; + LosslessCompressorType lossless_compressor; +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/CompressionLowLevel/HybridHierarchyCompressor.hpp b/include/mgard-x-160/CompressionLowLevel/HybridHierarchyCompressor.hpp new file mode 100644 index 000000000..3c887146a --- /dev/null +++ b/include/mgard-x-160/CompressionLowLevel/HybridHierarchyCompressor.hpp @@ -0,0 +1,465 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include +#include +#include +#include +#include +#include + +#include "../Config/Config.h" +#include "../Hierarchy/Hierarchy.h" +#include "../RuntimeX/RuntimeX.h" +#include "../Utilities/Types.h" +#include "CompressorCache.hpp" +#include "HybridHierarchyCompressor.h" + +#ifndef MGARD_X_160_HYBRID_HIERARCHY_COMPRESSOR_HPP +#define MGARD_X_160_HYBRID_HIERARCHY_COMPRESSOR_HPP + +namespace mgard_x_160 { + +static bool debug_print_compression = true; + +template +HybridHierarchyCompressor::HybridHierarchyCompressor() + : initialized(false) {} + +template +HybridHierarchyCompressor::HybridHierarchyCompressor( + HybridHierarchy &hierarchy, Config config) + : initialized(true), hierarchy(&hierarchy), config(config), + hybrid_refactor(hierarchy, config), + lossless_compressor(calculate_padded_size(hierarchy, config), config), + hybrid_quantizer(hierarchy, config) { + norm_array = Array<1, T, DeviceType>({1}); + hybrid_decomposed_array = + Array<1, T, DeviceType>({hybrid_refactor.DecomposedDataSize()}); + + // Reuse workspace. Warning: + if (sizeof(QUANTIZED_INT) <= sizeof(T)) { + if (config.num_local_refactoring_level > 0) { + norm_tmp_array = Array<1, T, DeviceType>( + {hierarchy.total_num_elems()}, + (T *)hybrid_refactor.local_refactor.coarse_buffers[0].data()); + // hybrid_quantized_array = Array<1, QUANTIZED_INT, DeviceType>( + // {hybrid_refactor.DecomposedDataSize()}, + // (QUANTIZED_INT*)hybrid_refactor.local_refactor.coarse_buffers[0] + // .data()); + } else { + // Reuse space from global refactor + norm_tmp_array = Array<1, T, DeviceType>( + {hierarchy.total_num_elems()}, + (T *)hybrid_refactor.global_refactor.w_array.data()); + // hybrid_quantized_array = Array<1, QUANTIZED_INT, DeviceType>( + // {hybrid_refactor.DecomposedDataSize()}, + // (QUANTIZED_INT*)hybrid_refactor.global_refactor.w_array.data()); + } + } else { + // if space is not enough + norm_tmp_array = Array<1, T, DeviceType>({hierarchy.total_num_elems()}); + // hybrid_quantized_array = Array<1, QUANTIZED_INT, DeviceType>( + // {hybrid_refactor.DecomposedDataSize()}); + } +} + +template +void HybridHierarchyCompressor::Adapt( + HybridHierarchy &hierarchy, Config config, + int queue_idx) { + hierarchy.check_config(config); + this->initialized = true; + this->hierarchy = &hierarchy; + this->config = config; + hybrid_refactor.Adapt(hierarchy, config, queue_idx); + lossless_compressor.Adapt(calculate_padded_size(hierarchy, config), config, + queue_idx); + hybrid_quantizer.Adapt(hierarchy, config, queue_idx); + norm_array.resize({1}, queue_idx); + hybrid_decomposed_array.resize({hybrid_refactor.DecomposedDataSize()}, + queue_idx); + hybrid_quantized_array = Array<1, QUANTIZED_INT, DeviceType>( + {hybrid_refactor.DecomposedDataSize()}); + + // Reuse workspace + if (sizeof(QUANTIZED_INT) <= sizeof(T)) { + if (config.num_local_refactoring_level > 0) { + norm_tmp_array = Array<1, T, DeviceType>( + {hierarchy.total_num_elems()}, + (T *)hybrid_refactor.local_refactor.coarse_buffers[0].data()); + // hybrid_quantized_array = Array<1, QUANTIZED_INT, DeviceType>( + // {hybrid_refactor.DecomposedDataSize()}, + // (QUANTIZED_INT*)hybrid_refactor.local_refactor.coarse_buffers[0] + // .data()); + } else { + // Reuse space from global refactor + norm_tmp_array = Array<1, T, DeviceType>( + {hierarchy.total_num_elems()}, + (T *)hybrid_refactor.global_refactor.w_array.data()); + // hybrid_quantized_array = Array<1, QUANTIZED_INT, DeviceType>( + // {hybrid_refactor.DecomposedDataSize()}, + // (QUANTIZED_INT*)hybrid_refactor.global_refactor.w_array.data()); + } + } else { + norm_tmp_array.resize({hierarchy.total_num_elems()}, queue_idx); + // hybrid_quantized_array.resize({hybrid_refactor.DecomposedDataSize()}, + // queue_idx); + } +} + +// May not be accurate +template +size_t HybridHierarchyCompressor::EstimateMemoryFootprint( + std::vector shape, Config config) { + size_t size = 0; + // size += + // BlockLocalHierarchyDataRefactorType::EstimateMemoryFootprint(shape); + size += + HybridHierarchyDataRefactorType::EstimateMemoryFootprint(shape, config); + // log::info( + // "Data refactor space: " + + // std::to_string( + // (double)(BlockLocalHierarchyDataRefactorType::EstimateMemoryFootprint( + // shape)) / + // 1e9) + + // " GB"); + // size += LocalQuantizerType::EstimateMemoryFootprint(shape); + size += HybridQuantizerType::EstimateMemoryFootprint(shape); + // log::info( + // "Quantizer space: " + + // std::to_string( + // (double)(LocalQuantizerType::EstimateMemoryFootprint(shape)) / 1e9) + // + + // " GB"); + size += LosslessCompressorType::EstimateMemoryFootprint( + calculate_padded_size(shape, config), config); + // log::info( + // "Lossless space: " + + // std::to_string((double)(LosslessCompressorType::EstimateMemoryFootprint( + // hierarchy.total_num_elems(), config)) / + // 1e9) + + // " GB"); + size += sizeof(T); + if (sizeof(QUANTIZED_INT) > sizeof(T)) { + size += sizeof(T) * calculate_padded_size(shape, config); + size += sizeof(QUANTIZED_INT) * calculate_padded_size(shape, config); + } + return size; +} + +template +void HybridHierarchyCompressor::CalculateNorm( + Array &original_data, enum error_bound_type ebtype, T s, + T &norm, int queue_idx) { + if (ebtype == error_bound_type::REL) { + norm = + norm_calculator(original_data, SubArray(norm_tmp_array), + SubArray(norm_array), s, config.normalize_coordinates); + } +} + +template +void HybridHierarchyCompressor::Decompose( + Array &original_data, int queue_idx) { + // DumpSubArray("/home/leonli/TestInCacheBlock/org.txt",SubArray(original_data)); + // PrintSubarray("Original before decompose", SubArray(original_data)); + // SubArray temp({3,3,3}, original_data.data()); + // PrintSubarray("Orginal 8x8x8 before decompose", temp); + // hybrid_refactor.Decompose(original_data, decomposed_array, queue_idx); + hybrid_refactor.Decompose(SubArray(original_data), + SubArray(hybrid_decomposed_array), queue_idx, + orthogonal_projection); + // PrintSubarray("Decomposed after decompose", + // SubArray(local_decomposed_array)); +} + +template +void HybridHierarchyCompressor::Quantize( + Array &original_data, enum error_bound_type ebtype, T tol, + T s, T norm, int queue_idx) { + // Only called from Compress(), which has already resolved + // orthogonal_projection and set it on hybrid_quantizer. + SIZE total_num_elems_1D = hybrid_refactor.DecomposedDataSize(); + + SubArray<1, T, DeviceType> data_subarray({total_num_elems_1D}, + hybrid_decomposed_array.data()); + hybrid_quantizer.Quantize(data_subarray, ebtype, tol, s, norm, + hybrid_quantized_array, lossless_compressor, + queue_idx); +} + +template +void HybridHierarchyCompressor::DecomposeQuantize( + Array &original_data, enum error_bound_type ebtype, T tol, + T s, T norm, int queue_idx) { + // Only called from Compress(), which has already resolved + // orthogonal_projection and set it on hybrid_quantizer. + SubArray<1, T, DeviceType> decomposed_subarray(hybrid_decomposed_array); + SubArray<1, QUANTIZED_INT, DeviceType> quantized_subarray( + hybrid_quantized_array); + hybrid_quantizer.DecomposeQuantize( + hybrid_refactor, SubArray(original_data), decomposed_subarray, + quantized_subarray, ebtype, tol, s, norm, lossless_compressor, queue_idx); +} + +template +void HybridHierarchyCompressor::LosslessCompress( + Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + lossless_compressor.Compress(hybrid_quantized_array, compressed_data, + queue_idx); +} + +template +void HybridHierarchyCompressor::Serialize( + Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + lossless_compressor.Serialize(compressed_data, queue_idx); +} + +template +void HybridHierarchyCompressor::Deserialize( + Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + lossless_compressor.Deserialize(compressed_data, queue_idx); +} + +template +void HybridHierarchyCompressor::Recompose( + Array &decompressed_data, bool orthogonal_projection, + int queue_idx) { + this->orthogonal_projection = orthogonal_projection; + // PrintSubarray("Decomposed before recompose", + // SubArray(local_decomposed_array)); + hybrid_refactor.Recompose(SubArray(decompressed_data), + SubArray(hybrid_decomposed_array), queue_idx, + this->orthogonal_projection); + + // SubArray temp({3,3,3}, decompressed_data.data()); + // PrintSubarray("Orginal 8x8x8 after decompose", temp); + // PrintSubarray("Decompressed after recompose", SubArray(decompressed_data)); + // DumpSubArray("/home/leonli/TestInCacheBlock/decomp.txt",SubArray(decompressed_data)); +} + +template +void HybridHierarchyCompressor::Dequantize( + Array &decompressed_data, enum error_bound_type ebtype, + T tol, T s, T norm, int queue_idx) { + // Only called from DequantizeRecompose(), which has already resolved + // orthogonal_projection and set it on hybrid_quantizer. + SIZE total_num_elems_1D = hybrid_refactor.DecomposedDataSize(); + SubArray<1, T, DeviceType> decompressed_data_subarray( + {total_num_elems_1D}, hybrid_decomposed_array.data()); + // Direct calculation + hybrid_quantizer.Dequantize(decompressed_data_subarray, ebtype, tol, s, norm, + hybrid_quantized_array, lossless_compressor, + queue_idx); +} + +template +void HybridHierarchyCompressor::DequantizeRecompose( + Array &decompressed_data, enum error_bound_type ebtype, + T tol, T s, T norm, int queue_idx) { + // The decompress-side resolution point: the domain-decomposition pipelines + // (GPUPipelines/CPUPipelines) call this directly, bypassing Decompress(), so + // it has to resolve for itself rather than relying on a caller having done + // it already. + orthogonal_projection = + infer_orthogonal_projection(config.projection_mode, s); + hybrid_quantizer.SetOrthogonalProjection(orthogonal_projection); + if (config.fuse_dequantize_recompose && hybrid_quantizer.CanFuseQuantize(s)) { + log::info("Local dequantize+recompose kernels: fused"); + DequantizeRecomposeFused(decompressed_data, ebtype, tol, s, norm, + queue_idx); + } else { + log::info("Local dequantize+recompose kernels: separate (" + + (config.fuse_dequantize_recompose + ? hybrid_quantizer.WhyCannotFuseQuantize(s) + : std::string("kernel fusion disabled")) + + ")"); + Dequantize(decompressed_data, ebtype, tol, s, norm, queue_idx); + Recompose(decompressed_data, orthogonal_projection, queue_idx); + } +} + +template +void HybridHierarchyCompressor::DequantizeRecomposeFused( + Array &decompressed_data, enum error_bound_type ebtype, + T tol, T s, T norm, int queue_idx) { + SubArray<1, T, DeviceType> decomposed_subarray(hybrid_decomposed_array); + SubArray<1, QUANTIZED_INT, DeviceType> quantized_subarray( + hybrid_quantized_array); + // Build the output view from the hierarchy shape (like the unfused + // Recompose does for its final copy) rather than the array metadata, which + // the pipelines may not have resized yet. + SubArray data_subarray( + hierarchy->level_shape(hierarchy->l_target()), decompressed_data.data()); + hybrid_quantizer.DequantizeRecompose( + hybrid_refactor, data_subarray, decomposed_subarray, quantized_subarray, + ebtype, tol, s, norm, lossless_compressor, queue_idx); +} + +template +void HybridHierarchyCompressor::LosslessDecompress( + Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + lossless_compressor.Decompress(compressed_data, hybrid_quantized_array, + queue_idx); +} + +template +void HybridHierarchyCompressor::Compress( + Array &original_data, enum error_bound_type ebtype, T tol, + T s, T &norm, Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + config.apply(); + + DeviceRuntime::SelectDevice(config.dev_id); + log::info("Select device: " + DeviceRuntime::GetDeviceName()); + Timer timer_total; + Timer timer_compress_kernel; + for (int d = D - 1; d >= 0; d--) { + if (hierarchy->level_shape(hierarchy->l_target(), d) != + original_data.shape(d)) { + log::err("The shape of input array does not match the shape initilized " + "in hierarchy!"); + return; + } + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer_total.start(); + } + + // The compress-side resolution point: Decompose()/DecomposeQuantize()/ + // Quantize() below all just consume the result. + orthogonal_projection = + infer_orthogonal_projection(config.projection_mode, s); + hybrid_quantizer.SetOrthogonalProjection(orthogonal_projection); + + CalculateNorm(original_data, ebtype, s, norm, queue_idx); + // log::info(std::to_string(original_data.totalNumElems())); + // PrintSubarray("Original before decompose", SubArray(original_data)); + // log::info("Before decompose()"); + + if (log::level & log::TIME) + timer_compress_kernel.start(); + if (config.fuse_decompose_quantize && hybrid_quantizer.CanFuseQuantize(s)) { + log::info("Local decompose+quantize kernels: fused"); + DecomposeQuantize(original_data, ebtype, tol, s, norm, queue_idx); + } else { + log::info("Local decompose+quantize kernels: separate (" + + (config.fuse_decompose_quantize + ? hybrid_quantizer.WhyCannotFuseQuantize(s) + : std::string("kernel fusion disabled")) + + ")"); + Decompose(original_data, queue_idx); + // log::info("After decompose()"); + // log::info(std::to_string(original_data.totalNumElems())); + // PrintSubarray("Original after decompose", SubArray(original_data)); + // // PrintSubarray("Decomposed", SubArray(decomposed_array)); + // log::info("Before quantize"); + Quantize(original_data, ebtype, tol, s, norm, queue_idx); + } + // log::info("After quantize"); + // log::info("Num of Original data after quantization:"); + // log::info(std::to_string(original_data.totalNumElems())); + // PrintSubarray("Quantized", SubArray(local_quantized_array)); + // PrintSubarray("Compressed data before lossless",SubArray(compressed_data)); + // log::info("Before lossless"); + LosslessCompress(compressed_data, queue_idx); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(0); + timer_compress_kernel.end(); + timer_compress_kernel.print("Compression Kernel"); + timer_compress_kernel.clear(); + } + Serialize(compressed_data, queue_idx); + // log::info("After lossless"); + // PrintSubarray("Quantized data before + // lossless",SubArray(local_quantized_array)); PrintSubarray("Compressed data + // after lossless",SubArray(compressed_data)); From printing result, we found + // lossless didn't do anything to compressed_data + if (config.compress_with_dryrun) { + DequantizeRecompose(original_data, ebtype, tol, s, norm, queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(0); + timer_total.end(); + timer_total.print("Low-level compression", + hierarchy->total_num_elems() * sizeof(T)); + timer_total.clear(); + } +} + +template +void HybridHierarchyCompressor::Decompress( + Array<1, Byte, DeviceType> &compressed_data, enum error_bound_type ebtype, + T tol, T s, T &norm, Array &decompressed_data, + int queue_idx) { + config.apply(); + + DeviceRuntime::SelectDevice(config.dev_id); + log::info("Select device: " + DeviceRuntime::GetDeviceName()); + Timer timer_total, timer_each; + + if (log::level & log::TIME) + timer_total.start(); + + decompressed_data.resize(hierarchy->level_shape(hierarchy->l_target())); + Deserialize(compressed_data, queue_idx); + LosslessDecompress(compressed_data, queue_idx); + // DequantizeRecompose resolves orthogonal_projection itself. + DequantizeRecompose(decompressed_data, ebtype, tol, s, norm, queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(0); + timer_total.end(); + timer_total.print("Low-level decompression", + hierarchy->total_num_elems() * sizeof(T)); + timer_total.clear(); + } +} + +// Only calculating padding to 8x8x8 for once +template +SIZE HybridHierarchyCompressor::calculate_padded_size( + std::vector shape, Config config) { + SIZE L = config.num_local_refactoring_level; + SIZE total_num_elems_1D = 1; + if (L > 0) { + std::vector> fine_shapes, coarse_shapes; + HybridHierarchy::compute_local_shapes( + shape, L, fine_shapes, coarse_shapes); + for (SIZE l = 0; l < L; l++) { + SIZE last_level_size = 1, curr_level_size = 1; + for (DIM d = 0; d < D; d++) { + last_level_size *= fine_shapes[l][d]; + curr_level_size *= coarse_shapes[l][d]; + } + total_num_elems_1D += (last_level_size - curr_level_size); + if (l == L - 1) { + total_num_elems_1D += curr_level_size; + } + } + } else { + total_num_elems_1D = 1; + for (DIM d = 0; d < D; d++) { + total_num_elems_1D *= shape[d]; + } + } + + return total_num_elems_1D; +} + +template +SIZE HybridHierarchyCompressor::calculate_padded_size( + HybridHierarchy &hierarchy, Config config) { + return calculate_padded_size(hierarchy.input_shape(), config); +} + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/CompressionLowLevel/LossyCompressorInterface.hpp b/include/mgard-x-160/CompressionLowLevel/LossyCompressorInterface.hpp new file mode 100644 index 000000000..42ccc3337 --- /dev/null +++ b/include/mgard-x-160/CompressionLowLevel/LossyCompressorInterface.hpp @@ -0,0 +1,24 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_LOSSY_COMPRESSOR_INTERFACE_HPP +#define MGARD_X_160_LOSSY_COMPRESSOR_INTERFACE_HPP +namespace mgard_x_160 { +template +class LossyCompressorInterface { + virtual void Compress(Array &original_data, + enum error_bound_type type, T tol, T s, T &norm, + Array<1, Byte, DeviceType> &compressed_data, + int queue_idx) = 0; + virtual void Decompress(Array<1, Byte, DeviceType> &compressed_data, + enum error_bound_type type, T tol, T s, T &norm, + Array &decompressed_data, + int queue_idx) = 0; +}; +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/CompressionLowLevel/NormCalculator.hpp b/include/mgard-x-160/CompressionLowLevel/NormCalculator.hpp new file mode 100644 index 000000000..4cd982d3e --- /dev/null +++ b/include/mgard-x-160/CompressionLowLevel/NormCalculator.hpp @@ -0,0 +1,84 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_NORM_CALCULATOR_HPP +#define MGARD_X_160_NORM_CALCULATOR_HPP + +namespace mgard_x_160 { +template +T norm_calculator(Array &original_array, + SubArray<1, T, DeviceType> workspace_subarray, + SubArray<1, T, DeviceType> norm_subarray, T s, + bool normalize_coordinates) { + Timer timer; + if (log::level & log::TIME) + timer.start(); + SIZE total_elems = 1; + for (DIM d = 0; d < D; d++) + total_elems *= original_array.shape(d); + + T norm = 0; + SubArray<1, T, DeviceType> temp_subarray; + if (!original_array.isPitched()) { // zero copy + log::dbg("Use zero copy when calculating norm"); + temp_subarray = + SubArray<1, T, DeviceType>({total_elems}, original_array.data()); + } else { // need to linearized + log::info("Explicit copy used when calculating norm"); + temp_subarray = workspace_subarray; + SIZE linearized_width = 1; + for (DIM d = 0; d < D - 1; d++) + linearized_width *= original_array.shape(d); + MemoryManager::CopyND( + temp_subarray.data(), original_array.shape(D - 1), + original_array.data(), original_array.ld(D - 1), + original_array.shape(D - 1), linearized_width, 0); + } + DeviceRuntime::SyncQueue(0); + if (s == std::numeric_limits::infinity()) { + Array<1, Byte, DeviceType> abs_max_workspace; + DeviceCollective::AbsMax( + total_elems, temp_subarray, norm_subarray, abs_max_workspace, false, 0); + DeviceCollective::AbsMax( + total_elems, temp_subarray, norm_subarray, abs_max_workspace, true, 0); + MemoryManager::Copy1D(&norm, norm_subarray.data(), 1, 0); + DeviceRuntime::SyncQueue(0); + // Avoiding issue with norm == 0 + if (norm == 0) + norm = std::numeric_limits::epsilon(); + log::info("L_inf norm: " + std::to_string(norm)); + } else { + Array<1, Byte, DeviceType> square_sum_workspace; + DeviceCollective::SquareSum(total_elems, temp_subarray, + norm_subarray, square_sum_workspace, + false, 0); + DeviceCollective::SquareSum(total_elems, temp_subarray, + norm_subarray, square_sum_workspace, + true, 0); + MemoryManager::Copy1D(&norm, norm_subarray.data(), 1, 0); + DeviceRuntime::SyncQueue(0); + if (!normalize_coordinates) { + norm = std::sqrt(norm); + } else { + norm = std::sqrt(norm / total_elems); + } + // Avoiding issue with norm == 0 + if (norm == 0) + norm = std::numeric_limits::epsilon(); + log::info("L_2 norm: " + std::to_string(norm)); + } + if (log::level & log::TIME) { + timer.end(); + timer.print("Calculate norm", total_elems * sizeof(T)); + timer.clear(); + } + return norm; +} + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Config/Config.h b/include/mgard-x-160/Config/Config.h new file mode 100644 index 000000000..238618b38 --- /dev/null +++ b/include/mgard-x-160/Config/Config.h @@ -0,0 +1,71 @@ +#ifndef MGARD_X_160_CONFIG_HPP +#define MGARD_X_160_CONFIG_HPP + +#include "../RuntimeX/RuntimeXPublic.h" +#include "../RuntimeX/Utilities/Log.h" +#include "../Utilities/Types.h" + +namespace mgard_x_160 { + +struct Config { + device_type dev_type; + int dev_id; + enum compressor_type compressor; + enum domain_decomposition_type domain_decomposition; + enum decomposition_type decomposition; + double estimate_outlier_ratio; + SIZE huff_dict_size; + SIZE huff_block_size; + SIZE block_delta_block_size; + enum block_delta_mode_type block_delta_mode; + SIZE lz4_block_size; + int zstd_compress_level; + bool normalize_coordinates; + enum lossless_type lossless; + // MGARD 1.6.0 level-linearized ("reorder") layout of the quantized data. + // Restored from the header (SHUFFLE preprocessor). + int reorder; + int log_level; + bool auto_pin_host_buffers; + SIZE max_memory_footprint; + SIZE total_num_bitplanes; + SIZE block_size; + SIZE domain_decomposition_dim; + std::vector domain_decomposition_sizes; + bool mdr_adaptive_resolution; + bool adjust_shape; + bool compress_with_dryrun; + // Block-local levels of the hybrid hierarchy (L). Default 1. + SIZE num_local_refactoring_level; + // Global refactoring levels (M): caps the levels of every multilevel + // hierarchy -- the plain multi-dimensional one and the global stage of the + // hybrid one. Recorded in the file header. Default 3. + SIZE num_global_refactoring_level; + bool auto_cache_release; + cpu_parallelization_mode cpu_mode; + bool mdr_qoi_mode; + int mdr_qoi_num_variables; + std::vector roi_tolerance_map; + bool enable_roi; + // Transform basis policy, shared by the plain Compressor and the hybrid + // (BlockMGARD) HybridHierarchyCompressor. Auto (default) picks Hierarchical + // under an L-infinity bound and Orthogonal otherwise; Orthogonal/ + // Hierarchical force a specific basis (Hierarchical still requires an + // L-infinity bound). See compression_projection_mode_type in Types.h. + enum compression_projection_mode_type projection_mode; + // The hybrid (BlockMGARD) local stage fuses its decompose/recompose kernels + // with quantization/dequantization so coefficients never round-trip through + // global memory as T. These select the older separate-pass implementation, + // which the test suite still exercises. Purely a performance choice: both + // paths reconstruct identically, so a file compressed either way decompresses + // either way and nothing about the choice is recorded in the file header. + bool fuse_decompose_quantize; // compression + bool fuse_dequantize_recompose; // decompression + + Config(); + void apply(); +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/BlockLocalHierarchyDataRefactor.hpp b/include/mgard-x-160/DataRefactoring/BlockLocalHierarchyDataRefactor.hpp new file mode 100644 index 000000000..21ada4bdb --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/BlockLocalHierarchyDataRefactor.hpp @@ -0,0 +1,484 @@ +#include "../Hierarchy/HybridHierarchy.h" +#include "InCacheBlock/DataRefactoring.h" +#include "MultiDimension/DataRefactoring.h" + +#ifndef MGARD_X_160_BLOCK_LOCAL_HIERARCHY_DATA_REFACTOR_HPP +#define MGARD_X_160_BLOCK_LOCAL_HIERARCHY_DATA_REFACTOR_HPP + +namespace mgard_x_160 { + +namespace data_refactoring { + +template +class BlockLocalHierarchyDataRefactor { +public: + BlockLocalHierarchyDataRefactor() : initialized(false) {} + + // Removing all checks for L since processed in HybridHierarchyDataRefactor + BlockLocalHierarchyDataRefactor(HybridHierarchy &hierarchy, + Config config) + : initialized(true), hierarchy(&hierarchy), config(config) { + hierarchy.check_config(config); + this->L = config.num_local_refactoring_level; + compute_local_ranges(); + w_array = Array<1, T, DeviceType>({DecomposedCoeffSize()}); + temp_coarest = Array<1, T, DeviceType>({coarse_num_elems[this->L - 1]}); + coarse_buffers.resize(2); + coarse_buffers[0] = Array(fine_shapes[0]); + coarse_buffers[1] = Array(fine_shapes[0]); + } + + void Adapt(HybridHierarchy &hierarchy, Config config, + int queue_idx) { + hierarchy.check_config(config); + this->initialized = true; + this->hierarchy = &hierarchy; + this->config = config; + this->L = config.num_local_refactoring_level; + compute_local_ranges(); + + w_array.resize({DecomposedCoeffSize()}, queue_idx); + temp_coarest.resize({coarse_num_elems[this->L - 1]}, queue_idx); + coarse_buffers.resize(2); + coarse_buffers[0].resize(fine_shapes[0], queue_idx); + coarse_buffers[1].resize(fine_shapes[0], queue_idx); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + // We have 2 arrays for shape switch and another one for output coeff and + // coarest + size_t size = 3; + for (DIM d = 0; d < shape.size(); d++) { + int dim8 = ((shape[d] - 1) / 8 + 1) * 8; + size *= dim8; + } + return size * sizeof(T); + } + + size_t DecomposedDataSize() { + SIZE decomposed_size = coarse_num_elems[this->L - 1]; + for (SIZE l = 0; l < this->L; l++) { + decomposed_size += local_coeff_size[l]; + } + return decomposed_size; + } + + size_t DecomposedCoeffSize() { + SIZE decomposed_coeff_size = 0; + for (SIZE l = 0; l < this->L; l++) { + decomposed_coeff_size += local_coeff_size[l]; + } + return decomposed_coeff_size; + } + + void compute_local_ranges() { + coarse_shape = hierarchy->input_shape(); + fine_num_elems.clear(); + coarse_num_elems.clear(); + local_coeff_size.clear(); + coarse_shapes.clear(); + fine_shapes.clear(); + + for (int l = 0; l < this->L; ++l) { + SIZE last_level_size = 1, curr_level_size = 1; + std::vector fine_shape = hierarchy->local_fine_shape(l); + coarse_shape = hierarchy->local_coarse_shape(l); + for (DIM d = 0; d < D; ++d) { + last_level_size *= fine_shape[d]; + curr_level_size *= coarse_shape[d]; + } + fine_num_elems.push_back(last_level_size); + coarse_num_elems.push_back(curr_level_size); + local_coeff_size.push_back(last_level_size - curr_level_size); + coarse_shapes.push_back(coarse_shape); + fine_shapes.push_back(fine_shape); + } + } + + void Decompose(SubArray data, + SubArray<1, T, DeviceType> output_decomposed, int queue_idx, + bool orthogonal_projection = true) { + SubArray fine(coarse_buffers[1]); + SubArray coarse; + // Zero the level-0 fine buffer when the input needs padding up to the + // next multiple of 8: CopyND below only writes the original extent, and + // 8x8x8 blocks straddling the boundary would otherwise mix uninitialized + // values into their coefficients. Deeper levels are unaffected (their + // fine buffer is a coarse buffer that is fully memset before use). + bool needs_padding = false; + for (DIM d = 0; d < D; d++) { + if (data.shape(d) != fine_shapes[0][d]) { + needs_padding = true; + break; + } + } + if (needs_padding) { + coarse_buffers[1].memset(0, queue_idx); + } + // CopyND follows the shape of 1st param + multi_dimension::CopyND(data, fine, queue_idx); + SubArray<1, T, DeviceType> decomposed_coeff(w_array); + + // Times the transform kernels only; the surrounding copies are covered by + // the Hybrid Decomposition timer in HybridHierarchyDataRefactor. + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + // Will be reused between decompose and recompose + accumulated_local_coeff_size = 0; + for (SIZE l = 0; l < this->L; l++) { + accumulated_local_coeff_size += local_coeff_size[l]; + // Think about a way to change this local_coeff + SubArray<1, T, DeviceType> local_coeff( + {local_coeff_size[l]}, + decomposed_coeff(decomposed_coeff.shape(0) - + accumulated_local_coeff_size)); + + int buffer_idx = l % 2; + coarse_buffers[buffer_idx].memset(0, queue_idx); + coarse = SubArray(coarse_shapes[l], + coarse_buffers[buffer_idx].data()); + for (DIM d = 0; d < D; d++) { + coarse.setLd(d, fine_shapes[0][d]); + } + coarse.project(D - 3, D - 2, D - 1); + + in_cache_block::decompose( + fine, coarse, local_coeff, orthogonal_projection, queue_idx); + if (l < this->L - 1) { + fine = SubArray(fine_shapes[l + 1], + coarse_buffers[buffer_idx].data()); + for (DIM d = 0; d < D; d++) { + fine.setLd(d, fine_shapes[0][d]); + } + fine.project(D - 3, D - 2, D - 1); + } + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Decomposition", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + + int final_buffer_id = (this->L - 1) % 2; + SubArray coarsest(coarse_shapes[this->L - 1], + coarse_buffers[final_buffer_id].data()); + for (DIM d = 0; d < D; d++) { + coarsest.setLd(d, fine_shapes[0][d]); + } + coarsest.project(D - 3, D - 2, D - 1); + // log::info("Find read buffer idx: " + std::to_string(final_buffer_id)); + + // Write the coarsest level directly into output_decomposed instead of + // staging through temp_coarest: temp_coarest is unused between here and + // the next Recompose() call, which repopulates it independently from + // input_decomposed (see below), so the staging copy was pure overhead. + SubArray coarsest_out(coarse_shapes[this->L - 1], + output_decomposed.data()); + for (DIM d = 0; d < D; d++) { + coarsest_out.setLd(d, coarse_shapes[this->L - 1][d]); + } + coarsest_out.project(D - 3, D - 2, D - 1); + multi_dimension::CopyND(coarsest, coarsest_out, queue_idx); + + SubArray<1, T, DeviceType> data_coeff({DecomposedCoeffSize()}, + output_decomposed.data() + + coarse_num_elems[this->L - 1]); + multi_dimension::CopyND(decomposed_coeff, data_coeff, queue_idx); + + // PrintSubarray("Temp in decompose:",SubArray(temp_coarest)); + } + + // Fused decompose+quantize. Runs the same per-level 8x8x8 decomposition as + // Decompose(), but each level's coefficients are quantized in-kernel and + // written directly to their final location in output_quantized, so the + // T-typed coefficient staging (w_array) and the copies into + // output_decomposed disappear. Level 0 reads straight from the (possibly + // unpadded) input, and deeper levels read the previous coarse buffer at its + // true extent — the fused kernel zero-fills out-of-range reads, replacing + // the padding memsets. Only the coarsest level is emitted in T, compacted + // at the front of output_decomposed for the global stage / coarsest + // quantization. + // + // Level l's quantizer: level_quantizers[l] (reciprocal), or per-block + // level_block_quantizers[l] in ROI mode (level_quantizers empty). + template + void DecomposeQuantize( + SubArray data, + SubArray<1, T, DeviceType> output_decomposed, + SubArray<1, Q, DeviceType> output_quantized, + const std::vector &level_quantizers, + const std::vector> &level_block_quantizers, + bool prep_huffman, SIZE dict_size, int queue_idx, + bool orthogonal_projection = true) { + bool use_block_quantizers = level_quantizers.empty(); + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SubArray fine = data; + SIZE accumulated = 0; + for (SIZE l = 0; l < this->L; l++) { + accumulated += local_coeff_size[l]; + SubArray<1, Q, DeviceType> level_quantized( + {local_coeff_size[l]}, + output_quantized(output_quantized.shape(0) - accumulated)); + + int buffer_idx = l % 2; + SubArray coarse; + if (l == this->L - 1) { + // Last level: write the coarsest data compactly to its final + // location instead of staging it in a padded buffer and copying. + coarse = SubArray(coarse_shapes[l], + output_decomposed.data()); + for (DIM d = 0; d < D; d++) { + coarse.setLd(d, coarse_shapes[l][d]); + } + } else { + coarse = SubArray(coarse_shapes[l], + coarse_buffers[buffer_idx].data()); + for (DIM d = 0; d < D; d++) { + coarse.setLd(d, fine_shapes[0][d]); + } + } + coarse.project(D - 3, D - 2, D - 1); + + in_cache_block::decompose_quantize( + fine, coarse, level_quantized, + use_block_quantizers ? (T)0 : level_quantizers[l], + use_block_quantizers ? level_block_quantizers[l] + : SubArray<1, T, DeviceType>(), + use_block_quantizers, prep_huffman, dict_size, orthogonal_projection, + queue_idx); + + if (l < this->L - 1) { + // Next level reads the coarse output at its true extent; the fused + // kernel's boundary handling supplies the zero padding. + fine = coarse; + } + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Decomposition+Quantization (fused)", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + // Fused dequantize+recompose. Runs the same per-level 8x8x8 recomposition + // as Recompose(), but each level's coefficients are read from their final + // location in input_quantized and dequantized in-kernel, so the T-typed + // coefficient region of the decomposed array is never materialized. Only + // the coarsest level is consumed in T from the front of input_decomposed + // (produced by the global stage / coarsest dequantization). The staging + // copies of the unfused path also disappear: the coarsest level is read + // compactly in place (no temp_coarest restore), the ping-pong buffers are + // not memset (every coarse value read at level l was written by level l+1, + // or comes from input_decomposed), and the final level writes directly to + // the unpadded output (the fused kernel bounds-checks its stores). + // + // Level l's dequantizer: level_dequantizers[l] (non-reciprocal), or + // per-block level_block_dequantizers[l] in ROI mode (level_dequantizers + // empty). Indexing matches DecomposeQuantize (level 0 = finest). + template + void RecomposeDequantize( + SubArray data, + SubArray<1, T, DeviceType> input_decomposed, + SubArray<1, Q, DeviceType> input_quantized, + const std::vector &level_dequantizers, + const std::vector> &level_block_dequantizers, + bool prep_huffman, SIZE dict_size, int queue_idx, + bool orthogonal_projection = true) { + bool use_block_quantizers = level_dequantizers.empty(); + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + // Coarsest level, read compactly in place from the front of + // input_decomposed instead of staging through temp_coarest. + SubArray coarse(coarse_shapes[this->L - 1], + input_decomposed.data()); + for (DIM d = 0; d < D; d++) { + coarse.setLd(d, coarse_shapes[this->L - 1][d]); + } + coarse.project(D - 3, D - 2, D - 1); + + SIZE accumulated = DecomposedCoeffSize(); + for (SIZE l = 0; l < this->L; l++) { + SIZE level_idx = this->L - l - 1; + + SubArray<1, Q, DeviceType> level_quantized( + {local_coeff_size[level_idx]}, + input_quantized(input_quantized.shape(0) - accumulated)); + + int buffer_idx = l % 2; + SubArray fine; + if (level_idx == 0) { + // Last level: write the reconstructed data directly to the unpadded + // output instead of staging it in a padded buffer and copying. + fine = data; + } else { + fine = SubArray(fine_shapes[level_idx], + coarse_buffers[buffer_idx].data()); + for (DIM d = 0; d < D; d++) { + fine.setLd(d, fine_shapes[0][d]); + } + fine.project(D - 3, D - 2, D - 1); + } + + in_cache_block::recompose_dequantize( + fine, coarse, level_quantized, + use_block_quantizers ? (T)0 : level_dequantizers[level_idx], + use_block_quantizers ? level_block_dequantizers[level_idx] + : SubArray<1, T, DeviceType>(), + use_block_quantizers, prep_huffman, dict_size, orthogonal_projection, + queue_idx); + + if (l < this->L - 1) { + coarse = SubArray(coarse_shapes[level_idx - 1], + coarse_buffers[buffer_idx].data()); + for (DIM d = 0; d < D; d++) { + coarse.setLd(d, fine_shapes[0][d]); + } + coarse.project(D - 3, D - 2, D - 1); + } + accumulated -= local_coeff_size[level_idx]; + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Recomposition+Dequantization (fused)", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + void Recompose(SubArray data, + SubArray<1, T, DeviceType> input_decomposed, int queue_idx, + bool orthogonal_projection = true) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + // Initialize accumulated_local_coeff_size so that Recompose works correctly + // regardless of whether Decompose was called first (e.g., standalone + // decompress). + accumulated_local_coeff_size = DecomposedCoeffSize(); + + // Restore temp_coarest from input_decomposed (the first + // coarse_num_elems[L-1] elements). This is critical for standalone + // decompression where Decompose was never called and temp_coarest was never + // populated. After global Recompose, + // input_decomposed[0..coarse_num_elems[L-1]-1] holds the correctly + // reconstructed coarsest values, which we must use here instead of + // stale/zero temp_coarest. + multi_dimension::CopyND( + SubArray<1, T, DeviceType>({coarse_num_elems[this->L - 1]}, + input_decomposed.data()), + SubArray(temp_coarest), queue_idx); + + coarse_buffers[0].memset(0, queue_idx); + coarse_buffers[1].memset(0, queue_idx); + + SubArray coarse(coarse_shapes[this->L - 1], + temp_coarest.data()); + + for (SIZE l = 0; l < this->L; l++) { + SIZE level_idx = this->L - l - 1; + + SubArray<1, T, DeviceType> local_coeff( + {local_coeff_size[level_idx]}, + input_decomposed(input_decomposed.shape(0) - + accumulated_local_coeff_size)); + + int buffer_idx = l % 2; + + SubArray fine(fine_shapes[level_idx], + coarse_buffers[buffer_idx].data()); + for (DIM d = 0; d < D; d++) { + fine.setLd(d, fine_shapes[0][d]); + } + fine.project(D - 3, D - 2, D - 1); + // log::info("Buffer idx for fine buffer: " + + // std::to_string(buffer_idx)); + + in_cache_block::recompose( + fine, coarse, local_coeff, orthogonal_projection, queue_idx); + + if (l < this->L - 1) { + coarse = SubArray(coarse_shapes[level_idx - 1], + coarse_buffers[buffer_idx].data()); + for (DIM d = 0; d < D; d++) { + coarse.setLd(d, fine_shapes[0][d]); + } + coarse.project(D - 3, D - 2, D - 1); + } + accumulated_local_coeff_size -= local_coeff_size[level_idx]; + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Recomposition", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + + // copy back, using ND + SubArray src( + hierarchy->level_shape(hierarchy->l_target()), + coarse_buffers[(this->L - 1) % 2].data()); + + for (DIM d = 0; d < D; d++) { + src.setLd(d, fine_shapes[0][d]); + } + src.project(D - 3, D - 2, D - 1); + + SubArray dst( + hierarchy->level_shape(hierarchy->l_target()), data.data()); + + multi_dimension::CopyND(src, dst, queue_idx); + } + + std::vector coarse_shape; + SIZE accumulated_local_coeff_size = 0; + bool initialized; + SIZE L; + HybridHierarchy *hierarchy; + Config config; + + std::vector fine_num_elems; + std::vector coarse_num_elems; + std::vector local_coeff_size; + std::vector> coarse_shapes; + std::vector> fine_shapes; + std::vector original_input_shape; + std::vector padded_input_shape; + + std::vector> coarse_buffers; + + Array<1, T, DeviceType> w_array; + Array<1, T, DeviceType> temp_coarest; +}; + +} // namespace data_refactoring +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/BlockLocalHierarchyDataRefactorInterface.hpp b/include/mgard-x-160/DataRefactoring/BlockLocalHierarchyDataRefactorInterface.hpp new file mode 100644 index 000000000..54d55d30d --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/BlockLocalHierarchyDataRefactorInterface.hpp @@ -0,0 +1,17 @@ +#ifndef MGARD_X_160_BLOCK_LOCAL_HIERARCHY_DATA_REFACTOR_INTERFACE_HPP +#define MGARD_X_160_BLOCK_LOCAL_HIERARCHY_DATA_REFACTOR_INTERFACE_HPP +namespace mgard_x_160 { + +namespace data_refactoring { + +template +class BlockLocalHierarchyDataRefactor { + virtual void Decompose(SubArray data, int queue_idx) = 0; + virtual void Recompose(SubArray data, int queue_idx) = 0; +}; + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/DataRefactor.hpp b/include/mgard-x-160/DataRefactoring/DataRefactor.hpp new file mode 100644 index 000000000..ddfde7fc3 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/DataRefactor.hpp @@ -0,0 +1,151 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "DataRefactorInterface.hpp" +#include "InCacheBlock/DataRefactoring.h" +#include "MultiDimension/DataRefactoring.h" +#include "SingleDimension/DataRefactoring.h" + +#ifndef MGARD_X_160_DATA_REFACTOROR_HPP +#define MGARD_X_160_DATA_REFACTOROR_HPP +namespace mgard_x_160 { + +namespace data_refactoring { + +template +class DataRefactor : public DataRefactorInterface { +public: + DataRefactor() : initialized(false) {} + DataRefactor(Hierarchy &hierarchy, Config config) + : initialized(true), hierarchy(&hierarchy), config(config) { + std::vector workspace_shape = + hierarchy.level_shape(hierarchy.l_target()); + for (DIM d = 0; d < D; d++) + workspace_shape[d] += 2; + w_array = Array(workspace_shape); + if (D > 3) { + b_array = Array(workspace_shape); + } + } + + void Adapt(Hierarchy &hierarchy, Config config, + int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + this->config = config; + std::vector workspace_shape = + hierarchy.level_shape(hierarchy.l_target()); + for (DIM d = 0; d < D; d++) + workspace_shape[d] += 2; + w_array.resize(workspace_shape, queue_idx); + if (D > 3) { + b_array.resize(workspace_shape, queue_idx); + } + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + Array<1, T, DeviceType> array_with_pitch({1}); + size_t pitch_size = array_with_pitch.ld(0) * sizeof(T); + + size_t size = 0; + size += sizeof(T); + size_t workspace_size = 1; + for (DIM d = 0; d < D; d++) { + if (d == D - 1) { + workspace_size *= + roundup((size_t)(shape[d] + 2) * sizeof(T), pitch_size); + } else { + workspace_size *= shape[d] + 2; + } + } + size += workspace_size; + if (D > 3) { + size += workspace_size; + } + return size; + } + + void Decompose(SubArray data, int start_level, + int stop_level, bool orthogonal_projection, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + SubArray w_subarray(w_array); + SubArray b_subarray; + if (D > 3) { + b_subarray = SubArray(b_array); + } + + if (config.decomposition == decomposition_type::MultiDim || + config.decomposition == decomposition_type::Hybrid) { + multi_dimension::decompose( + *hierarchy, data, w_subarray, b_subarray, start_level, stop_level, + orthogonal_projection, queue_idx); + } else if (config.decomposition == decomposition_type::SingleDim) { + single_dimension::decompose( + *hierarchy, data, start_level, stop_level, queue_idx); + } + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Global Decomposition", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + void Recompose(SubArray data, int start_level, + int stop_level, bool orthogonal_projection, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + SubArray w_subarray(w_array); + SubArray b_subarray; + if (D > 3) + b_subarray = SubArray(b_array); + if (config.decomposition == decomposition_type::MultiDim) { + multi_dimension::recompose( + *hierarchy, data, w_subarray, b_subarray, start_level, stop_level, + orthogonal_projection, queue_idx); + } else if (config.decomposition == decomposition_type::SingleDim) { + single_dimension::recompose( + *hierarchy, data, start_level, stop_level, queue_idx); + } + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Global Recomposition", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + void Decompose(SubArray data, bool orthogonal_projection, + int queue_idx) { + Decompose(data, hierarchy->l_target(), 0, orthogonal_projection, queue_idx); + } + + void Recompose(SubArray data, bool orthogonal_projection, + int queue_idx) { + Recompose(data, 0, hierarchy->l_target(), orthogonal_projection, queue_idx); + } + + bool initialized; + Hierarchy *hierarchy; + Config config; + Array w_array; + Array b_array; +}; + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/DataRefactorInterface.hpp b/include/mgard-x-160/DataRefactoring/DataRefactorInterface.hpp new file mode 100644 index 000000000..243bc57ae --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/DataRefactorInterface.hpp @@ -0,0 +1,31 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_DATA_REFACTOR_INTERFACE_HPP +#define MGARD_X_160_DATA_REFACTOR_INTERFACE_HPP +namespace mgard_x_160 { + +namespace data_refactoring { + +template class DataRefactorInterface { + virtual void Decompose(SubArray data, int start_level, + int stop_level, bool orthogonal_projection, + int queue_idx) = 0; + virtual void Recompose(SubArray data, int start_level, + int stop_level, bool orthogonal_projection, + int queue_idx) = 0; + virtual void Decompose(SubArray data, + bool orthogonal_projection, int queue_idx) = 0; + virtual void Recompose(SubArray data, + bool orthogonal_projection, int queue_idx) = 0; +}; + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/HybridHierarchyDataRefactor.hpp b/include/mgard-x-160/DataRefactoring/HybridHierarchyDataRefactor.hpp new file mode 100644 index 000000000..88bb71bbd --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/HybridHierarchyDataRefactor.hpp @@ -0,0 +1,253 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../Hierarchy/HybridHierarchy.h" +#include "../RuntimeX/Utilities/Exceptions.h" +#include "BlockLocalHierarchyDataRefactor.hpp" +#include "DataRefactor.hpp" +#include "HybridHierarchyDataRefactorInterface.hpp" +#include "InCacheBlock/DataRefactoring.h" +#include "MultiDimension/DataRefactoring.h" +#include "SingleDimension/DataRefactoring.h" + +#ifndef MGARD_X_160_HYBRID_HIERARCHY_DATA_REFACTOROR_HPP +#define MGARD_X_160_HYBRID_HIERARCHY_DATA_REFACTOROR_HPP +namespace mgard_x_160 { + +namespace data_refactoring { + +template +class HybridHierarchyDataRefactor + : public HybridHierarchyDataRefactorInterface { +public: + HybridHierarchyDataRefactor() : initialized(false) {} + HybridHierarchyDataRefactor(HybridHierarchy &hierarchy, + Config config) + : initialized(true), hierarchy(&hierarchy), config(config) { + this->L = config.num_local_refactoring_level; + this->M = config.num_global_refactoring_level; + } + + void Adapt(HybridHierarchy &hierarchy, Config config, + int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + this->config = config; + this->L = config.num_local_refactoring_level; + this->M = config.num_global_refactoring_level; + + // Adaptive intialization for local and global + if (this->L == 0 && this->M == 0) { + throw ProcessingException("Both L and M cannot be zero"); + } + hierarchy.check_config(config); + + if (this->L > 0) { + local_refactor.Adapt(hierarchy, config, queue_idx); + } + + if (this->M > 0) { + // The global hierarchy is built (once) by the HybridHierarchy, on the + // coarsest local grid or, without a local stage, on the input grid. + global_refactor.Adapt( + hierarchy.global_hierarchy(), + HybridHierarchy::global_config(this->M), queue_idx); + } + } + + static size_t EstimateMemoryFootprint(std::vector shape, + Config config) { + size_t size = 0; + + SIZE L = config.num_local_refactoring_level; + SIZE M = config.num_global_refactoring_level; + + if (L > 0) { + size += BlockLocalHierarchyDataRefactor< + D, T, DeviceType>::EstimateMemoryFootprint(shape); + if (M > 0) { + // Calculate Coarest shape from local + std::vector coarest_shape = shape; + for (SIZE l = 0; l < config.num_local_refactoring_level; l++) { + for (DIM d = 0; d < D; d++) { + coarest_shape[d] = ((coarest_shape[d] - 1) / 8 + 1) * 5; + } + } + size += DataRefactor::EstimateMemoryFootprint( + coarest_shape); + } + } else { + size += DataRefactor::EstimateMemoryFootprint(shape); + } + return size; + } + + size_t DecomposedDataSize() { + if (this->L > 0) { + return local_refactor.DecomposedDataSize(); + } + + return hierarchy->total_num_elems(); + } + + // Global-stage decomposition over the coarsest region at the front of + // decomposed_data (in-place). Factored out so the fused + // decompose+quantize path can run it separately from the local stage. + void DecomposeGlobal(SubArray<1, T, DeviceType> decomposed_data, + int queue_idx, bool orthogonal_projection = true) { + std::vector global_shape = hierarchy->global_shape(); + SubArray global_input_data(global_shape, + decomposed_data.data()); + for (DIM d = 0; d < D; d++) { + global_input_data.setLd(d, global_shape[d]); + } + global_input_data.project(D - 3, D - 2, D - 1); + + global_refactor.Decompose(global_input_data, orthogonal_projection, + queue_idx); + } + + // Global-stage recomposition over the coarsest region at the front of + // decomposed_data (in-place). Factored out so the fused + // dequantize+recompose path can run it separately from the local stage. + void RecomposeGlobal(SubArray<1, T, DeviceType> decomposed_data, + int queue_idx, bool orthogonal_projection = true) { + std::vector global_shape = hierarchy->global_shape(); + SubArray global_input_data(global_shape, + decomposed_data.data()); + for (DIM d = 0; d < D; d++) { + global_input_data.setLd(d, global_shape[d]); + } + global_input_data.project(D - 3, D - 2, D - 1); + + global_refactor.Recompose(global_input_data, orthogonal_projection, + queue_idx); + } + + // Need revise further to exclude copy time + void Decompose(SubArray data, + SubArray<1, T, DeviceType> decomposed_data, int queue_idx, + bool orthogonal_projection = true) { + if (this->L == 0 && this->M == 0) { + throw ProcessingException("Both L and M cannot be zero"); + } + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + if (this->L == 0) { + // Pure Global (In-Place) + std::vector original_shape = hierarchy->input_shape(); + SubArray global_input_data(original_shape, + decomposed_data.data()); + for (DIM d = 0; d < D; d++) { + global_input_data.setLd(d, original_shape[d]); + } + global_input_data.project(D - 3, D - 2, D - 1); + + multi_dimension::CopyND(data, global_input_data, queue_idx); + + global_refactor.Decompose(global_input_data, orthogonal_projection, + queue_idx); + } else if (this->M == 0) { + // Pure Local + local_refactor.Decompose(data, decomposed_data, queue_idx, + orthogonal_projection); + } else { + // Local decomposition + local_refactor.Decompose(data, decomposed_data, queue_idx, + orthogonal_projection); + + // Global decomposition + DecomposeGlobal(decomposed_data, queue_idx, orthogonal_projection); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Hybrid Decomposition", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + // Need revise further to exclude copy time + void Recompose(SubArray data, + SubArray<1, T, DeviceType> decomposed_data, int queue_idx, + bool orthogonal_projection = true) { + if (this->L == 0 && this->M == 0) { + throw ProcessingException("Both L and M cannot be zero"); + } + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + if (this->L == 0) { + // Pure Global (In-Place) + std::vector original_shape = hierarchy->input_shape(); + SubArray global_input_data(original_shape, + decomposed_data.data()); + for (DIM d = 0; d < D; d++) { + global_input_data.setLd(d, original_shape[d]); + } + global_input_data.project(D - 3, D - 2, D - 1); + + global_refactor.Recompose(global_input_data, orthogonal_projection, + queue_idx); + + // Copy back to data + multi_dimension::CopyND(global_input_data, data, queue_idx); + } else if (this->M == 0) { + // Pure Local + local_refactor.Recompose(data, decomposed_data, queue_idx, + orthogonal_projection); + } else { + std::vector local_coarest_shape = + local_refactor.coarse_shapes[this->L - 1]; + SubArray global_input_data({local_coarest_shape}, + decomposed_data.data()); + for (DIM d = 0; d < D; d++) { + global_input_data.setLd(d, local_coarest_shape[d]); + } + global_input_data.project(D - 3, D - 2, D - 1); + + // Global recomposition + global_refactor.Recompose(global_input_data, orthogonal_projection, + queue_idx); + + // Local recomposition + local_refactor.Recompose(data, decomposed_data, queue_idx, + orthogonal_projection); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Hybrid Recomposition", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + bool initialized; + HybridHierarchy *hierarchy; + Config config; + + SIZE L; // Number of local levels + SIZE M; // Number of global levels + + BlockLocalHierarchyDataRefactor local_refactor; + DataRefactor global_refactor; +}; + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/HybridHierarchyDataRefactorInterface.hpp b/include/mgard-x-160/DataRefactoring/HybridHierarchyDataRefactorInterface.hpp new file mode 100644 index 000000000..0a38598f2 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/HybridHierarchyDataRefactorInterface.hpp @@ -0,0 +1,28 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_HYBRID_HIERARCHY_DATA_REFACTOR_INTERFACE_HPP +#define MGARD_X_160_HYBRID_HIERARCHY_DATA_REFACTOR_INTERFACE_HPP +namespace mgard_x_160 { + +namespace data_refactoring { + +template +class HybridHierarchyDataRefactorInterface { + virtual void Decompose(SubArray data, + SubArray<1, T, DeviceType> decomposed_data, + int queue_idx, bool orthogonal_projection = true) = 0; + virtual void Recompose(SubArray data, + SubArray<1, T, DeviceType> decomposed_data, + int queue_idx, bool orthogonal_projection = true) = 0; +}; + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/InCacheBlock/Autocorrelation8x8x8.hpp b/include/mgard-x-160/DataRefactoring/InCacheBlock/Autocorrelation8x8x8.hpp new file mode 100644 index 000000000..89d5fe7fc --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/InCacheBlock/Autocorrelation8x8x8.hpp @@ -0,0 +1,239 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_AUTOCORRELATION_8x8x8_KERNEL_TEMPLATE +#define MGARD_X_160_AUTOCORRELATION_8x8x8_KERNEL_TEMPLATE + +#include "../../RuntimeX/RuntimeX.h" + +// #include "IndexTable3x3x3.hpp" +// #include "IndexTable5x5x5.hpp" +// #include "IndexTable8x8x8.hpp" + +#define DECOMPOSE 0 +#define RECOMPOSE 1 + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace in_cache_block { + +/* + +v x y z c total +8*8*8(512) 5*8*8(320) 5*5*8(200) 5*5*5(125) 0 1157 +5*5*5(125) 3*5*5(75) 3*3*5(45) 3*3*3(27) 8*8*8-5*5*5(387) 659 +3*3*3(27) 2*3*3(18) 2*2*3(12) 2*2*2(8) 8*8*8-3*3*3(485) 550 + + v(512) x(320) y(200) z(125) +c8(512) v(125) x( 75) y( 45) z(27) +c8(512) c5( 98) x( 18) y( 12) z( 8) +c8(512) c5( 98) c3(19) c2( 8) +*/ + +template +class Autocorrelation8x8x8Functor : public Functor { +public: + MGARDX_CONT Autocorrelation8x8x8Functor() {} + MGARDX_CONT + Autocorrelation8x8x8Functor(SubArray v, + SubArray autocorrelation_x, + SubArray autocorrelation_y, + SubArray autocorrelation_z, + int lag) + : v(v), autocorrelation_x(autocorrelation_x), + autocorrelation_y(autocorrelation_y), + autocorrelation_z(autocorrelation_z), lag(lag) { + Functor(); + } + + MGARDX_EXEC void initialize_sm_8x8x8() { + sm_v = (T *)FunctorBase::GetSharedMemory(); + sm_x = sm_v + 8 * 8 * 8; + sm_y = sm_x + 5 * 8 * 8; + sm_z = sm_y + 5 * 5 * 8; + } + + MGARDX_EXEC T Mean(T *data, SIZE ld, SIZE n) { + T sum = 0; + for (SIZE i = 0; i < n; i++) { + sum += data[i * ld]; + } + return sum / n; + } + + MGARDX_EXEC T Autocorrelation(T *data, SIZE ld, SIZE n, int lag) { + T mean = Mean(data, ld, n); + T cov = 0; + T sdv = 0; + for (SIZE i = 0; i < n; i++) { + T diff1 = data[i * ld] - mean; + T diff2 = data[((i + lag) % n) * ld] - mean; + cov += diff1 * diff2; + sdv += diff1 * diff1; + } + return cov / sdv; + } + + // Interpolation + MGARDX_EXEC void Operation1() { + initialize_sm_8x8x8(); + x = FunctorBase::GetThreadIdX(); + y = FunctorBase::GetThreadIdY(); + z = FunctorBase::GetThreadIdZ(); + x_tb = FunctorBase::GetBlockIdX(); + y_tb = FunctorBase::GetBlockIdY(); + z_tb = FunctorBase::GetBlockIdZ(); + x_gl = X * x_tb + x; + y_gl = Y * y_tb + y; + z_gl = Z * z_tb + z; + + tid = z * X * Y + y * X + x; + bid = z_tb * FunctorBase::GetGridDimX() * + FunctorBase::GetGridDimY() + + y_tb * FunctorBase::GetGridDimX() + x_tb; + if (z == 0 && y == 0 && x == 0) + sm_v[zero_const_offset] = (T)0; + + offset = get_idx(ld1, ld2, z, y, x); + sm_v[offset] = 0.0; + // Removing this check can speed up + if (z_gl < v.shape(D - 3) && y_gl < v.shape(D - 2) && + x_gl < v.shape(D - 1)) { + sm_v[offset] = *v(z_gl, y_gl, x_gl); + } + if (tid == 0) { + + T ac_x = 0, ac; + // printf("ac_x: "); + for (int z_id = 0; z_id < 8; z_id++) { + for (int y_id = 0; y_id < 8; y_id++) { + ac = Autocorrelation(&sm_v[y_id * 8 + z_id * 8 * 8], 1, 8, lag); + // printf("%f ", ac); + ac_x += ac; + } + } + // printf("\n"); + ac_x /= 64; + // printf("ac_x: %f\n", ac_x); + + T ac_y = 0; + // printf("ac_x: "); + for (int z_id = 0; z_id < 8; z_id++) { + for (int x_id = 0; x_id < 8; x_id++) { + ac = Autocorrelation(&sm_v[x_id + z_id * 8 * 8], 8, 8, lag); + ac_y += ac; + // printf("%f ", ac); + } + } + // printf("\n"); + ac_y /= 64; + // printf("ac_y: %f\n", ac_y); + + T ac_z = 0; + // printf("ac_x: "); + for (int y_id = 0; y_id < 8; y_id++) { + for (int x_id = 0; x_id < 8; x_id++) { + ac = Autocorrelation(&sm_v[x_id + y_id * 8], 8 * 8, 8, lag); + ac_z += ac; + // printf("%f ", ac); + } + } + // printf("\n"); + ac_z /= 64; + // printf("ac_z: %f\n", ac_z); + + *autocorrelation_x(z_tb, y_tb, x_tb) = ac_x; + *autocorrelation_y(z_tb, y_tb, x_tb) = ac_y; + *autocorrelation_z(z_tb, y_tb, x_tb) = ac_z; + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = (Z * Y * X) + Z * Y * (X / 2 + 1) + + Z * (Y / 2 + 1) * (X / 2 + 1) + + (Z / 2 + 1) * (Y / 2 + 1) * (X / 2 + 1) + 1; + return size * sizeof(T); + } + +private: + SubArray v; + SubArray autocorrelation_x; + SubArray autocorrelation_y; + SubArray autocorrelation_z; + int lag; + T *sm_v, *sm_x, *sm_y, *sm_z, *sm_c8, *sm_c5, *sm_c3, *sm_c2; + int ld1 = X; + int ld2 = Y; + int z, y, x, z_tb, y_tb, x_tb, z_gl, y_gl, x_gl; + int tid, bid, op_tid; + T left, right, middle; + int offset; + int zero_const_offset = (Z * Y * X) + Z * Y * (X / 2 + 1) + + Z * (Y / 2 + 1) * (X / 2 + 1) + + (Z / 2 + 1) * (Y / 2 + 1) * (X / 2 + 1); + // #ifdef MGARDX_COMPILE_CUDA + // clock_t start, end; + // #endif +}; + +template +class Autocorrelation8x8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk"; + MGARDX_CONT + Autocorrelation8x8x8Kernel(SubArray v, + SubArray autocorrelation_x, + SubArray autocorrelation_y, + SubArray autocorrelation_z, + int lag) + : v(v), autocorrelation_x(autocorrelation_x), + autocorrelation_y(autocorrelation_y), + autocorrelation_z(autocorrelation_z), lag(lag) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + Autocorrelation8x8x8Functor; + FunctorType functor(v, autocorrelation_x, autocorrelation_y, + autocorrelation_z, lag); + + SIZE total_thread_z = v.shape(D - 3); + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 8; + tby = 8; + tbx = 8; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray autocorrelation_x; + SubArray autocorrelation_y; + SubArray autocorrelation_z; + int lag; +}; + +} // namespace in_cache_block + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/InCacheBlock/DataRefactoring.h b/include/mgard-x-160/DataRefactoring/InCacheBlock/DataRefactoring.h new file mode 100644 index 000000000..e93d66aeb --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/InCacheBlock/DataRefactoring.h @@ -0,0 +1,71 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_IN_CACHE_DATA_REFACTORING +#define MGARD_X_160_IN_CACHE_DATA_REFACTORING + +// #include "Common.h" +#include "../../Hierarchy/Hierarchy.h" +#include "../../RuntimeX/RuntimeXPublic.h" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace in_cache_block { + +// Block-local (in-cache) refactoring for D = 1, 2 and 3. Every dimension of a +// block holds 8 fine nodes that coarsen to 5, so one block emits 5^D coarse +// values and 8^D - 5^D coefficients. Higher D is a no-op: the hybrid +// hierarchy has no block-local stage there. + +template +void decompose(SubArray v, SubArray coarse, + SubArray<1, T, DeviceType> coeff, bool orthogonal_projection, + int queue_idx); + +// Fused decompose+quantize: same block decomposition as decompose(), but the +// coefficients are quantized in-kernel and written as Q symbols. Reads v with +// bounds checks, so v does not need to be padded to a multiple of 8. Uses the +// per-block quantizers (indexed by the block's row-major index) when +// use_block_quantizers is set (ROI mode), the scalar quantizer otherwise. +template +void decompose_quantize(SubArray v, + SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection, + int queue_idx); + +template +void recompose(SubArray v, SubArray coarse, + SubArray<1, T, DeviceType> coeff, bool orthogonal_projection, + int queue_idx); + +// Fused dequantize+recompose: same block recomposition as recompose(), but the +// coefficients are read as Q symbols and dequantized in-kernel. Writes v with +// bounds checks, so v does not need to be padded to a multiple of 8. Uses the +// per-block quantizers (indexed by the block's row-major index) when +// use_block_quantizers is set (ROI mode), the scalar quantizer otherwise. +template +void recompose_dequantize(SubArray v, + SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, + T quantizer, + SubArray<1, T, DeviceType> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection, + int queue_idx); + +} // namespace in_cache_block + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/InCacheBlock/DataRefactoring.hpp b/include/mgard-x-160/DataRefactoring/InCacheBlock/DataRefactoring.hpp new file mode 100644 index 000000000..fbb2d5ca0 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/InCacheBlock/DataRefactoring.hpp @@ -0,0 +1,150 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include + +#include "../../Hierarchy/Hierarchy.h" +#include "../../RuntimeX/RuntimeX.h" +#include "Autocorrelation8x8x8.hpp" +#include "DataRefactoring.h" +#include "Decompose8.hpp" +#include "Decompose8x8.hpp" +#include "Decompose8x8x8.hpp" +#include "Recompose8.hpp" +#include "Recompose8x8.hpp" +#include "Recompose8x8x8.hpp" + +#include + +#ifndef MGARD_X_160_IN_CACHE_BLOCK_DATA_REFACTORING_HPP +#define MGARD_X_160_IN_CACHE_BLOCK_DATA_REFACTORING_HPP + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace in_cache_block { + +template +void decompose(SubArray v, SubArray coarse, + SubArray<1, T, DeviceType> coeff, bool orthogonal_projection, + int queue_idx) { + // One kernel per dimensionality: the block geometry (8 fine nodes to 5 + // coarse ones per dimension) is shared, but the coefficient layout and the + // number of transform passes are not. + if constexpr (D == 1) { + DeviceLauncher::Execute( + Decompose8Kernel(v, coarse, coeff, + orthogonal_projection), + queue_idx); + } else if constexpr (D == 2) { + DeviceLauncher::Execute( + Decompose8x8Kernel(v, coarse, coeff, + orthogonal_projection), + queue_idx); + } else if constexpr (D == 3) { + DeviceLauncher::Execute( + Decompose8x8x8Kernel(v, coarse, coeff, + orthogonal_projection), + queue_idx); + } +} + +template +void decompose_quantize(SubArray v, + SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection, + int queue_idx) { + if constexpr (D == 1) { + DeviceLauncher::Execute( + DecomposeQuantize8Kernel( + v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection), + queue_idx); + } else if constexpr (D == 2) { + DeviceLauncher::Execute( + DecomposeQuantize8x8Kernel( + v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection), + queue_idx); + } else if constexpr (D == 3) { + DeviceLauncher::Execute( + DecomposeQuantize8x8x8Kernel( + v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection), + queue_idx); + } +} + +template +void recompose(SubArray v, SubArray coarse, + SubArray<1, T, DeviceType> coeff, bool orthogonal_projection, + int queue_idx) { + if constexpr (D == 1) { + DeviceLauncher::Execute( + Recompose8Kernel(v, coarse, coeff, + orthogonal_projection), + queue_idx); + } else if constexpr (D == 2) { + DeviceLauncher::Execute( + Recompose8x8Kernel(v, coarse, coeff, + orthogonal_projection), + queue_idx); + } else if constexpr (D == 3) { + DeviceLauncher::Execute( + Recompose8x8x8Kernel(v, coarse, coeff, + orthogonal_projection), + queue_idx); + } +} + +template +void recompose_dequantize(SubArray v, + SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, + T quantizer, + SubArray<1, T, DeviceType> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection, + int queue_idx) { + if constexpr (D == 1) { + DeviceLauncher::Execute( + RecomposeDequantize8Kernel( + v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection), + queue_idx); + } else if constexpr (D == 2) { + DeviceLauncher::Execute( + RecomposeDequantize8x8Kernel( + v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection), + queue_idx); + } else if constexpr (D == 3) { + DeviceLauncher::Execute( + RecomposeDequantize8x8x8Kernel( + v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection), + queue_idx); + } +} + +} // namespace in_cache_block + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/InCacheBlock/Decompose8.hpp b/include/mgard-x-160/DataRefactoring/InCacheBlock/Decompose8.hpp new file mode 100644 index 000000000..304f5add7 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/InCacheBlock/Decompose8.hpp @@ -0,0 +1,331 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_DECOMPOSE_8_KERNEL_TEMPLATE +#define MGARD_X_160_DECOMPOSE_8_KERNEL_TEMPLATE + +#include "../../RuntimeX/RuntimeX.h" + +#include "IndexTableLowDim.hpp" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace in_cache_block { + +/* +1D counterpart of Decompose8x8x8Functor: + +v x c total +8 5(cor) 8-5(3) 13 per tile + +A single 8-element tile is far too little work for a thread block, so each +block owns TILES tiles side by side: threadIdx.y selects the tile and +threadIdx.x the element within it. Every shared-memory table entry is a +within-tile offset, to which the kernel adds the tile's base. + +The 3 coefficients per tile are written to coeff[tile * 3 ...] and the 5 +coarse values to coarse[tile * 5 ...], where `tile` is the tile's index in +row-major order over the whole array -- the same convention the 3D block uses +for its linearized thread-block id. +*/ +template +class Decompose8Functor : public Functor { +public: + MGARDX_CONT Decompose8Functor() {} + MGARDX_CONT Decompose8Functor(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) { + Functor(); + } + + MGARDX_EXEC void initialize_sm_8() { + sm_v = (T *)FunctorBase::GetSharedMemory(); + sm_x = sm_v + TILES * SMV_STRIDE_8; + } + + MGARDX_EXEC void init_indices() { + item = FunctorBase::GetThreadIdX(); + tile = FunctorBase::GetThreadIdY(); + bid = FunctorBase::GetBlockIdX() * TILES + tile; + num_tiles = (v.shape(D - 1) + X - 1) / X; + active = bid < (int)num_tiles; + base_v = tile * SMV_STRIDE_8; + base_x = tile * SMX_STRIDE_8; + x_gl = bid * X + item; + if (item == 0) + sm_v[base_v + SMV_ZERO_8] = (T)0; + } + + // Load + MGARDX_EXEC void Operation1() { + initialize_sm_8(); + init_indices(); + if (active) { + sm_v[base_v + item] = *v(x_gl); + } + } + + // Interpolation + MGARDX_EXEC void Operation2() { + if (active && item < NumCoeff_8) { + int const *index = Coeff1D_Offset_8(item); + T middle = sm_v[base_v + index[0]]; + T left = sm_v[base_v + index[1]]; + T right = sm_v[base_v + index[2]]; + sm_v[base_v + index[0]] = middle - (left + right) * (T)0.5; + } + } + + // MassTransX + MGARDX_EXEC void Operation3() { + if (!orthogonal_projection) + return; + if (active && item < NumMassTransX_8) { + int const *index = MassTrans_X_Offset_8(item); + T a = sm_v[base_v + index[0]]; + T b = sm_v[base_v + index[1]]; + T c = sm_v[base_v + index[2]]; + T d = sm_v[base_v + index[3]]; + T e = sm_v[base_v + index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_x[base_x + index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + } + + MGARDX_EXEC void solve_tridiag() { + T a = sm_x[base_x + 0]; + T b = sm_x[base_x + 1]; + T c = sm_x[base_x + 2]; + T d = sm_x[base_x + 3]; + T e = sm_x[base_x + 4]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_x[base_x + 0] = a; + sm_x[base_x + 1] = b; + sm_x[base_x + 2] = c; + sm_x[base_x + 3] = d; + sm_x[base_x + 4] = e; + } + + // TridiagX + MGARDX_EXEC void Operation4() { + if (!orthogonal_projection) + return; + if (active && item == 0) { + solve_tridiag(); + } + } + + // Apply correction and write out + MGARDX_EXEC void Operation5() { + if (!active) + return; + if (item < NumCoarse_8) { + if (orthogonal_projection) + sm_v[base_v + Coarse_Offset_8(item)] += sm_x[base_x + item]; + *coarse(bid * LowDim_Coarse + item) = + sm_v[base_v + Coarse_Offset_8(item)]; + } else { + int op_tid = item - NumCoarse_8; + *coeff(bid * NumCoeff_8 + op_tid) = sm_v[base_v + Coeff_Offset_8(op_tid)]; + } + } + + MGARDX_CONT size_t shared_memory_size() { + return (size_t)TILES * (SMV_STRIDE_8 + SMX_STRIDE_8) * sizeof(T); + } + +protected: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; + T *sm_v, *sm_x; + int item, tile, bid, x_gl; + int base_v, base_x; + SIZE num_tiles; + bool active; +}; + +// Fused decompose+quantize variant. See DecomposeQuantize8x8x8Functor: the +// input is read with bounds checks so unpadded inputs can be consumed +// directly, and the 3 per-tile coefficients are quantized on write-out. +template +class DecomposeQuantize8Functor + : public Decompose8Functor { + using Base = Decompose8Functor; + +public: + MGARDX_CONT DecomposeQuantize8Functor() {} + MGARDX_CONT DecomposeQuantize8Functor( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : Base(v, coarse, SubArray<1, T, DeviceType>(), orthogonal_projection), + quantized_coeff(quantized_coeff), quantizer(quantizer), + block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size) {} + + MGARDX_EXEC void Operation1() { + this->initialize_sm_8(); + this->init_indices(); + if (this->active) { + this->sm_v[this->base_v + this->item] = (T)0; + if (this->x_gl < (int)this->v.shape(D - 1)) { + this->sm_v[this->base_v + this->item] = *this->v(this->x_gl); + } + } + } + + MGARDX_EXEC void Operation5() { + if (!this->active) + return; + if (this->item < NumCoarse_8) { + if (this->orthogonal_projection) + this->sm_v[this->base_v + Coarse_Offset_8(this->item)] += + this->sm_x[this->base_x + this->item]; + *this->coarse(this->bid * LowDim_Coarse + this->item) = + this->sm_v[this->base_v + Coarse_Offset_8(this->item)]; + } else { + int op_tid = this->item - NumCoarse_8; + T t = this->sm_v[this->base_v + Coeff_Offset_8(op_tid)]; + T q = use_block_quantizers ? *block_quantizers(this->bid) : quantizer; + // Must stay bit-identical to QuantizeLocalLevelFunctor (volume == 1). + Q quantized_data; + if constexpr (sizeof(T) == sizeof(double)) { + quantized_data = copysign((T)0.5 + fabs(t * q), t); + } else if constexpr (sizeof(T) == sizeof(float)) { + quantized_data = copysign((T)0.5 + fabsf(t * q), t); + } + if (prep_huffman) { + quantized_data += dict_size / 2; + } + *quantized_coeff(this->bid * NumCoeff_8 + op_tid) = quantized_data; + } + } + +protected: + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; +}; + +template +class Decompose8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_1d"; + MGARDX_CONT + Decompose8Kernel(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Decompose8Functor; + FunctorType functor(v, coarse, coeff, orthogonal_projection); + + SIZE num_tiles = (v.shape(D - 1) + 7) / 8; + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = LowDim_Tiles_1D, tbx = 8; + SIZE gridz = 1, gridy = 1; + SIZE gridx = ceil((double)num_tiles / LowDim_Tiles_1D); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; +}; + +template +class DecomposeQuantize8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_1d_fq"; + MGARDX_CONT + DecomposeQuantize8Kernel(SubArray v, + SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, + T quantizer, + SubArray<1, T, DeviceType> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection = true) + : v(v), coarse(coarse), quantized_coeff(quantized_coeff), + quantizer(quantizer), block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size), orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT + Task> + GenTask(int queue_idx) { + using FunctorType = + DecomposeQuantize8Functor; + FunctorType functor(v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection); + + // Same launch geometry as Decompose8Kernel; v may be unpadded here but + // ceil(shape / 8) matches the padded tile count exactly. + SIZE num_tiles = (v.shape(D - 1) + 7) / 8; + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = LowDim_Tiles_1D, tbx = 8; + SIZE gridz = 1, gridy = 1; + SIZE gridx = ceil((double)num_tiles / LowDim_Tiles_1D); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; + bool orthogonal_projection; +}; + +} // namespace in_cache_block + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/InCacheBlock/Decompose8x8.hpp b/include/mgard-x-160/DataRefactoring/InCacheBlock/Decompose8x8.hpp new file mode 100644 index 000000000..8fc08e212 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/InCacheBlock/Decompose8x8.hpp @@ -0,0 +1,368 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_DECOMPOSE_8x8_KERNEL_TEMPLATE +#define MGARD_X_160_DECOMPOSE_8x8_KERNEL_TEMPLATE + +#include "../../RuntimeX/RuntimeX.h" + +#include "IndexTableLowDim.hpp" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace in_cache_block { + +/* +2D counterpart of Decompose8x8x8Functor. One thread block owns one 8x8 tile +and 64 threads; the stages are the same as the 3D block's, minus the z pass: + +v x y c total +8*8(64) 5*8(40) 5*5(25) 0 129 + 8*8-5*5(39) + +The 39 coefficients per tile are written to coeff[bid * 39 ...] in the +row-major non-coarse order of Coeff_Offset_8x8, and the 25 coarse values to +coarse[y_tb * 5 + jy][x_tb * 5 + jx]. +*/ +template +class Decompose8x8Functor : public Functor { +public: + MGARDX_CONT Decompose8x8Functor() {} + MGARDX_CONT Decompose8x8Functor(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) { + Functor(); + } + + MGARDX_EXEC void initialize_sm_8x8() { + sm_v = (T *)FunctorBase::GetSharedMemory(); + sm_x = sm_v + SMV_SIZE_8x8; // sm_v is padded for bank-conflict avoidance + sm_y = sm_x + SMX_SIZE_8x8; + } + + MGARDX_EXEC void init_indices() { + x = FunctorBase::GetThreadIdX(); + y = FunctorBase::GetThreadIdY(); + x_tb = FunctorBase::GetBlockIdX(); + y_tb = FunctorBase::GetBlockIdY(); + x_gl = X * x_tb + x; + y_gl = Y * y_tb + y; + tid = y * X + x; + bid = y_tb * FunctorBase::GetGridDimX() + x_tb; + if (tid == 0) + sm_v[ZERO_V_8x8] = (T)0; + offset = offset8x8(y, x); + } + + // Load + MGARDX_EXEC void Operation1() { + initialize_sm_8x8(); + init_indices(); + sm_v[offset] = *v(y_gl, x_gl); + } + + // Interpolation. The 1D and 2D coefficient updates are independent: a 1D + // update writes a node with exactly one odd coordinate, while the 2D update + // reads only nodes with two even coordinates. + MGARDX_EXEC void Operation2() { + if (tid < NumCoeff1D_8x8) { + int const *index = Coeff1D_Offset_8x8(tid); + T middle = sm_v[index[0]]; + T left = sm_v[index[1]]; + T right = sm_v[index[2]]; + sm_v[index[0]] = middle - (left + right) * (T)0.5; + } else if (tid >= 32 && tid < 32 + NumCoeff2D_8x8) { + int const *index = Coeff2D_Offset_8x8(tid - 32); + T c11 = sm_v[index[0]]; + T c00 = sm_v[index[1]]; + T c02 = sm_v[index[2]]; + T c20 = sm_v[index[3]]; + T c22 = sm_v[index[4]]; + sm_v[index[0]] = c11 - (c00 + c02 + c20 + c22) / 4; + } + } + + // MassTransX + MGARDX_EXEC void Operation3() { + if (!orthogonal_projection) + return; + if (tid < NumMassTransX_8x8) { + int const *index = MassTrans_X_Offset_8x8(tid); + T a = sm_v[index[0]]; + T b = sm_v[index[1]]; + T c = sm_v[index[2]]; + T d = sm_v[index[3]]; + T e = sm_v[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_x[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + } + + // MassTransY + MGARDX_EXEC void Operation4() { + if (!orthogonal_projection) + return; + if (tid < NumMassTransY_8x8) { + int const *index = MassTrans_Y_Offset_8x8(tid); + T a = sm_x[index[0]]; + T b = sm_x[index[1]]; + T c = sm_x[index[2]]; + T d = sm_x[index[3]]; + T e = sm_x[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_y[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + } + + MGARDX_EXEC void solve_tridiag(int const *index) { + T a = sm_y[index[0]]; + T b = sm_y[index[1]]; + T c = sm_y[index[2]]; + T d = sm_y[index[3]]; + T e = sm_y[index[4]]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_y[index[0]] = a; + sm_y[index[1]] = b; + sm_y[index[2]] = c; + sm_y[index[3]] = d; + sm_y[index[4]] = e; + } + + // TridiagX + MGARDX_EXEC void Operation5() { + if (!orthogonal_projection) + return; + if (tid < LowDim_Coarse) { + solve_tridiag(TriDiag_X_Offset_8x8(tid)); + } + } + + // TridiagY + MGARDX_EXEC void Operation6() { + if (!orthogonal_projection) + return; + if (tid < LowDim_Coarse) { + solve_tridiag(TriDiag_Y_Offset_8x8(tid)); + } + } + + // Apply correction and write out + MGARDX_EXEC void Operation7() { + if (tid < NumCoarse_8x8) { + if (orthogonal_projection) + sm_v[Coarse_Offset_8x8(tid)] += sm_y[tid]; + *coarse(y_tb * LowDim_Coarse + tid / LowDim_Coarse, + x_tb * LowDim_Coarse + tid % LowDim_Coarse) = + sm_v[Coarse_Offset_8x8(tid)]; + } else { + int op_tid = tid - NumCoarse_8x8; + *coeff(bid * NumCoeff_8x8 + op_tid) = sm_v[Coeff_Offset_8x8(op_tid)]; + } + } + + MGARDX_CONT size_t shared_memory_size() { + return (size_t)SM_SIZE_8x8 * sizeof(T); + } + +protected: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; + T *sm_v, *sm_x, *sm_y; + int y, x, y_tb, x_tb, y_gl, x_gl; + int tid, bid; + int offset; +}; + +// Fused decompose+quantize variant: identical transform pipeline, but +// (1) the input is read with bounds checks so unpadded inputs can be consumed +// directly (out-of-range positions contribute zeros, matching the +// memset+copy padding of the unfused path bit for bit), and +// (2) the 39 per-tile coefficients are quantized on write-out while still in +// shared memory, emitting Q symbols (with the Huffman dictionary shift +// folded in, mirroring QuantizeLocalLevelFunctor) instead of T values. +// The 5x5 coarse output stays in T: it is the next level's input. +template +class DecomposeQuantize8x8Functor + : public Decompose8x8Functor { + using Base = Decompose8x8Functor; + +public: + MGARDX_CONT DecomposeQuantize8x8Functor() {} + MGARDX_CONT DecomposeQuantize8x8Functor( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : Base(v, coarse, SubArray<1, T, DeviceType>(), orthogonal_projection), + quantized_coeff(quantized_coeff), quantizer(quantizer), + block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size) {} + + MGARDX_EXEC void Operation1() { + this->initialize_sm_8x8(); + this->init_indices(); + this->sm_v[this->offset] = (T)0; + // Unlike the unfused functor, keep the bounds check: the input here is + // not pre-padded to a multiple of 8, so edge tiles must zero-fill. + if (this->y_gl < (int)this->v.shape(D - 2) && + this->x_gl < (int)this->v.shape(D - 1)) { + this->sm_v[this->offset] = *this->v(this->y_gl, this->x_gl); + } + } + + MGARDX_EXEC void Operation7() { + if (this->tid < NumCoarse_8x8) { + if (this->orthogonal_projection) + this->sm_v[Coarse_Offset_8x8(this->tid)] += this->sm_y[this->tid]; + *this->coarse(this->y_tb * LowDim_Coarse + this->tid / LowDim_Coarse, + this->x_tb * LowDim_Coarse + this->tid % LowDim_Coarse) = + this->sm_v[Coarse_Offset_8x8(this->tid)]; + } else { + int op_tid = this->tid - NumCoarse_8x8; + T t = this->sm_v[Coeff_Offset_8x8(op_tid)]; + T q = use_block_quantizers ? *block_quantizers(this->bid) : quantizer; + // Must stay bit-identical to QuantizeLocalLevelFunctor (volume == 1). + Q quantized_data; + if constexpr (sizeof(T) == sizeof(double)) { + quantized_data = copysign((T)0.5 + fabs(t * q), t); + } else if constexpr (sizeof(T) == sizeof(float)) { + quantized_data = copysign((T)0.5 + fabsf(t * q), t); + } + if (prep_huffman) { + quantized_data += dict_size / 2; + } + *quantized_coeff(this->bid * NumCoeff_8x8 + op_tid) = quantized_data; + } + } + +protected: + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; +}; + +template +class Decompose8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_2d"; + MGARDX_CONT + Decompose8x8Kernel(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Decompose8x8Functor; + FunctorType functor(v, coarse, coeff, orthogonal_projection); + + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = 8, tbx = 8; + SIZE gridz = 1; + SIZE gridy = ceil((double)total_thread_y / tby); + SIZE gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; +}; + +template +class DecomposeQuantize8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_2d_fq"; + MGARDX_CONT + DecomposeQuantize8x8Kernel(SubArray v, + SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, + T quantizer, + SubArray<1, T, DeviceType> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection = true) + : v(v), coarse(coarse), quantized_coeff(quantized_coeff), + quantizer(quantizer), block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size), orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = DecomposeQuantize8x8Functor; + FunctorType functor(v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection); + + // Same launch geometry as Decompose8x8Kernel; v may be unpadded here but + // ceil(shape / 8) matches the padded grid exactly. + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = 8, tbx = 8; + SIZE gridz = 1; + SIZE gridy = ceil((double)total_thread_y / tby); + SIZE gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; + bool orthogonal_projection; +}; + +} // namespace in_cache_block + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/InCacheBlock/Decompose8x8x8.hpp b/include/mgard-x-160/DataRefactoring/InCacheBlock/Decompose8x8x8.hpp new file mode 100644 index 000000000..b59e9359c --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/InCacheBlock/Decompose8x8x8.hpp @@ -0,0 +1,673 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_DECOMPOSE_8x8x8_KERNEL_TEMPLATE +#define MGARD_X_160_DECOMPOSE_8x8x8_KERNEL_TEMPLATE + +#include "../../RuntimeX/RuntimeX.h" + +#include "../MultiDimension/Correction/LPKFunctor.h" + +#include "../MultiDimension/Correction/IPKFunctor.h" + +#include "IndexTable3x3x3.hpp" +#include "IndexTable5x5x5.hpp" +#include "IndexTable8x8x8.hpp" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace in_cache_block { + +/* + +v x y z c total +8*8*8(512) 5*8*8(320) 5*5*8(200) 5*5*5(125) 0 1157 +5*5*5(125) 3*5*5(75) 3*3*5(45) 3*3*3(27) 8*8*8-5*5*5(387) 659 +3*3*3(27) 2*3*3(18) 2*2*3(12) 2*2*2(8) 8*8*8-3*3*3(485) 550 + + v(512) x(320) y(200) z(125) +c8(512) v(125) x( 75) y( 45) z(27) +c8(512) c5( 98) x( 18) y( 12) z( 8) +c8(512) c5( 98) c3(19) c2( 8) +*/ + +template +class Decompose8x8x8Functor : public Functor { +public: + MGARDX_CONT Decompose8x8x8Functor() {} + MGARDX_CONT Decompose8x8x8Functor(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) { + Functor(); + } + + MGARDX_EXEC void initialize_sm_8x8x8() { + sm_v = (T *)FunctorBase::GetSharedMemory(); + sm_x = sm_v + SMV_SIZE_8x8x8; // sm_v is padded for bank-conflict avoidance + sm_y = sm_x + 5 * 8 * 8; + sm_z = sm_y + 5 * 5 * 8; + } + + MGARDX_EXEC void initialize_sm_5x5x5() { + sm_c8 = (T *)FunctorBase::GetSharedMemory(); + sm_v = sm_c8 + 8 * 8 * 8; + sm_x = sm_v + 5 * 5 * 5; + sm_y = sm_x + 5 * 5 * 3; + sm_z = sm_y + 5 * 3 * 3; + } + + MGARDX_EXEC void initialize_sm_3x3x3() { + sm_c8 = (T *)FunctorBase::GetSharedMemory(); + sm_c5 = sm_c8 + 8 * 8 * 8; + sm_v = sm_c5 + 5 * 5 * 5; + sm_x = sm_v + 3 * 3 * 3; + sm_y = sm_x + 3 * 3 * 2; + sm_z = sm_y + 3 * 2 * 2; + } + + MGARDX_EXEC void initialize_sm_2x2x2() { + sm_c8 = (T *)FunctorBase::GetSharedMemory(); + sm_c5 = sm_c8 + 8 * 8 * 8; + sm_c3 = sm_c5 + 5 * 5 * 5; + sm_c2 = sm_c3 + 3 * 3 * 3; + } + + // Interpolation + MGARDX_EXEC void Operation1() { + initialize_sm_8x8x8(); + x = FunctorBase::GetThreadIdX(); + y = FunctorBase::GetThreadIdY(); + z = FunctorBase::GetThreadIdZ(); + x_tb = FunctorBase::GetBlockIdX(); + y_tb = FunctorBase::GetBlockIdY(); + z_tb = FunctorBase::GetBlockIdZ(); + x_gl = X * x_tb + x; + y_gl = Y * y_tb + y; + z_gl = Z * z_tb + z; + + tid = z * X * Y + y * X + x; + bid = z_tb * FunctorBase::GetGridDimX() * + FunctorBase::GetGridDimY() + + y_tb * FunctorBase::GetGridDimX() + x_tb; + if (z == 0 && y == 0 && x == 0) + sm_v[zero_const_offset] = (T)0; + + offset = offset8x8x8(z, y, x); // padded sm_v layout + sm_v[offset] = 0.0; + // Removing this check can speed up + // if (z_gl < v.shape(D - 3) && y_gl < v.shape(D - 2) && + // x_gl < v.shape(D - 1)) { + sm_v[offset] = *v(z_gl, y_gl, x_gl); + // printf("v[%d, %d, %d] = %f\n", z_gl, y_gl, x_gl, sm_v[offset]); + // } + } + + MGARDX_EXEC void Operation2() { + // #ifdef MGARDX_COMPILE_CUDA + // start = clock(); + // #endif + + op_tid = tid; + if (tid < 225) { + left = sm_v[Coeff1D_L_Offset_8x8x8(op_tid)]; + right = sm_v[Coeff1D_R_Offset_8x8x8(op_tid)]; + middle = sm_v[Coeff1D_M_Offset_8x8x8(op_tid)]; + // printf("l %f, r %f, m %f\n", left, right, middle); + middle = middle - (left + right) * (T)0.5; + sm_v[Coeff1D_M_Offset_8x8x8(op_tid)] = middle; + } else if (tid >= 256 && tid < 256 + 135) { + op_tid -= 256; + T c00 = sm_v[Coeff2D_LL_Offset_8x8x8(op_tid)]; + T c02 = sm_v[Coeff2D_LR_Offset_8x8x8(op_tid)]; + T c20 = sm_v[Coeff2D_RL_Offset_8x8x8(op_tid)]; + T c22 = sm_v[Coeff2D_RR_Offset_8x8x8(op_tid)]; + T c11 = sm_v[Coeff2D_MM_Offset_8x8x8(op_tid)]; + c11 -= (c00 + c02 + c20 + c22) / 4; + sm_v[Coeff2D_MM_Offset_8x8x8(op_tid)] = c11; + } else if (tid >= 416 && tid < 416 + 27) { + op_tid -= 416; + T c000 = sm_v[Coeff3D_LLL_Offset_8x8x8(op_tid)]; + T c002 = sm_v[Coeff3D_LLR_Offset_8x8x8(op_tid)]; + T c020 = sm_v[Coeff3D_LRL_Offset_8x8x8(op_tid)]; + T c022 = sm_v[Coeff3D_LRR_Offset_8x8x8(op_tid)]; + T c200 = sm_v[Coeff3D_RLL_Offset_8x8x8(op_tid)]; + T c202 = sm_v[Coeff3D_RLR_Offset_8x8x8(op_tid)]; + T c220 = sm_v[Coeff3D_RRL_Offset_8x8x8(op_tid)]; + T c222 = sm_v[Coeff3D_RRR_Offset_8x8x8(op_tid)]; + T c111 = sm_v[Coeff3D_MMM_Offset_8x8x8(op_tid)]; + c111 -= (c000 + c002 + c020 + c022 + c200 + c202 + c220 + c222) / 8; + sm_v[Coeff3D_MMM_Offset_8x8x8(op_tid)] = c111; + } + } + + // MassTransX + MGARDX_EXEC void Operation3() { + if (!orthogonal_projection) + return; + + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 0) { + // printf("int\n"); + // for (int i = 0; i < 8; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 8; j++) { + // for (int k = 0; k < 8; k++) { + // printf("%.6f ", sm_v[get_idx(8, 8, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + + if (tid < 320) { + int const *index = MassTrans_X_Offset_8x8x8(tid); + T a = sm_v[index[0]]; + T b = sm_v[index[1]]; + T c = sm_v[index[2]]; + T d = sm_v[index[3]]; + T e = sm_v[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_x[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 5) { + // printf("tra - x\n"); + // for (int i = 0; i < 8; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 8; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%.6f ", sm_x[get_idx(5, 8, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // MassTransY + MGARDX_EXEC void Operation4() { + if (!orthogonal_projection) + return; + if (tid < 200) { + int const *index = MassTrans_Y_Offset_8x8x8(tid); + T a = sm_x[index[0]]; + T b = sm_x[index[1]]; + T c = sm_x[index[2]]; + T d = sm_x[index[3]]; + T e = sm_x[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_y[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 0) { + // printf("tra - y\n"); + // for (int i = 0; i < 8; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 5; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%.6f ", sm_y[get_idx(5, 5, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // MassTransZ + MGARDX_EXEC void Operation5() { + if (!orthogonal_projection) + return; + if (tid < 125) { + int const *index = MassTrans_Z_Offset_8x8x8(tid); + T a = sm_y[index[0]]; + T b = sm_y[index[1]]; + T c = sm_y[index[2]]; + T d = sm_y[index[3]]; + T e = sm_y[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_z[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 0) { + // printf("tra - z\n"); + // for (int i = 0; i < 5; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 5; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%.6f ", sm_z[get_idx(5, 5, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // TriadiagX + MGARDX_EXEC void Operation6() { + if (!orthogonal_projection) + return; + if (tid < 25) { + int const *index = TriDiag_X_Offset_8x8x8(tid); + T a = sm_z[index[0]]; + T b = sm_z[index[1]]; + T c = sm_z[index[2]]; + T d = sm_z[index[3]]; + T e = sm_z[index[4]]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_z[index[0]] = a; + sm_z[index[1]] = b; + sm_z[index[2]] = c; + sm_z[index[3]] = d; + sm_z[index[4]] = e; + } + + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 0) { + // printf("tri - x\n"); + // for (int i = 0; i < 5; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 5; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%.6f ", sm_z[get_idx(5, 5, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // TriadiagY + MGARDX_EXEC void Operation7() { + if (!orthogonal_projection) + return; + if (tid < 25) { + int const *index = TriDiag_Y_Offset_8x8x8(tid); + T a = sm_z[index[0]]; + T b = sm_z[index[1]]; + T c = sm_z[index[2]]; + T d = sm_z[index[3]]; + T e = sm_z[index[4]]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_z[index[0]] = a; + sm_z[index[1]] = b; + sm_z[index[2]] = c; + sm_z[index[3]] = d; + sm_z[index[4]] = e; + } + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 0) { + // printf("tri - y\n"); + // for (int i = 0; i < 5; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 5; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%.6f ", sm_z[get_idx(5, 5, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // TriadiagZ + MGARDX_EXEC void Operation8() { + if (!orthogonal_projection) + return; + if (tid < 25) { + int const *index = TriDiag_Z_Offset_8x8x8(tid); + T a = sm_z[index[0]]; + T b = sm_z[index[1]]; + T c = sm_z[index[2]]; + T d = sm_z[index[3]]; + T e = sm_z[index[4]]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_z[index[0]] = a; + sm_z[index[1]] = b; + sm_z[index[2]] = c; + sm_z[index[3]] = d; + sm_z[index[4]] = e; + } + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (bid == 0 && tid == 0) { + // printf("tri - z\n"); + // for (int i = 0; i < 5; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 5; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%10.2f ", sm_z[get_idx(5, 5, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // Apply Correction + MGARDX_EXEC void Operation9() { + if (tid < 125) { + if (orthogonal_projection) + sm_v[Coarse_Offset_8x8x8(tid)] += sm_z[tid]; + int const *index = Coarse_Reorder_8x8x8(tid); + *coarse(z_tb * 5 + index[0], y_tb * 5 + index[1], x_tb * 5 + index[2]) = + sm_v[Coarse_Offset_8x8x8(tid)]; + } else { + int op_tid = tid - 125; + *coeff(bid * 387 + op_tid) = sm_v[Coeff_Offset_8x8x8(op_tid)]; + // if (bid == 0 && op_tid == 0) { + // for (int i = 0; i < 387; i++) { + // printf("%f ", sm_v[Coeff_Offset_8x8x8(i)]); + // } + // printf("\n"); + // } + } + + // *w(z_gl, y_gl, x_gl) = sm_v[offset]; + // initialize_sm_3x3x3(); + // if (tid < 125) { + // sm_v[tid] = coarse + correction; + // } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = SMV_SIZE_8x8x8 + Z * Y * (X / 2 + 1) + + Z * (Y / 2 + 1) * (X / 2 + 1) + + (Z / 2 + 1) * (Y / 2 + 1) * (X / 2 + 1) + 1; + return size * sizeof(T); + } + +protected: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; + T *sm_v, *sm_x, *sm_y, *sm_z, *sm_c8, *sm_c5, *sm_c3, *sm_c2; + int ld1 = X; + int ld2 = Y; + int z, y, x, z_tb, y_tb, x_tb, z_gl, y_gl, x_gl; + int tid, bid, op_tid; + T left, right, middle; + int offset; + int zero_const_offset = SMV_SIZE_8x8x8 + Z * Y * (X / 2 + 1) + + Z * (Y / 2 + 1) * (X / 2 + 1) + + (Z / 2 + 1) * (Y / 2 + 1) * (X / 2 + 1); + // #ifdef MGARDX_COMPILE_CUDA + // clock_t start, end; + // #endif +}; + +// Fused decompose+quantize variant: identical transform pipeline, but +// (1) the input is read with bounds checks so unpadded inputs can be consumed +// directly (out-of-range positions contribute zeros, matching the +// memset+copy padding of the unfused path bit for bit), and +// (2) the 387 per-block coefficients are quantized on write-out while still +// in shared memory, emitting Q symbols (with the Huffman dictionary shift +// folded in, mirroring QuantizeLocalLevelFunctor) instead of T values. +// The 5x5x5 coarse output stays in T: it is the next level's input. +template +class DecomposeQuantize8x8x8Functor + : public Decompose8x8x8Functor { + using Base = Decompose8x8x8Functor; + +public: + MGARDX_CONT DecomposeQuantize8x8x8Functor() {} + MGARDX_CONT DecomposeQuantize8x8x8Functor( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : Base(v, coarse, SubArray<1, T, DeviceType>(), orthogonal_projection), + quantized_coeff(quantized_coeff), quantizer(quantizer), + block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size) {} + + MGARDX_EXEC void Operation1() { + this->initialize_sm_8x8x8(); + this->x = FunctorBase::GetThreadIdX(); + this->y = FunctorBase::GetThreadIdY(); + this->z = FunctorBase::GetThreadIdZ(); + this->x_tb = FunctorBase::GetBlockIdX(); + this->y_tb = FunctorBase::GetBlockIdY(); + this->z_tb = FunctorBase::GetBlockIdZ(); + this->x_gl = X * this->x_tb + this->x; + this->y_gl = Y * this->y_tb + this->y; + this->z_gl = Z * this->z_tb + this->z; + + this->tid = this->z * X * Y + this->y * X + this->x; + this->bid = this->z_tb * FunctorBase::GetGridDimX() * + FunctorBase::GetGridDimY() + + this->y_tb * FunctorBase::GetGridDimX() + + this->x_tb; + if (this->z == 0 && this->y == 0 && this->x == 0) + this->sm_v[this->zero_const_offset] = (T)0; + + this->offset = offset8x8x8(this->z, this->y, this->x); // padded sm_v layout + this->sm_v[this->offset] = 0.0; + // Unlike the unfused functor, keep the bounds check: the input here is + // not pre-padded to a multiple of 8, so edge blocks must zero-fill. + if (this->z_gl < (int)this->v.shape(D - 3) && + this->y_gl < (int)this->v.shape(D - 2) && + this->x_gl < (int)this->v.shape(D - 1)) { + this->sm_v[this->offset] = *this->v(this->z_gl, this->y_gl, this->x_gl); + } + } + + MGARDX_EXEC void Operation9() { + if (this->tid < 125) { + if (this->orthogonal_projection) + this->sm_v[Coarse_Offset_8x8x8(this->tid)] += this->sm_z[this->tid]; + int const *index = Coarse_Reorder_8x8x8(this->tid); + *this->coarse(this->z_tb * 5 + index[0], this->y_tb * 5 + index[1], + this->x_tb * 5 + index[2]) = + this->sm_v[Coarse_Offset_8x8x8(this->tid)]; + } else { + int op_tid = this->tid - 125; + T t = this->sm_v[Coeff_Offset_8x8x8(op_tid)]; + T q = use_block_quantizers ? *block_quantizers(this->bid) : quantizer; + // Must stay bit-identical to QuantizeLocalLevelFunctor (volume == 1). + Q quantized_data; + if constexpr (sizeof(T) == sizeof(double)) { + quantized_data = copysign((T)0.5 + fabs(t * q), t); + } else if constexpr (sizeof(T) == sizeof(float)) { + quantized_data = copysign((T)0.5 + fabsf(t * q), t); + } + if (prep_huffman) { + quantized_data += dict_size / 2; + } + *quantized_coeff(this->bid * 387 + op_tid) = quantized_data; + } + } + +protected: + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; +}; + +template +class Decompose8x8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk"; + MGARDX_CONT + Decompose8x8x8Kernel(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Decompose8x8x8Functor; + FunctorType functor(v, coarse, coeff, orthogonal_projection); + + SIZE total_thread_z = v.shape(D - 3); + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 8; + tby = 8; + tbx = 8; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; +}; + +template +class DecomposeQuantize8x8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_fq"; + MGARDX_CONT + DecomposeQuantize8x8x8Kernel( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : v(v), coarse(coarse), quantized_coeff(quantized_coeff), + quantizer(quantizer), block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size), orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + DecomposeQuantize8x8x8Functor; + FunctorType functor(v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection); + + // Same launch geometry as Decompose8x8x8Kernel; v may be unpadded here + // but ceil(shape / 8) matches the padded grid exactly. + SIZE total_thread_z = v.shape(D - 3); + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 8; + tby = 8; + tbx = 8; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; + bool orthogonal_projection; +}; + +} // namespace in_cache_block + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTable3x3x3.hpp b/include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTable3x3x3.hpp new file mode 100644 index 000000000..f66e2f903 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTable3x3x3.hpp @@ -0,0 +1,371 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_INDEX_TABLE_3X3X3_HPP +#define MGARD_X_160_INDEX_TABLE_3X3X3_HPP + +namespace mgard_x_160 { +// clang-format off + +MGARDX_EXEC constexpr int offset3x3x3(SIZE z, SIZE y, SIZE x) { + return z * 3 * 3 + y * 3 + x; +} + +MGARDX_EXEC constexpr int offset3x3x3(SIZE z, SIZE y, SIZE x, SIZE ld1, SIZE ld2) { + return z * ld1 * ld2 + y * ld1 + x; +} + +static constexpr int c1d_x_3x3x3[4][3] = { {0, 0, 1}, // X + {0, 2, 1}, + {2, 0, 1}, + {2, 2, 1} + }; + +static constexpr int c1d_y_3x3x3[4][3] = { {0, 1, 0},// Y + {0, 1, 2}, + {2, 1, 0}, + {2, 1, 2} + }; + +static constexpr int c1d_z_3x3x3[4][3] = { {1, 0, 0}, // Z + {1, 0, 2}, + {1, 2, 0}, + {1, 2, 2} + }; + +static constexpr int c2d_xy_3x3x3[2][3] = { {0, 1, 1}, // XY + {2, 1, 1} + }; + +static constexpr int c2d_yz_3x3x3[2][3] = { {1, 1, 0}, // YZ + {1, 1, 2} + }; + +static constexpr int c2d_xz_3x3x3[2][3] = { {1, 0, 1}, // XZ + {1, 2, 1} + }; + +static constexpr int c3d_xyz_3x3x3[1][3] = { {1, 1, 1}// XYZ + }; + +static constexpr int coarse_x_3x3x3[2] = {0, 2}; + +static constexpr int coarse_y_3x3x3[2] = {0, 2}; + +static constexpr int coarse_z_3x3x3[2] = {0, 2}; + +MGARDX_EXEC int Coeff1D_M_Offset_3x3x3(SIZE i) { + static constexpr int offset[12] = { offset3x3x3(c1d_x_3x3x3[0][0], c1d_x_3x3x3[0][1], c1d_x_3x3x3[0][2]), // X + offset3x3x3(c1d_x_3x3x3[1][0], c1d_x_3x3x3[1][1], c1d_x_3x3x3[1][2]), + offset3x3x3(c1d_x_3x3x3[2][0], c1d_x_3x3x3[2][1], c1d_x_3x3x3[2][2]), + offset3x3x3(c1d_x_3x3x3[3][0], c1d_x_3x3x3[3][1], c1d_x_3x3x3[3][2]), + + offset3x3x3(c1d_y_3x3x3[0][0], c1d_y_3x3x3[0][1], c1d_y_3x3x3[0][2]), // Y + offset3x3x3(c1d_y_3x3x3[1][0], c1d_y_3x3x3[1][1], c1d_y_3x3x3[1][2]), + offset3x3x3(c1d_y_3x3x3[2][0], c1d_y_3x3x3[2][1], c1d_y_3x3x3[2][2]), + offset3x3x3(c1d_y_3x3x3[3][0], c1d_y_3x3x3[3][1], c1d_y_3x3x3[3][2]), + + offset3x3x3(c1d_z_3x3x3[0][0], c1d_z_3x3x3[0][1], c1d_z_3x3x3[0][2]), // Z + offset3x3x3(c1d_z_3x3x3[1][0], c1d_z_3x3x3[1][1], c1d_z_3x3x3[1][2]), + offset3x3x3(c1d_z_3x3x3[2][0], c1d_z_3x3x3[2][1], c1d_z_3x3x3[2][2]), + offset3x3x3(c1d_z_3x3x3[3][0], c1d_z_3x3x3[3][1], c1d_z_3x3x3[3][2]) + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff1D_L_Offset_3x3x3(SIZE i) { + static constexpr int offset[12] = { offset3x3x3(c1d_x_3x3x3[0][0], c1d_x_3x3x3[0][1], c1d_x_3x3x3[0][2]-1), // X + offset3x3x3(c1d_x_3x3x3[1][0], c1d_x_3x3x3[1][1], c1d_x_3x3x3[1][2]-1), + offset3x3x3(c1d_x_3x3x3[2][0], c1d_x_3x3x3[2][1], c1d_x_3x3x3[2][2]-1), + offset3x3x3(c1d_x_3x3x3[3][0], c1d_x_3x3x3[3][1], c1d_x_3x3x3[3][2]-1), + + offset3x3x3(c1d_y_3x3x3[0][0], c1d_y_3x3x3[0][1]-1, c1d_y_3x3x3[0][2]), // Y + offset3x3x3(c1d_y_3x3x3[1][0], c1d_y_3x3x3[1][1]-1, c1d_y_3x3x3[1][2]), + offset3x3x3(c1d_y_3x3x3[2][0], c1d_y_3x3x3[2][1]-1, c1d_y_3x3x3[2][2]), + offset3x3x3(c1d_y_3x3x3[3][0], c1d_y_3x3x3[3][1]-1, c1d_y_3x3x3[3][2]), + + offset3x3x3(c1d_z_3x3x3[0][0]-1, c1d_z_3x3x3[0][1], c1d_z_3x3x3[0][2]), // Z + offset3x3x3(c1d_z_3x3x3[1][0]-1, c1d_z_3x3x3[1][1], c1d_z_3x3x3[1][2]), + offset3x3x3(c1d_z_3x3x3[2][0]-1, c1d_z_3x3x3[2][1], c1d_z_3x3x3[2][2]), + offset3x3x3(c1d_z_3x3x3[3][0]-1, c1d_z_3x3x3[3][1], c1d_z_3x3x3[3][2]) + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff1D_R_Offset_3x3x3(SIZE i) { + static constexpr int offset[12] = { offset3x3x3(c1d_x_3x3x3[0][0], c1d_x_3x3x3[0][1], c1d_x_3x3x3[0][2]+1), // X + offset3x3x3(c1d_x_3x3x3[1][0], c1d_x_3x3x3[1][1], c1d_x_3x3x3[1][2]+1), + offset3x3x3(c1d_x_3x3x3[2][0], c1d_x_3x3x3[2][1], c1d_x_3x3x3[2][2]+1), + offset3x3x3(c1d_x_3x3x3[3][0], c1d_x_3x3x3[3][1], c1d_x_3x3x3[3][2]+1), + + offset3x3x3(c1d_y_3x3x3[0][0], c1d_y_3x3x3[0][1]+1, c1d_y_3x3x3[0][2]), // Y + offset3x3x3(c1d_y_3x3x3[1][0], c1d_y_3x3x3[1][1]+1, c1d_y_3x3x3[1][2]), + offset3x3x3(c1d_y_3x3x3[2][0], c1d_y_3x3x3[2][1]+1, c1d_y_3x3x3[2][2]), + offset3x3x3(c1d_y_3x3x3[3][0], c1d_y_3x3x3[3][1]+1, c1d_y_3x3x3[3][2]), + + offset3x3x3(c1d_z_3x3x3[0][0]+1, c1d_z_3x3x3[0][1], c1d_z_3x3x3[0][2]), // Z + offset3x3x3(c1d_z_3x3x3[1][0]+1, c1d_z_3x3x3[1][1], c1d_z_3x3x3[1][2]), + offset3x3x3(c1d_z_3x3x3[2][0]+1, c1d_z_3x3x3[2][1], c1d_z_3x3x3[2][2]), + offset3x3x3(c1d_z_3x3x3[3][0]+1, c1d_z_3x3x3[3][1], c1d_z_3x3x3[3][2]) + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff2D_MM_Offset_3x3x3(SIZE i) { + static constexpr int offset[6] = { offset3x3x3(c2d_xy_3x3x3[0][0], c2d_xy_3x3x3[0][1], c2d_xy_3x3x3[0][2]), // XY + offset3x3x3(c2d_xy_3x3x3[1][0], c2d_xy_3x3x3[1][1], c2d_xy_3x3x3[1][2]), + + offset3x3x3(c2d_xz_3x3x3[0][0], c2d_xz_3x3x3[0][1], c2d_xz_3x3x3[0][2]), // XZ + offset3x3x3(c2d_xz_3x3x3[1][0], c2d_xz_3x3x3[1][1], c2d_xz_3x3x3[1][2]), + + offset3x3x3(c2d_yz_3x3x3[0][0], c2d_yz_3x3x3[0][1], c2d_yz_3x3x3[0][2]), // YZ + offset3x3x3(c2d_yz_3x3x3[1][0], c2d_yz_3x3x3[1][1], c2d_yz_3x3x3[1][2]) + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff2D_LL_Offset_3x3x3(SIZE i) { + static constexpr int offset[6] = { offset3x3x3(c2d_xy_3x3x3[0][0], c2d_xy_3x3x3[0][1]-1, c2d_xy_3x3x3[0][2]-1), // XY + offset3x3x3(c2d_xy_3x3x3[1][0], c2d_xy_3x3x3[1][1]-1, c2d_xy_3x3x3[1][2]-1), + + offset3x3x3(c2d_xz_3x3x3[0][0]-1, c2d_xz_3x3x3[0][1], c2d_xz_3x3x3[0][2]-1), // XZ + offset3x3x3(c2d_xz_3x3x3[1][0]-1, c2d_xz_3x3x3[1][1], c2d_xz_3x3x3[1][2]-1), + + offset3x3x3(c2d_yz_3x3x3[0][0]-1, c2d_yz_3x3x3[0][1]-1, c2d_yz_3x3x3[0][2]), // YZ + offset3x3x3(c2d_yz_3x3x3[1][0]-1, c2d_yz_3x3x3[1][1]-1, c2d_yz_3x3x3[1][2]) + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff2D_LR_Offset_3x3x3(SIZE i) { + static constexpr int offset[6] = { offset3x3x3(c2d_xy_3x3x3[0][0], c2d_xy_3x3x3[0][1]-1, c2d_xy_3x3x3[0][2]+1), // XY + offset3x3x3(c2d_xy_3x3x3[1][0], c2d_xy_3x3x3[1][1]-1, c2d_xy_3x3x3[1][2]+1), + + offset3x3x3(c2d_xz_3x3x3[0][0]-1, c2d_xz_3x3x3[0][1], c2d_xz_3x3x3[0][2]+1), // XZ + offset3x3x3(c2d_xz_3x3x3[1][0]-1, c2d_xz_3x3x3[1][1], c2d_xz_3x3x3[1][2]+1), + + offset3x3x3(c2d_yz_3x3x3[0][0]-1, c2d_yz_3x3x3[0][1]+1, c2d_yz_3x3x3[0][2]), // YZ + offset3x3x3(c2d_yz_3x3x3[1][0]-1, c2d_yz_3x3x3[1][1]+1, c2d_yz_3x3x3[1][2]) + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff2D_RL_Offset_3x3x3(SIZE i) { + static constexpr int offset[6] = { offset3x3x3(c2d_xy_3x3x3[0][0], c2d_xy_3x3x3[0][1]+1, c2d_xy_3x3x3[0][2]-1), // XY + offset3x3x3(c2d_xy_3x3x3[1][0], c2d_xy_3x3x3[1][1]+1, c2d_xy_3x3x3[1][2]-1), + + offset3x3x3(c2d_xz_3x3x3[0][0]+1, c2d_xz_3x3x3[0][1], c2d_xz_3x3x3[0][2]-1), // XZ + offset3x3x3(c2d_xz_3x3x3[1][0]+1, c2d_xz_3x3x3[1][1], c2d_xz_3x3x3[1][2]-1), + + offset3x3x3(c2d_yz_3x3x3[0][0]+1, c2d_yz_3x3x3[0][1]-1, c2d_yz_3x3x3[0][2]), // YZ + offset3x3x3(c2d_yz_3x3x3[1][0]+1, c2d_yz_3x3x3[1][1]-1, c2d_yz_3x3x3[1][2]), + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff2D_RR_Offset_3x3x3(SIZE i) { + static constexpr int offset[6] = { offset3x3x3(c2d_xy_3x3x3[0][0], c2d_xy_3x3x3[0][1]+1, c2d_xy_3x3x3[0][2]+1), // XY + offset3x3x3(c2d_xy_3x3x3[1][0], c2d_xy_3x3x3[1][1]+1, c2d_xy_3x3x3[1][2]+1), + + offset3x3x3(c2d_xz_3x3x3[0][0]+1, c2d_xz_3x3x3[0][1], c2d_xz_3x3x3[0][2]+1), // XZ + offset3x3x3(c2d_xz_3x3x3[1][0]+1, c2d_xz_3x3x3[1][1], c2d_xz_3x3x3[1][2]+1), + + offset3x3x3(c2d_yz_3x3x3[0][0]+1, c2d_yz_3x3x3[0][1]+1, c2d_yz_3x3x3[0][2]), // YZ + offset3x3x3(c2d_yz_3x3x3[1][0]+1, c2d_yz_3x3x3[1][1]+1, c2d_yz_3x3x3[1][2]) + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_MMM_Offset_3x3x3(SIZE i) { + static constexpr int offset[1] = {offset3x3x3(c3d_xyz_3x3x3[0][0], c3d_xyz_3x3x3[0][1], c3d_xyz_3x3x3[0][2])}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_LLL_Offset_3x3x3(SIZE i) { + static constexpr int offset[1] = {offset3x3x3(c3d_xyz_3x3x3[0][0]-1, c3d_xyz_3x3x3[0][1]-1, c3d_xyz_3x3x3[0][2]-1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_LLR_Offset_3x3x3(SIZE i) { + static constexpr int offset[1] = {offset3x3x3(c3d_xyz_3x3x3[0][0]-1, c3d_xyz_3x3x3[0][1]-1, c3d_xyz_3x3x3[0][2]+1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_LRL_Offset_3x3x3(SIZE i) { + static constexpr int offset[1] = {offset3x3x3(c3d_xyz_3x3x3[0][0]-1, c3d_xyz_3x3x3[0][1]+1, c3d_xyz_3x3x3[0][2]-1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_LRR_Offset_3x3x3(SIZE i) { + static constexpr int offset[1] = {offset3x3x3(c3d_xyz_3x3x3[0][0]-1, c3d_xyz_3x3x3[0][1]+1, c3d_xyz_3x3x3[0][2]+1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_RLL_Offset_3x3x3(SIZE i) { + static constexpr int offset[1] = {offset3x3x3(c3d_xyz_3x3x3[0][0]+1, c3d_xyz_3x3x3[0][1]-1, c3d_xyz_3x3x3[0][2]-1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_RLR_Offset_3x3x3(SIZE i) { + static constexpr int offset[1] = {offset3x3x3(c3d_xyz_3x3x3[0][0]+1, c3d_xyz_3x3x3[0][1]-1, c3d_xyz_3x3x3[0][2]+1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_RRL_Offset_3x3x3(SIZE i) { + static constexpr int offset[1] = {offset3x3x3(c3d_xyz_3x3x3[0][0]+1, c3d_xyz_3x3x3[0][1]+1, c3d_xyz_3x3x3[0][2]-1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_RRR_Offset_3x3x3(SIZE i) { + static constexpr int offset[1] = {offset3x3x3(c3d_xyz_3x3x3[0][0]+1, c3d_xyz_3x3x3[0][1]+1, c3d_xyz_3x3x3[0][2]+1)}; + + return offset[i]; +} + +MGARDX_EXEC int const *MassTrans_X_Offset_3x3x3(SIZE i) { + static constexpr int zero_offset = 0; + #define OFFSET(Z, Y) \ + { \ + zero_offset, \ + zero_offset, \ + offset3x3x3(Z, Y, coarse_x_3x3x3[0], 3, 3), \ + offset3x3x3(Z, Y, coarse_x_3x3x3[0]+1, 3, 3), \ + offset3x3x3(Z, Y, coarse_x_3x3x3[0]+2, 3, 3), \ + offset3x3x3(Z, Y, coarse_x_3x3x3[0], 2, 3) \ + }, \ + { \ + offset3x3x3(Z, Y, coarse_x_3x3x3[1]-2, 3, 3), \ + offset3x3x3(Z, Y, coarse_x_3x3x3[1]-1, 3, 3), \ + offset3x3x3(Z, Y, coarse_x_3x3x3[1], 3, 3), \ + zero_offset, \ + zero_offset, \ + offset3x3x3(Z, Y, coarse_x_3x3x3[1], 2, 3) \ + } + + static constexpr int offset[18][6] = { + OFFSET(0, 0), OFFSET(0, 1), OFFSET(0, 2), + OFFSET(1, 0), OFFSET(1, 1), OFFSET(1, 2), + OFFSET(2, 0), OFFSET(2, 1), OFFSET(2, 2) + }; + #undef OFFSET + return offset[i]; +} + +MGARDX_EXEC int const *MassTrans_Y_Offset_3x3x3(SIZE i) { + static constexpr int zero_offset = 0; + #define OFFSET(Z, X) \ + { \ + zero_offset, \ + zero_offset, \ + offset3x3x3(Z, coarse_y_3x3x3[0], X, 2, 3), \ + offset3x3x3(Z, coarse_y_3x3x3[0]+1, X, 2, 3), \ + offset3x3x3(Z, coarse_y_3x3x3[0]+2, X, 2, 3), \ + offset3x3x3(Z, coarse_y_3x3x3[0], X, 2, 2) \ + }, \ + { \ + offset3x3x3(Z, coarse_y_3x3x3[1]-2, X, 3, 3), \ + offset3x3x3(Z, coarse_y_3x3x3[1]-1, X, 3, 3), \ + offset3x3x3(Z, coarse_y_3x3x3[1], X, 3, 3), \ + zero_offset, \ + zero_offset, \ + offset3x3x3(Z, coarse_y_3x3x3[1], X, 2, 2) \ + } + + static constexpr int offset[12][6] = { + OFFSET(0, 0), OFFSET(0, 1), + OFFSET(1, 0), OFFSET(1, 1), + OFFSET(2, 0), OFFSET(2, 1) + }; + #undef OFFSET + return offset[i]; +} + +MGARDX_EXEC int const *MassTrans_Z_Offset_3x3x3(SIZE i) { + static constexpr int zero_offset = 0; + #define OFFSET(Y, X) \ + { \ + zero_offset, \ + zero_offset, \ + offset3x3x3(coarse_z_3x3x3[0], Y, X, 2, 2), \ + offset3x3x3(coarse_z_3x3x3[0]+1, Y, X, 2, 2), \ + offset3x3x3(coarse_z_3x3x3[0]+2, Y, X, 2, 2), \ + offset3x3x3(coarse_z_3x3x3[0], Y, X, 2, 2) \ + }, \ + { \ + offset3x3x3(coarse_z_3x3x3[1]-2, Y, X, 2, 2), \ + offset3x3x3(coarse_z_3x3x3[1]-1, Y, X, 2, 2), \ + offset3x3x3(coarse_z_3x3x3[1], Y, X, 2, 2), \ + zero_offset, \ + zero_offset, \ + offset3x3x3(coarse_z_3x3x3[1], Y, X, 2, 2) \ + } + + static constexpr int offset[8][6] = { + OFFSET(0, 0), OFFSET(0, 1), + OFFSET(1, 0), OFFSET(1, 1) + }; + #undef OFFSET + return offset[i]; +} + +MGARDX_EXEC int const *TriDiag_X_Offset_3x3x3(SIZE i) { + static constexpr int offset[4][3] = { + {offset3x3x3(0, 0, 0, 2, 2), offset3x3x3(0, 0, 1, 2, 2)}, + {offset3x3x3(0, 1, 0, 2, 2), offset3x3x3(0, 1, 1, 2, 2)}, + + {offset3x3x3(1, 0, 0, 2, 2), offset3x3x3(1, 0, 1, 2, 2)}, + {offset3x3x3(1, 1, 0, 2, 2), offset3x3x3(1, 1, 1, 2, 2)} + }; + return offset[i]; +} + +MGARDX_EXEC int const *TriDiag_Y_Offset_3x3x3(SIZE i) { + static constexpr int offset[4][3] = { + {offset3x3x3(0, 0, 0, 2, 2), offset3x3x3(0, 1, 0, 2, 2)}, + {offset3x3x3(0, 0, 1, 2, 2), offset3x3x3(0, 1, 1, 2, 2)}, + + {offset3x3x3(1, 0, 0, 2, 2), offset3x3x3(1, 1, 0, 2, 2)}, + {offset3x3x3(1, 0, 1, 2, 2), offset3x3x3(1, 1, 1, 2, 2)} + + }; + return offset[i]; +} + +MGARDX_EXEC int const *TriDiag_Z_Offset_3x3x3(SIZE i) { + static constexpr int offset[4][3] = { + {offset3x3x3(0, 0, 0, 2, 2), offset3x3x3(1, 0, 0, 2, 2)}, + {offset3x3x3(0, 0, 1, 2, 2), offset3x3x3(1, 0, 1, 2, 2)}, + + {offset3x3x3(0, 1, 0, 2, 2), offset3x3x3(1, 1, 0, 2, 2)}, + {offset3x3x3(0, 1, 1, 2, 2), offset3x3x3(1, 1, 1, 2, 2)} + }; + return offset[i]; +} + +MGARDX_EXEC int Coarse_Offset_3x3x3(SIZE i) { + static constexpr int offset[8] = { + offset3x3x3(0, 0, 0, 3, 3), offset3x3x3(0, 0, 2, 3, 3), + offset3x3x3(0, 2, 0, 3, 3), offset3x3x3(0, 2, 2, 3, 3), + + offset3x3x3(2, 0, 0, 3, 3), offset3x3x3(2, 0, 2, 3, 3), + offset3x3x3(2, 2, 0, 3, 3), offset3x3x3(2, 2, 2, 3, 3) + }; + return offset[i]; +} + +// clang-format on +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTable5x5x5.hpp b/include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTable5x5x5.hpp new file mode 100644 index 000000000..8ac55e8b0 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTable5x5x5.hpp @@ -0,0 +1,790 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ +#ifndef MGARD_X_160_INDEX_TABLE_5X5X5_HPP +#define MGARD_X_160_INDEX_TABLE_5X5X5_HPP + +namespace mgard_x_160 { +// clang-format off + +MGARDX_EXEC constexpr int offset5x5x5(SIZE z, SIZE y, SIZE x) { + return z * 5 * 5 + y * 5 + x; +} + +MGARDX_EXEC constexpr int offset5x5x5(SIZE z, SIZE y, SIZE x, SIZE ld1, SIZE ld2) { + return z * ld1 * ld2 + y * ld1 + x; +} + +static constexpr int c1d_x_5x5x5[18][3] = { {0, 0, 1}, {0, 0, 3}, // X + {0, 2, 1}, {0, 2, 3}, + {0, 4, 1}, {0, 4, 3}, + {2, 0, 1}, {2, 0, 3}, + {2, 2, 1}, {2, 2, 3}, + {2, 4, 1}, {2, 4, 3}, + {4, 0, 1}, {4, 0, 3}, + {4, 2, 1}, {4, 2, 3}, + {4, 4, 1}, {4, 4, 3} + }; + +static constexpr int c1d_y_5x5x5[18][3] = { {0, 1, 0}, {0, 3, 0}, // Y + {0, 1, 2}, {0, 3, 2}, + {0, 1, 4}, {0, 3, 4}, + {2, 1, 0}, {2, 3, 0}, + {2, 1, 2}, {2, 3, 2}, + {2, 1, 4}, {2, 3, 4}, + {4, 1, 0}, {4, 3, 0}, + {4, 1, 2}, {4, 3, 2}, + {4, 1, 4}, {4, 3, 4} + }; + +static constexpr int c1d_z_5x5x5[18][3] = { {1, 0, 0}, {3, 0, 0}, // Z + {1, 0, 2}, {3, 0, 2}, + {1, 0, 4}, {3, 0, 4}, + {1, 2, 0}, {3, 2, 0}, + {1, 2, 2}, {3, 2, 2}, + {1, 2, 4}, {3, 2, 4}, + {1, 4, 0}, {3, 4, 0}, + {1, 4, 2}, {3, 4, 2}, + {1, 4, 4}, {3, 4, 4}, + }; +static constexpr int c2d_xy_5x5x5[12][3] = {{0, 1, 1}, {0, 1, 3}, // XY + {0, 3, 1}, {0, 3, 3}, + {2, 1, 1}, {2, 1, 3}, + {2, 3, 1}, {2, 3, 3}, + {4, 1, 1}, {4, 1, 3}, + {4, 3, 1}, {4, 3, 3} + }; + +static constexpr int c2d_yz_5x5x5[12][3] = {{1, 1, 0}, {1, 3, 0}, // YZ + {3, 1, 0}, {3, 3, 0}, + {1, 1, 2}, {1, 3, 2}, + {3, 1, 2}, {3, 3, 2}, + {1, 1, 4}, {1, 3, 4}, + {3, 1, 4}, {3, 3, 4} + }; + +static constexpr int c2d_xz_5x5x5[12][3] = {{1, 0, 1}, {1, 0, 3}, // XZ + {3, 0, 1}, {3, 0, 3}, + {1, 2, 1}, {1, 2, 3}, + {3, 2, 1}, {3, 2, 3}, + {1, 4, 1}, {1, 4, 3}, + {3, 4, 1}, {3, 4, 3} + }; + +static constexpr int c3d_xyz_5x5x5[8][3] = { {1, 1, 1}, {1, 1, 3}, // XYZ + {1, 3, 1}, {1, 3, 3}, + {3, 1, 1}, {3, 1, 3}, + {3, 3, 1}, {3, 3, 3} + }; + +static constexpr int coarse_x_5x5x5[3] = {0, 2, 4}; + +static constexpr int coarse_y_5x5x5[3] = {0, 2, 4}; + +static constexpr int coarse_z_5x5x5[3] = {0, 2, 4}; + +MGARDX_EXEC int Coeff1D_M_Offset_5x5x5(SIZE i) { + static constexpr int offset[54] = { offset5x5x5(c1d_x_5x5x5[0][0], c1d_x_5x5x5[0][1], c1d_x_5x5x5[0][2]), // X + offset5x5x5(c1d_x_5x5x5[1][0], c1d_x_5x5x5[1][1], c1d_x_5x5x5[1][2]), + offset5x5x5(c1d_x_5x5x5[2][0], c1d_x_5x5x5[2][1], c1d_x_5x5x5[2][2]), + offset5x5x5(c1d_x_5x5x5[3][0], c1d_x_5x5x5[3][1], c1d_x_5x5x5[3][2]), + offset5x5x5(c1d_x_5x5x5[4][0], c1d_x_5x5x5[4][1], c1d_x_5x5x5[4][2]), + offset5x5x5(c1d_x_5x5x5[5][0], c1d_x_5x5x5[5][1], c1d_x_5x5x5[5][2]), + offset5x5x5(c1d_x_5x5x5[6][0], c1d_x_5x5x5[6][1], c1d_x_5x5x5[6][2]), + offset5x5x5(c1d_x_5x5x5[7][0], c1d_x_5x5x5[7][1], c1d_x_5x5x5[7][2]), + offset5x5x5(c1d_x_5x5x5[8][0], c1d_x_5x5x5[8][1], c1d_x_5x5x5[8][2]), + offset5x5x5(c1d_x_5x5x5[9][0], c1d_x_5x5x5[9][1], c1d_x_5x5x5[9][2]), + offset5x5x5(c1d_x_5x5x5[10][0], c1d_x_5x5x5[10][1], c1d_x_5x5x5[10][2]), + offset5x5x5(c1d_x_5x5x5[11][0], c1d_x_5x5x5[11][1], c1d_x_5x5x5[11][2]), + offset5x5x5(c1d_x_5x5x5[12][0], c1d_x_5x5x5[12][1], c1d_x_5x5x5[12][2]), + offset5x5x5(c1d_x_5x5x5[13][0], c1d_x_5x5x5[13][1], c1d_x_5x5x5[13][2]), + offset5x5x5(c1d_x_5x5x5[14][0], c1d_x_5x5x5[14][1], c1d_x_5x5x5[14][2]), + offset5x5x5(c1d_x_5x5x5[15][0], c1d_x_5x5x5[15][1], c1d_x_5x5x5[15][2]), + offset5x5x5(c1d_x_5x5x5[16][0], c1d_x_5x5x5[16][1], c1d_x_5x5x5[16][2]), + offset5x5x5(c1d_x_5x5x5[17][0], c1d_x_5x5x5[17][1], c1d_x_5x5x5[17][2]), + + offset5x5x5(c1d_y_5x5x5[0][0], c1d_y_5x5x5[0][1], c1d_y_5x5x5[0][2]), // Y + offset5x5x5(c1d_y_5x5x5[1][0], c1d_y_5x5x5[1][1], c1d_y_5x5x5[1][2]), + offset5x5x5(c1d_y_5x5x5[2][0], c1d_y_5x5x5[2][1], c1d_y_5x5x5[2][2]), + offset5x5x5(c1d_y_5x5x5[3][0], c1d_y_5x5x5[3][1], c1d_y_5x5x5[3][2]), + offset5x5x5(c1d_y_5x5x5[4][0], c1d_y_5x5x5[4][1], c1d_y_5x5x5[4][2]), + offset5x5x5(c1d_y_5x5x5[5][0], c1d_y_5x5x5[5][1], c1d_y_5x5x5[5][2]), + offset5x5x5(c1d_y_5x5x5[6][0], c1d_y_5x5x5[6][1], c1d_y_5x5x5[6][2]), + offset5x5x5(c1d_y_5x5x5[7][0], c1d_y_5x5x5[7][1], c1d_y_5x5x5[7][2]), + offset5x5x5(c1d_y_5x5x5[8][0], c1d_y_5x5x5[8][1], c1d_y_5x5x5[8][2]), + offset5x5x5(c1d_y_5x5x5[9][0], c1d_y_5x5x5[9][1], c1d_y_5x5x5[9][2]), + offset5x5x5(c1d_y_5x5x5[10][0], c1d_y_5x5x5[10][1], c1d_y_5x5x5[10][2]), + offset5x5x5(c1d_y_5x5x5[11][0], c1d_y_5x5x5[11][1], c1d_y_5x5x5[11][2]), + offset5x5x5(c1d_y_5x5x5[12][0], c1d_y_5x5x5[12][1], c1d_y_5x5x5[12][2]), + offset5x5x5(c1d_y_5x5x5[13][0], c1d_y_5x5x5[13][1], c1d_y_5x5x5[13][2]), + offset5x5x5(c1d_y_5x5x5[14][0], c1d_y_5x5x5[14][1], c1d_y_5x5x5[14][2]), + offset5x5x5(c1d_y_5x5x5[15][0], c1d_y_5x5x5[15][1], c1d_y_5x5x5[15][2]), + offset5x5x5(c1d_y_5x5x5[16][0], c1d_y_5x5x5[16][1], c1d_y_5x5x5[16][2]), + offset5x5x5(c1d_y_5x5x5[17][0], c1d_y_5x5x5[17][1], c1d_y_5x5x5[17][2]), + + offset5x5x5(c1d_z_5x5x5[0][0], c1d_z_5x5x5[0][1], c1d_z_5x5x5[0][2]), // Z + offset5x5x5(c1d_z_5x5x5[1][0], c1d_z_5x5x5[1][1], c1d_z_5x5x5[1][2]), + offset5x5x5(c1d_z_5x5x5[2][0], c1d_z_5x5x5[2][1], c1d_z_5x5x5[2][2]), + offset5x5x5(c1d_z_5x5x5[3][0], c1d_z_5x5x5[3][1], c1d_z_5x5x5[3][2]), + offset5x5x5(c1d_z_5x5x5[4][0], c1d_z_5x5x5[4][1], c1d_z_5x5x5[4][2]), + offset5x5x5(c1d_z_5x5x5[5][0], c1d_z_5x5x5[5][1], c1d_z_5x5x5[5][2]), + offset5x5x5(c1d_z_5x5x5[6][0], c1d_z_5x5x5[6][1], c1d_z_5x5x5[6][2]), + offset5x5x5(c1d_z_5x5x5[7][0], c1d_z_5x5x5[7][1], c1d_z_5x5x5[7][2]), + offset5x5x5(c1d_z_5x5x5[8][0], c1d_z_5x5x5[8][1], c1d_z_5x5x5[8][2]), + offset5x5x5(c1d_z_5x5x5[9][0], c1d_z_5x5x5[9][1], c1d_z_5x5x5[9][2]), + offset5x5x5(c1d_z_5x5x5[10][0], c1d_z_5x5x5[10][1], c1d_z_5x5x5[10][2]), + offset5x5x5(c1d_z_5x5x5[11][0], c1d_z_5x5x5[11][1], c1d_z_5x5x5[11][2]), + offset5x5x5(c1d_z_5x5x5[12][0], c1d_z_5x5x5[12][1], c1d_z_5x5x5[12][2]), + offset5x5x5(c1d_z_5x5x5[13][0], c1d_z_5x5x5[13][1], c1d_z_5x5x5[13][2]), + offset5x5x5(c1d_z_5x5x5[14][0], c1d_z_5x5x5[14][1], c1d_z_5x5x5[14][2]), + offset5x5x5(c1d_z_5x5x5[15][0], c1d_z_5x5x5[15][1], c1d_z_5x5x5[15][2]), + offset5x5x5(c1d_z_5x5x5[16][0], c1d_z_5x5x5[16][1], c1d_z_5x5x5[16][2]), + offset5x5x5(c1d_z_5x5x5[17][0], c1d_z_5x5x5[17][1], c1d_z_5x5x5[17][2]) + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff1D_L_Offset_5x5x5(SIZE i) { + static constexpr int offset[54] = { offset5x5x5(c1d_x_5x5x5[0][0], c1d_x_5x5x5[0][1], c1d_x_5x5x5[0][2]-1), // X + offset5x5x5(c1d_x_5x5x5[1][0], c1d_x_5x5x5[1][1], c1d_x_5x5x5[1][2]-1), + offset5x5x5(c1d_x_5x5x5[2][0], c1d_x_5x5x5[2][1], c1d_x_5x5x5[2][2]-1), + offset5x5x5(c1d_x_5x5x5[3][0], c1d_x_5x5x5[3][1], c1d_x_5x5x5[3][2]-1), + offset5x5x5(c1d_x_5x5x5[4][0], c1d_x_5x5x5[4][1], c1d_x_5x5x5[4][2]-1), + offset5x5x5(c1d_x_5x5x5[5][0], c1d_x_5x5x5[5][1], c1d_x_5x5x5[5][2]-1), + offset5x5x5(c1d_x_5x5x5[6][0], c1d_x_5x5x5[6][1], c1d_x_5x5x5[6][2]-1), + offset5x5x5(c1d_x_5x5x5[7][0], c1d_x_5x5x5[7][1], c1d_x_5x5x5[7][2]-1), + offset5x5x5(c1d_x_5x5x5[8][0], c1d_x_5x5x5[8][1], c1d_x_5x5x5[8][2]-1), + offset5x5x5(c1d_x_5x5x5[9][0], c1d_x_5x5x5[9][1], c1d_x_5x5x5[9][2]-1), + offset5x5x5(c1d_x_5x5x5[10][0], c1d_x_5x5x5[10][1], c1d_x_5x5x5[10][2]-1), + offset5x5x5(c1d_x_5x5x5[11][0], c1d_x_5x5x5[11][1], c1d_x_5x5x5[11][2]-1), + offset5x5x5(c1d_x_5x5x5[12][0], c1d_x_5x5x5[12][1], c1d_x_5x5x5[12][2]-1), + offset5x5x5(c1d_x_5x5x5[13][0], c1d_x_5x5x5[13][1], c1d_x_5x5x5[13][2]-1), + offset5x5x5(c1d_x_5x5x5[14][0], c1d_x_5x5x5[14][1], c1d_x_5x5x5[14][2]-1), + offset5x5x5(c1d_x_5x5x5[15][0], c1d_x_5x5x5[15][1], c1d_x_5x5x5[15][2]-1), + offset5x5x5(c1d_x_5x5x5[16][0], c1d_x_5x5x5[16][1], c1d_x_5x5x5[16][2]-1), + offset5x5x5(c1d_x_5x5x5[17][0], c1d_x_5x5x5[17][1], c1d_x_5x5x5[17][2]-1), + + offset5x5x5(c1d_y_5x5x5[0][0], c1d_y_5x5x5[0][1]-1, c1d_y_5x5x5[0][2]), // Y + offset5x5x5(c1d_y_5x5x5[1][0], c1d_y_5x5x5[1][1]-1, c1d_y_5x5x5[1][2]), + offset5x5x5(c1d_y_5x5x5[2][0], c1d_y_5x5x5[2][1]-1, c1d_y_5x5x5[2][2]), + offset5x5x5(c1d_y_5x5x5[3][0], c1d_y_5x5x5[3][1]-1, c1d_y_5x5x5[3][2]), + offset5x5x5(c1d_y_5x5x5[4][0], c1d_y_5x5x5[4][1]-1, c1d_y_5x5x5[4][2]), + offset5x5x5(c1d_y_5x5x5[5][0], c1d_y_5x5x5[5][1]-1, c1d_y_5x5x5[5][2]), + offset5x5x5(c1d_y_5x5x5[6][0], c1d_y_5x5x5[6][1]-1, c1d_y_5x5x5[6][2]), + offset5x5x5(c1d_y_5x5x5[7][0], c1d_y_5x5x5[7][1]-1, c1d_y_5x5x5[7][2]), + offset5x5x5(c1d_y_5x5x5[8][0], c1d_y_5x5x5[8][1]-1, c1d_y_5x5x5[8][2]), + offset5x5x5(c1d_y_5x5x5[9][0], c1d_y_5x5x5[9][1]-1, c1d_y_5x5x5[9][2]), + offset5x5x5(c1d_y_5x5x5[10][0], c1d_y_5x5x5[10][1]-1, c1d_y_5x5x5[10][2]), + offset5x5x5(c1d_y_5x5x5[11][0], c1d_y_5x5x5[11][1]-1, c1d_y_5x5x5[11][2]), + offset5x5x5(c1d_y_5x5x5[12][0], c1d_y_5x5x5[12][1]-1, c1d_y_5x5x5[12][2]), + offset5x5x5(c1d_y_5x5x5[13][0], c1d_y_5x5x5[13][1]-1, c1d_y_5x5x5[13][2]), + offset5x5x5(c1d_y_5x5x5[14][0], c1d_y_5x5x5[14][1]-1, c1d_y_5x5x5[14][2]), + offset5x5x5(c1d_y_5x5x5[15][0], c1d_y_5x5x5[15][1]-1, c1d_y_5x5x5[15][2]), + offset5x5x5(c1d_y_5x5x5[16][0], c1d_y_5x5x5[16][1]-1, c1d_y_5x5x5[16][2]), + offset5x5x5(c1d_y_5x5x5[17][0], c1d_y_5x5x5[17][1]-1, c1d_y_5x5x5[17][2]), + + offset5x5x5(c1d_z_5x5x5[0][0]-1, c1d_z_5x5x5[0][1], c1d_z_5x5x5[0][2]), // Z + offset5x5x5(c1d_z_5x5x5[1][0]-1, c1d_z_5x5x5[1][1], c1d_z_5x5x5[1][2]), + offset5x5x5(c1d_z_5x5x5[2][0]-1, c1d_z_5x5x5[2][1], c1d_z_5x5x5[2][2]), + offset5x5x5(c1d_z_5x5x5[3][0]-1, c1d_z_5x5x5[3][1], c1d_z_5x5x5[3][2]), + offset5x5x5(c1d_z_5x5x5[4][0]-1, c1d_z_5x5x5[4][1], c1d_z_5x5x5[4][2]), + offset5x5x5(c1d_z_5x5x5[5][0]-1, c1d_z_5x5x5[5][1], c1d_z_5x5x5[5][2]), + offset5x5x5(c1d_z_5x5x5[6][0]-1, c1d_z_5x5x5[6][1], c1d_z_5x5x5[6][2]), + offset5x5x5(c1d_z_5x5x5[7][0]-1, c1d_z_5x5x5[7][1], c1d_z_5x5x5[7][2]), + offset5x5x5(c1d_z_5x5x5[8][0]-1, c1d_z_5x5x5[8][1], c1d_z_5x5x5[8][2]), + offset5x5x5(c1d_z_5x5x5[9][0]-1, c1d_z_5x5x5[9][1], c1d_z_5x5x5[9][2]), + offset5x5x5(c1d_z_5x5x5[10][0]-1, c1d_z_5x5x5[10][1], c1d_z_5x5x5[10][2]), + offset5x5x5(c1d_z_5x5x5[11][0]-1, c1d_z_5x5x5[11][1], c1d_z_5x5x5[11][2]), + offset5x5x5(c1d_z_5x5x5[12][0]-1, c1d_z_5x5x5[12][1], c1d_z_5x5x5[12][2]), + offset5x5x5(c1d_z_5x5x5[13][0]-1, c1d_z_5x5x5[13][1], c1d_z_5x5x5[13][2]), + offset5x5x5(c1d_z_5x5x5[14][0]-1, c1d_z_5x5x5[14][1], c1d_z_5x5x5[14][2]), + offset5x5x5(c1d_z_5x5x5[15][0]-1, c1d_z_5x5x5[15][1], c1d_z_5x5x5[15][2]), + offset5x5x5(c1d_z_5x5x5[16][0]-1, c1d_z_5x5x5[16][1], c1d_z_5x5x5[16][2]), + offset5x5x5(c1d_z_5x5x5[17][0]-1, c1d_z_5x5x5[17][1], c1d_z_5x5x5[17][2]) + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff1D_R_Offset_5x5x5(SIZE i) { + static constexpr int offset[54] = { offset5x5x5(c1d_x_5x5x5[0][0], c1d_x_5x5x5[0][1], c1d_x_5x5x5[0][2]+1), // X + offset5x5x5(c1d_x_5x5x5[1][0], c1d_x_5x5x5[1][1], c1d_x_5x5x5[1][2]+1), + offset5x5x5(c1d_x_5x5x5[2][0], c1d_x_5x5x5[2][1], c1d_x_5x5x5[2][2]+1), + offset5x5x5(c1d_x_5x5x5[3][0], c1d_x_5x5x5[3][1], c1d_x_5x5x5[3][2]+1), + offset5x5x5(c1d_x_5x5x5[4][0], c1d_x_5x5x5[4][1], c1d_x_5x5x5[4][2]+1), + offset5x5x5(c1d_x_5x5x5[5][0], c1d_x_5x5x5[5][1], c1d_x_5x5x5[5][2]+1), + offset5x5x5(c1d_x_5x5x5[6][0], c1d_x_5x5x5[6][1], c1d_x_5x5x5[6][2]+1), + offset5x5x5(c1d_x_5x5x5[7][0], c1d_x_5x5x5[7][1], c1d_x_5x5x5[7][2]+1), + offset5x5x5(c1d_x_5x5x5[8][0], c1d_x_5x5x5[8][1], c1d_x_5x5x5[8][2]+1), + offset5x5x5(c1d_x_5x5x5[9][0], c1d_x_5x5x5[9][1], c1d_x_5x5x5[9][2]+1), + offset5x5x5(c1d_x_5x5x5[10][0], c1d_x_5x5x5[10][1], c1d_x_5x5x5[10][2]+1), + offset5x5x5(c1d_x_5x5x5[11][0], c1d_x_5x5x5[11][1], c1d_x_5x5x5[11][2]+1), + offset5x5x5(c1d_x_5x5x5[12][0], c1d_x_5x5x5[12][1], c1d_x_5x5x5[12][2]+1), + offset5x5x5(c1d_x_5x5x5[13][0], c1d_x_5x5x5[13][1], c1d_x_5x5x5[13][2]+1), + offset5x5x5(c1d_x_5x5x5[14][0], c1d_x_5x5x5[14][1], c1d_x_5x5x5[14][2]+1), + offset5x5x5(c1d_x_5x5x5[15][0], c1d_x_5x5x5[15][1], c1d_x_5x5x5[15][2]+1), + offset5x5x5(c1d_x_5x5x5[16][0], c1d_x_5x5x5[16][1], c1d_x_5x5x5[16][2]+1), + offset5x5x5(c1d_x_5x5x5[17][0], c1d_x_5x5x5[17][1], c1d_x_5x5x5[17][2]+1), + + offset5x5x5(c1d_y_5x5x5[0][0], c1d_y_5x5x5[0][1]+1, c1d_y_5x5x5[0][2]), // Y + offset5x5x5(c1d_y_5x5x5[1][0], c1d_y_5x5x5[1][1]+1, c1d_y_5x5x5[1][2]), + offset5x5x5(c1d_y_5x5x5[2][0], c1d_y_5x5x5[2][1]+1, c1d_y_5x5x5[2][2]), + offset5x5x5(c1d_y_5x5x5[3][0], c1d_y_5x5x5[3][1]+1, c1d_y_5x5x5[3][2]), + offset5x5x5(c1d_y_5x5x5[4][0], c1d_y_5x5x5[4][1]+1, c1d_y_5x5x5[4][2]), + offset5x5x5(c1d_y_5x5x5[5][0], c1d_y_5x5x5[5][1]+1, c1d_y_5x5x5[5][2]), + offset5x5x5(c1d_y_5x5x5[6][0], c1d_y_5x5x5[6][1]+1, c1d_y_5x5x5[6][2]), + offset5x5x5(c1d_y_5x5x5[7][0], c1d_y_5x5x5[7][1]+1, c1d_y_5x5x5[7][2]), + offset5x5x5(c1d_y_5x5x5[8][0], c1d_y_5x5x5[8][1]+1, c1d_y_5x5x5[8][2]), + offset5x5x5(c1d_y_5x5x5[9][0], c1d_y_5x5x5[9][1]+1, c1d_y_5x5x5[9][2]), + offset5x5x5(c1d_y_5x5x5[10][0], c1d_y_5x5x5[10][1]+1, c1d_y_5x5x5[10][2]), + offset5x5x5(c1d_y_5x5x5[11][0], c1d_y_5x5x5[11][1]+1, c1d_y_5x5x5[11][2]), + offset5x5x5(c1d_y_5x5x5[12][0], c1d_y_5x5x5[12][1]+1, c1d_y_5x5x5[12][2]), + offset5x5x5(c1d_y_5x5x5[13][0], c1d_y_5x5x5[13][1]+1, c1d_y_5x5x5[13][2]), + offset5x5x5(c1d_y_5x5x5[14][0], c1d_y_5x5x5[14][1]+1, c1d_y_5x5x5[14][2]), + offset5x5x5(c1d_y_5x5x5[15][0], c1d_y_5x5x5[15][1]+1, c1d_y_5x5x5[15][2]), + offset5x5x5(c1d_y_5x5x5[16][0], c1d_y_5x5x5[16][1]+1, c1d_y_5x5x5[16][2]), + offset5x5x5(c1d_y_5x5x5[17][0], c1d_y_5x5x5[17][1]+1, c1d_y_5x5x5[17][2]), + + offset5x5x5(c1d_z_5x5x5[0][0]+1, c1d_z_5x5x5[0][1], c1d_z_5x5x5[0][2]), // Z + offset5x5x5(c1d_z_5x5x5[1][0]+1, c1d_z_5x5x5[1][1], c1d_z_5x5x5[1][2]), + offset5x5x5(c1d_z_5x5x5[2][0]+1, c1d_z_5x5x5[2][1], c1d_z_5x5x5[2][2]), + offset5x5x5(c1d_z_5x5x5[3][0]+1, c1d_z_5x5x5[3][1], c1d_z_5x5x5[3][2]), + offset5x5x5(c1d_z_5x5x5[4][0]+1, c1d_z_5x5x5[4][1], c1d_z_5x5x5[4][2]), + offset5x5x5(c1d_z_5x5x5[5][0]+1, c1d_z_5x5x5[5][1], c1d_z_5x5x5[5][2]), + offset5x5x5(c1d_z_5x5x5[6][0]+1, c1d_z_5x5x5[6][1], c1d_z_5x5x5[6][2]), + offset5x5x5(c1d_z_5x5x5[7][0]+1, c1d_z_5x5x5[7][1], c1d_z_5x5x5[7][2]), + offset5x5x5(c1d_z_5x5x5[8][0]+1, c1d_z_5x5x5[8][1], c1d_z_5x5x5[8][2]), + offset5x5x5(c1d_z_5x5x5[9][0]+1, c1d_z_5x5x5[9][1], c1d_z_5x5x5[9][2]), + offset5x5x5(c1d_z_5x5x5[10][0]+1, c1d_z_5x5x5[10][1], c1d_z_5x5x5[10][2]), + offset5x5x5(c1d_z_5x5x5[11][0]+1, c1d_z_5x5x5[11][1], c1d_z_5x5x5[11][2]), + offset5x5x5(c1d_z_5x5x5[12][0]+1, c1d_z_5x5x5[12][1], c1d_z_5x5x5[12][2]), + offset5x5x5(c1d_z_5x5x5[13][0]+1, c1d_z_5x5x5[13][1], c1d_z_5x5x5[13][2]), + offset5x5x5(c1d_z_5x5x5[14][0]+1, c1d_z_5x5x5[14][1], c1d_z_5x5x5[14][2]), + offset5x5x5(c1d_z_5x5x5[15][0]+1, c1d_z_5x5x5[15][1], c1d_z_5x5x5[15][2]), + offset5x5x5(c1d_z_5x5x5[16][0]+1, c1d_z_5x5x5[16][1], c1d_z_5x5x5[16][2]), + offset5x5x5(c1d_z_5x5x5[17][0]+1, c1d_z_5x5x5[17][1], c1d_z_5x5x5[17][2]) + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff2D_MM_Offset_5x5x5(SIZE i) { + static constexpr int offset[36] = { offset5x5x5(c2d_xy_5x5x5[0][0], c2d_xy_5x5x5[0][1], c2d_xy_5x5x5[0][2]), // XY + offset5x5x5(c2d_xy_5x5x5[1][0], c2d_xy_5x5x5[1][1], c2d_xy_5x5x5[1][2]), + offset5x5x5(c2d_xy_5x5x5[2][0], c2d_xy_5x5x5[2][1], c2d_xy_5x5x5[2][2]), + offset5x5x5(c2d_xy_5x5x5[3][0], c2d_xy_5x5x5[3][1], c2d_xy_5x5x5[3][2]), + offset5x5x5(c2d_xy_5x5x5[4][0], c2d_xy_5x5x5[4][1], c2d_xy_5x5x5[4][2]), + offset5x5x5(c2d_xy_5x5x5[5][0], c2d_xy_5x5x5[5][1], c2d_xy_5x5x5[5][2]), + offset5x5x5(c2d_xy_5x5x5[6][0], c2d_xy_5x5x5[6][1], c2d_xy_5x5x5[6][2]), + offset5x5x5(c2d_xy_5x5x5[7][0], c2d_xy_5x5x5[7][1], c2d_xy_5x5x5[7][2]), + offset5x5x5(c2d_xy_5x5x5[8][0], c2d_xy_5x5x5[8][1], c2d_xy_5x5x5[8][2]), + offset5x5x5(c2d_xy_5x5x5[9][0], c2d_xy_5x5x5[9][1], c2d_xy_5x5x5[9][2]), + offset5x5x5(c2d_xy_5x5x5[10][0], c2d_xy_5x5x5[10][1], c2d_xy_5x5x5[10][2]), + offset5x5x5(c2d_xy_5x5x5[11][0], c2d_xy_5x5x5[11][1], c2d_xy_5x5x5[11][2]), + + offset5x5x5(c2d_xz_5x5x5[0][0], c2d_xz_5x5x5[0][1], c2d_xz_5x5x5[0][2]), // XZ + offset5x5x5(c2d_xz_5x5x5[1][0], c2d_xz_5x5x5[1][1], c2d_xz_5x5x5[1][2]), + offset5x5x5(c2d_xz_5x5x5[2][0], c2d_xz_5x5x5[2][1], c2d_xz_5x5x5[2][2]), + offset5x5x5(c2d_xz_5x5x5[3][0], c2d_xz_5x5x5[3][1], c2d_xz_5x5x5[3][2]), + offset5x5x5(c2d_xz_5x5x5[4][0], c2d_xz_5x5x5[4][1], c2d_xz_5x5x5[4][2]), + offset5x5x5(c2d_xz_5x5x5[5][0], c2d_xz_5x5x5[5][1], c2d_xz_5x5x5[5][2]), + offset5x5x5(c2d_xz_5x5x5[6][0], c2d_xz_5x5x5[6][1], c2d_xz_5x5x5[6][2]), + offset5x5x5(c2d_xz_5x5x5[7][0], c2d_xz_5x5x5[7][1], c2d_xz_5x5x5[7][2]), + offset5x5x5(c2d_xz_5x5x5[8][0], c2d_xz_5x5x5[8][1], c2d_xz_5x5x5[8][2]), + offset5x5x5(c2d_xz_5x5x5[9][0], c2d_xz_5x5x5[9][1], c2d_xz_5x5x5[9][2]), + offset5x5x5(c2d_xz_5x5x5[10][0], c2d_xz_5x5x5[10][1], c2d_xz_5x5x5[10][2]), + offset5x5x5(c2d_xz_5x5x5[11][0], c2d_xz_5x5x5[11][1], c2d_xz_5x5x5[11][2]), + + offset5x5x5(c2d_yz_5x5x5[0][0], c2d_yz_5x5x5[0][1], c2d_yz_5x5x5[0][2]), // YZ + offset5x5x5(c2d_yz_5x5x5[1][0], c2d_yz_5x5x5[1][1], c2d_yz_5x5x5[1][2]), + offset5x5x5(c2d_yz_5x5x5[2][0], c2d_yz_5x5x5[2][1], c2d_yz_5x5x5[2][2]), + offset5x5x5(c2d_yz_5x5x5[3][0], c2d_yz_5x5x5[3][1], c2d_yz_5x5x5[3][2]), + offset5x5x5(c2d_yz_5x5x5[4][0], c2d_yz_5x5x5[4][1], c2d_yz_5x5x5[4][2]), + offset5x5x5(c2d_yz_5x5x5[5][0], c2d_yz_5x5x5[5][1], c2d_yz_5x5x5[5][2]), + offset5x5x5(c2d_yz_5x5x5[6][0], c2d_yz_5x5x5[6][1], c2d_yz_5x5x5[6][2]), + offset5x5x5(c2d_yz_5x5x5[7][0], c2d_yz_5x5x5[7][1], c2d_yz_5x5x5[7][2]), + offset5x5x5(c2d_yz_5x5x5[8][0], c2d_yz_5x5x5[8][1], c2d_yz_5x5x5[8][2]), + offset5x5x5(c2d_yz_5x5x5[9][0], c2d_yz_5x5x5[9][1], c2d_yz_5x5x5[9][2]), + offset5x5x5(c2d_yz_5x5x5[10][0], c2d_yz_5x5x5[10][1], c2d_yz_5x5x5[10][2]), + offset5x5x5(c2d_yz_5x5x5[11][0], c2d_yz_5x5x5[11][1], c2d_yz_5x5x5[11][2]) + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff2D_LL_Offset_5x5x5(SIZE i) { + static constexpr int offset[36] = { offset5x5x5(c2d_xy_5x5x5[0][0], c2d_xy_5x5x5[0][1]-1, c2d_xy_5x5x5[0][2]-1), // XY + offset5x5x5(c2d_xy_5x5x5[1][0], c2d_xy_5x5x5[1][1]-1, c2d_xy_5x5x5[1][2]-1), + offset5x5x5(c2d_xy_5x5x5[2][0], c2d_xy_5x5x5[2][1]-1, c2d_xy_5x5x5[2][2]-1), + offset5x5x5(c2d_xy_5x5x5[3][0], c2d_xy_5x5x5[3][1]-1, c2d_xy_5x5x5[3][2]-1), + offset5x5x5(c2d_xy_5x5x5[4][0], c2d_xy_5x5x5[4][1]-1, c2d_xy_5x5x5[4][2]-1), + offset5x5x5(c2d_xy_5x5x5[5][0], c2d_xy_5x5x5[5][1]-1, c2d_xy_5x5x5[5][2]-1), + offset5x5x5(c2d_xy_5x5x5[6][0], c2d_xy_5x5x5[6][1]-1, c2d_xy_5x5x5[6][2]-1), + offset5x5x5(c2d_xy_5x5x5[7][0], c2d_xy_5x5x5[7][1]-1, c2d_xy_5x5x5[7][2]-1), + offset5x5x5(c2d_xy_5x5x5[8][0], c2d_xy_5x5x5[8][1]-1, c2d_xy_5x5x5[8][2]-1), + offset5x5x5(c2d_xy_5x5x5[9][0], c2d_xy_5x5x5[9][1]-1, c2d_xy_5x5x5[9][2]-1), + offset5x5x5(c2d_xy_5x5x5[10][0], c2d_xy_5x5x5[10][1]-1, c2d_xy_5x5x5[10][2]-1), + offset5x5x5(c2d_xy_5x5x5[11][0], c2d_xy_5x5x5[11][1]-1, c2d_xy_5x5x5[11][2]-1), + + offset5x5x5(c2d_xz_5x5x5[0][0]-1, c2d_xz_5x5x5[0][1], c2d_xz_5x5x5[0][2]-1), // XZ + offset5x5x5(c2d_xz_5x5x5[1][0]-1, c2d_xz_5x5x5[1][1], c2d_xz_5x5x5[1][2]-1), + offset5x5x5(c2d_xz_5x5x5[2][0]-1, c2d_xz_5x5x5[2][1], c2d_xz_5x5x5[2][2]-1), + offset5x5x5(c2d_xz_5x5x5[3][0]-1, c2d_xz_5x5x5[3][1], c2d_xz_5x5x5[3][2]-1), + offset5x5x5(c2d_xz_5x5x5[4][0]-1, c2d_xz_5x5x5[4][1], c2d_xz_5x5x5[4][2]-1), + offset5x5x5(c2d_xz_5x5x5[5][0]-1, c2d_xz_5x5x5[5][1], c2d_xz_5x5x5[5][2]-1), + offset5x5x5(c2d_xz_5x5x5[6][0]-1, c2d_xz_5x5x5[6][1], c2d_xz_5x5x5[6][2]-1), + offset5x5x5(c2d_xz_5x5x5[7][0]-1, c2d_xz_5x5x5[7][1], c2d_xz_5x5x5[7][2]-1), + offset5x5x5(c2d_xz_5x5x5[8][0]-1, c2d_xz_5x5x5[8][1], c2d_xz_5x5x5[8][2]-1), + offset5x5x5(c2d_xz_5x5x5[9][0]-1, c2d_xz_5x5x5[9][1], c2d_xz_5x5x5[9][2]-1), + offset5x5x5(c2d_xz_5x5x5[10][0]-1, c2d_xz_5x5x5[10][1], c2d_xz_5x5x5[10][2]-1), + offset5x5x5(c2d_xz_5x5x5[11][0]-1, c2d_xz_5x5x5[11][1], c2d_xz_5x5x5[11][2]-1), + + offset5x5x5(c2d_yz_5x5x5[0][0]-1, c2d_yz_5x5x5[0][1]-1, c2d_yz_5x5x5[0][2]), // YZ + offset5x5x5(c2d_yz_5x5x5[1][0]-1, c2d_yz_5x5x5[1][1]-1, c2d_yz_5x5x5[1][2]), + offset5x5x5(c2d_yz_5x5x5[2][0]-1, c2d_yz_5x5x5[2][1]-1, c2d_yz_5x5x5[2][2]), + offset5x5x5(c2d_yz_5x5x5[3][0]-1, c2d_yz_5x5x5[3][1]-1, c2d_yz_5x5x5[3][2]), + offset5x5x5(c2d_yz_5x5x5[4][0]-1, c2d_yz_5x5x5[4][1]-1, c2d_yz_5x5x5[4][2]), + offset5x5x5(c2d_yz_5x5x5[5][0]-1, c2d_yz_5x5x5[5][1]-1, c2d_yz_5x5x5[5][2]), + offset5x5x5(c2d_yz_5x5x5[6][0]-1, c2d_yz_5x5x5[6][1]-1, c2d_yz_5x5x5[6][2]), + offset5x5x5(c2d_yz_5x5x5[7][0]-1, c2d_yz_5x5x5[7][1]-1, c2d_yz_5x5x5[7][2]), + offset5x5x5(c2d_yz_5x5x5[8][0]-1, c2d_yz_5x5x5[8][1]-1, c2d_yz_5x5x5[8][2]), + offset5x5x5(c2d_yz_5x5x5[9][0]-1, c2d_yz_5x5x5[9][1]-1, c2d_yz_5x5x5[9][2]), + offset5x5x5(c2d_yz_5x5x5[10][0]-1, c2d_yz_5x5x5[10][1]-1, c2d_yz_5x5x5[10][2]), + offset5x5x5(c2d_yz_5x5x5[11][0]-1, c2d_yz_5x5x5[11][1]-1, c2d_yz_5x5x5[11][2]) + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff2D_LR_Offset_5x5x5(SIZE i) { + static constexpr int offset[36] = { offset5x5x5(c2d_xy_5x5x5[0][0], c2d_xy_5x5x5[0][1]-1, c2d_xy_5x5x5[0][2]+1), // XY + offset5x5x5(c2d_xy_5x5x5[1][0], c2d_xy_5x5x5[1][1]-1, c2d_xy_5x5x5[1][2]+1), + offset5x5x5(c2d_xy_5x5x5[2][0], c2d_xy_5x5x5[2][1]-1, c2d_xy_5x5x5[2][2]+1), + offset5x5x5(c2d_xy_5x5x5[3][0], c2d_xy_5x5x5[3][1]-1, c2d_xy_5x5x5[3][2]+1), + offset5x5x5(c2d_xy_5x5x5[4][0], c2d_xy_5x5x5[4][1]-1, c2d_xy_5x5x5[4][2]+1), + offset5x5x5(c2d_xy_5x5x5[5][0], c2d_xy_5x5x5[5][1]-1, c2d_xy_5x5x5[5][2]+1), + offset5x5x5(c2d_xy_5x5x5[6][0], c2d_xy_5x5x5[6][1]-1, c2d_xy_5x5x5[6][2]+1), + offset5x5x5(c2d_xy_5x5x5[7][0], c2d_xy_5x5x5[7][1]-1, c2d_xy_5x5x5[7][2]+1), + offset5x5x5(c2d_xy_5x5x5[8][0], c2d_xy_5x5x5[8][1]-1, c2d_xy_5x5x5[8][2]+1), + offset5x5x5(c2d_xy_5x5x5[9][0], c2d_xy_5x5x5[9][1]-1, c2d_xy_5x5x5[9][2]+1), + offset5x5x5(c2d_xy_5x5x5[10][0], c2d_xy_5x5x5[10][1]-1, c2d_xy_5x5x5[10][2]+1), + offset5x5x5(c2d_xy_5x5x5[11][0], c2d_xy_5x5x5[11][1]-1, c2d_xy_5x5x5[11][2]+1), + + offset5x5x5(c2d_xz_5x5x5[0][0]-1, c2d_xz_5x5x5[0][1], c2d_xz_5x5x5[0][2]+1), // XZ + offset5x5x5(c2d_xz_5x5x5[1][0]-1, c2d_xz_5x5x5[1][1], c2d_xz_5x5x5[1][2]+1), + offset5x5x5(c2d_xz_5x5x5[2][0]-1, c2d_xz_5x5x5[2][1], c2d_xz_5x5x5[2][2]+1), + offset5x5x5(c2d_xz_5x5x5[3][0]-1, c2d_xz_5x5x5[3][1], c2d_xz_5x5x5[3][2]+1), + offset5x5x5(c2d_xz_5x5x5[4][0]-1, c2d_xz_5x5x5[4][1], c2d_xz_5x5x5[4][2]+1), + offset5x5x5(c2d_xz_5x5x5[5][0]-1, c2d_xz_5x5x5[5][1], c2d_xz_5x5x5[5][2]+1), + offset5x5x5(c2d_xz_5x5x5[6][0]-1, c2d_xz_5x5x5[6][1], c2d_xz_5x5x5[6][2]+1), + offset5x5x5(c2d_xz_5x5x5[7][0]-1, c2d_xz_5x5x5[7][1], c2d_xz_5x5x5[7][2]+1), + offset5x5x5(c2d_xz_5x5x5[8][0]-1, c2d_xz_5x5x5[8][1], c2d_xz_5x5x5[8][2]+1), + offset5x5x5(c2d_xz_5x5x5[9][0]-1, c2d_xz_5x5x5[9][1], c2d_xz_5x5x5[9][2]+1), + offset5x5x5(c2d_xz_5x5x5[10][0]-1, c2d_xz_5x5x5[10][1], c2d_xz_5x5x5[10][2]+1), + offset5x5x5(c2d_xz_5x5x5[11][0]-1, c2d_xz_5x5x5[11][1], c2d_xz_5x5x5[11][2]+1), + + offset5x5x5(c2d_yz_5x5x5[0][0]-1, c2d_yz_5x5x5[0][1]+1, c2d_yz_5x5x5[0][2]), // YZ + offset5x5x5(c2d_yz_5x5x5[1][0]-1, c2d_yz_5x5x5[1][1]+1, c2d_yz_5x5x5[1][2]), + offset5x5x5(c2d_yz_5x5x5[2][0]-1, c2d_yz_5x5x5[2][1]+1, c2d_yz_5x5x5[2][2]), + offset5x5x5(c2d_yz_5x5x5[3][0]-1, c2d_yz_5x5x5[3][1]+1, c2d_yz_5x5x5[3][2]), + offset5x5x5(c2d_yz_5x5x5[4][0]-1, c2d_yz_5x5x5[4][1]+1, c2d_yz_5x5x5[4][2]), + offset5x5x5(c2d_yz_5x5x5[5][0]-1, c2d_yz_5x5x5[5][1]+1, c2d_yz_5x5x5[5][2]), + offset5x5x5(c2d_yz_5x5x5[6][0]-1, c2d_yz_5x5x5[6][1]+1, c2d_yz_5x5x5[6][2]), + offset5x5x5(c2d_yz_5x5x5[7][0]-1, c2d_yz_5x5x5[7][1]+1, c2d_yz_5x5x5[7][2]), + offset5x5x5(c2d_yz_5x5x5[8][0]-1, c2d_yz_5x5x5[8][1]+1, c2d_yz_5x5x5[8][2]), + offset5x5x5(c2d_yz_5x5x5[9][0]-1, c2d_yz_5x5x5[9][1]+1, c2d_yz_5x5x5[9][2]), + offset5x5x5(c2d_yz_5x5x5[10][0]-1, c2d_yz_5x5x5[10][1]+1, c2d_yz_5x5x5[10][2]), + offset5x5x5(c2d_yz_5x5x5[11][0]-1, c2d_yz_5x5x5[11][1]+1, c2d_yz_5x5x5[11][2]) + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff2D_RL_Offset_5x5x5(SIZE i) { + static constexpr int offset[36] = { offset5x5x5(c2d_xy_5x5x5[0][0], c2d_xy_5x5x5[0][1]+1, c2d_xy_5x5x5[0][2]-1), // XY + offset5x5x5(c2d_xy_5x5x5[1][0], c2d_xy_5x5x5[1][1]+1, c2d_xy_5x5x5[1][2]-1), + offset5x5x5(c2d_xy_5x5x5[2][0], c2d_xy_5x5x5[2][1]+1, c2d_xy_5x5x5[2][2]-1), + offset5x5x5(c2d_xy_5x5x5[3][0], c2d_xy_5x5x5[3][1]+1, c2d_xy_5x5x5[3][2]-1), + offset5x5x5(c2d_xy_5x5x5[4][0], c2d_xy_5x5x5[4][1]+1, c2d_xy_5x5x5[4][2]-1), + offset5x5x5(c2d_xy_5x5x5[5][0], c2d_xy_5x5x5[5][1]+1, c2d_xy_5x5x5[5][2]-1), + offset5x5x5(c2d_xy_5x5x5[6][0], c2d_xy_5x5x5[6][1]+1, c2d_xy_5x5x5[6][2]-1), + offset5x5x5(c2d_xy_5x5x5[7][0], c2d_xy_5x5x5[7][1]+1, c2d_xy_5x5x5[7][2]-1), + offset5x5x5(c2d_xy_5x5x5[8][0], c2d_xy_5x5x5[8][1]+1, c2d_xy_5x5x5[8][2]-1), + offset5x5x5(c2d_xy_5x5x5[9][0], c2d_xy_5x5x5[9][1]+1, c2d_xy_5x5x5[9][2]-1), + offset5x5x5(c2d_xy_5x5x5[10][0], c2d_xy_5x5x5[10][1]+1, c2d_xy_5x5x5[10][2]-1), + offset5x5x5(c2d_xy_5x5x5[11][0], c2d_xy_5x5x5[11][1]+1, c2d_xy_5x5x5[11][2]-1), + + offset5x5x5(c2d_xz_5x5x5[0][0]+1, c2d_xz_5x5x5[0][1], c2d_xz_5x5x5[0][2]-1), // XZ + offset5x5x5(c2d_xz_5x5x5[1][0]+1, c2d_xz_5x5x5[1][1], c2d_xz_5x5x5[1][2]-1), + offset5x5x5(c2d_xz_5x5x5[2][0]+1, c2d_xz_5x5x5[2][1], c2d_xz_5x5x5[2][2]-1), + offset5x5x5(c2d_xz_5x5x5[3][0]+1, c2d_xz_5x5x5[3][1], c2d_xz_5x5x5[3][2]-1), + offset5x5x5(c2d_xz_5x5x5[4][0]+1, c2d_xz_5x5x5[4][1], c2d_xz_5x5x5[4][2]-1), + offset5x5x5(c2d_xz_5x5x5[5][0]+1, c2d_xz_5x5x5[5][1], c2d_xz_5x5x5[5][2]-1), + offset5x5x5(c2d_xz_5x5x5[6][0]+1, c2d_xz_5x5x5[6][1], c2d_xz_5x5x5[6][2]-1), + offset5x5x5(c2d_xz_5x5x5[7][0]+1, c2d_xz_5x5x5[7][1], c2d_xz_5x5x5[7][2]-1), + offset5x5x5(c2d_xz_5x5x5[8][0]+1, c2d_xz_5x5x5[8][1], c2d_xz_5x5x5[8][2]-1), + offset5x5x5(c2d_xz_5x5x5[9][0]+1, c2d_xz_5x5x5[9][1], c2d_xz_5x5x5[9][2]-1), + offset5x5x5(c2d_xz_5x5x5[10][0]+1, c2d_xz_5x5x5[10][1], c2d_xz_5x5x5[10][2]-1), + offset5x5x5(c2d_xz_5x5x5[11][0]+1, c2d_xz_5x5x5[11][1], c2d_xz_5x5x5[11][2]-1), + + offset5x5x5(c2d_yz_5x5x5[0][0]+1, c2d_yz_5x5x5[0][1]-1, c2d_yz_5x5x5[0][2]), // YZ + offset5x5x5(c2d_yz_5x5x5[1][0]+1, c2d_yz_5x5x5[1][1]-1, c2d_yz_5x5x5[1][2]), + offset5x5x5(c2d_yz_5x5x5[2][0]+1, c2d_yz_5x5x5[2][1]-1, c2d_yz_5x5x5[2][2]), + offset5x5x5(c2d_yz_5x5x5[3][0]+1, c2d_yz_5x5x5[3][1]-1, c2d_yz_5x5x5[3][2]), + offset5x5x5(c2d_yz_5x5x5[4][0]+1, c2d_yz_5x5x5[4][1]-1, c2d_yz_5x5x5[4][2]), + offset5x5x5(c2d_yz_5x5x5[5][0]+1, c2d_yz_5x5x5[5][1]-1, c2d_yz_5x5x5[5][2]), + offset5x5x5(c2d_yz_5x5x5[6][0]+1, c2d_yz_5x5x5[6][1]-1, c2d_yz_5x5x5[6][2]), + offset5x5x5(c2d_yz_5x5x5[7][0]+1, c2d_yz_5x5x5[7][1]-1, c2d_yz_5x5x5[7][2]), + offset5x5x5(c2d_yz_5x5x5[8][0]+1, c2d_yz_5x5x5[8][1]-1, c2d_yz_5x5x5[8][2]), + offset5x5x5(c2d_yz_5x5x5[9][0]+1, c2d_yz_5x5x5[9][1]-1, c2d_yz_5x5x5[9][2]), + offset5x5x5(c2d_yz_5x5x5[10][0]+1, c2d_yz_5x5x5[10][1]-1, c2d_yz_5x5x5[10][2]), + offset5x5x5(c2d_yz_5x5x5[11][0]+1, c2d_yz_5x5x5[11][1]-1, c2d_yz_5x5x5[11][2]) + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff2D_RR_Offset_5x5x5(SIZE i) { + static constexpr int offset[36] = { offset5x5x5(c2d_xy_5x5x5[0][0], c2d_xy_5x5x5[0][1]+1, c2d_xy_5x5x5[0][2]+1), // XY + offset5x5x5(c2d_xy_5x5x5[1][0], c2d_xy_5x5x5[1][1]+1, c2d_xy_5x5x5[1][2]+1), + offset5x5x5(c2d_xy_5x5x5[2][0], c2d_xy_5x5x5[2][1]+1, c2d_xy_5x5x5[2][2]+1), + offset5x5x5(c2d_xy_5x5x5[3][0], c2d_xy_5x5x5[3][1]+1, c2d_xy_5x5x5[3][2]+1), + offset5x5x5(c2d_xy_5x5x5[4][0], c2d_xy_5x5x5[4][1]+1, c2d_xy_5x5x5[4][2]+1), + offset5x5x5(c2d_xy_5x5x5[5][0], c2d_xy_5x5x5[5][1]+1, c2d_xy_5x5x5[5][2]+1), + offset5x5x5(c2d_xy_5x5x5[6][0], c2d_xy_5x5x5[6][1]+1, c2d_xy_5x5x5[6][2]+1), + offset5x5x5(c2d_xy_5x5x5[7][0], c2d_xy_5x5x5[7][1]+1, c2d_xy_5x5x5[7][2]+1), + offset5x5x5(c2d_xy_5x5x5[8][0], c2d_xy_5x5x5[8][1]+1, c2d_xy_5x5x5[8][2]+1), + offset5x5x5(c2d_xy_5x5x5[9][0], c2d_xy_5x5x5[9][1]+1, c2d_xy_5x5x5[9][2]+1), + offset5x5x5(c2d_xy_5x5x5[10][0], c2d_xy_5x5x5[10][1]+1, c2d_xy_5x5x5[10][2]+1), + offset5x5x5(c2d_xy_5x5x5[11][0], c2d_xy_5x5x5[11][1]+1, c2d_xy_5x5x5[11][2]+1), + + offset5x5x5(c2d_xz_5x5x5[0][0]+1, c2d_xz_5x5x5[0][1], c2d_xz_5x5x5[0][2]+1), // XZ + offset5x5x5(c2d_xz_5x5x5[1][0]+1, c2d_xz_5x5x5[1][1], c2d_xz_5x5x5[1][2]+1), + offset5x5x5(c2d_xz_5x5x5[2][0]+1, c2d_xz_5x5x5[2][1], c2d_xz_5x5x5[2][2]+1), + offset5x5x5(c2d_xz_5x5x5[3][0]+1, c2d_xz_5x5x5[3][1], c2d_xz_5x5x5[3][2]+1), + offset5x5x5(c2d_xz_5x5x5[4][0]+1, c2d_xz_5x5x5[4][1], c2d_xz_5x5x5[4][2]+1), + offset5x5x5(c2d_xz_5x5x5[5][0]+1, c2d_xz_5x5x5[5][1], c2d_xz_5x5x5[5][2]+1), + offset5x5x5(c2d_xz_5x5x5[6][0]+1, c2d_xz_5x5x5[6][1], c2d_xz_5x5x5[6][2]+1), + offset5x5x5(c2d_xz_5x5x5[7][0]+1, c2d_xz_5x5x5[7][1], c2d_xz_5x5x5[7][2]+1), + offset5x5x5(c2d_xz_5x5x5[8][0]+1, c2d_xz_5x5x5[8][1], c2d_xz_5x5x5[8][2]+1), + offset5x5x5(c2d_xz_5x5x5[9][0]+1, c2d_xz_5x5x5[9][1], c2d_xz_5x5x5[9][2]+1), + offset5x5x5(c2d_xz_5x5x5[10][0]+1, c2d_xz_5x5x5[10][1], c2d_xz_5x5x5[10][2]+1), + offset5x5x5(c2d_xz_5x5x5[11][0]+1, c2d_xz_5x5x5[11][1], c2d_xz_5x5x5[11][2]+1), + + offset5x5x5(c2d_yz_5x5x5[0][0]+1, c2d_yz_5x5x5[0][1]+1, c2d_yz_5x5x5[0][2]), // YZ + offset5x5x5(c2d_yz_5x5x5[1][0]+1, c2d_yz_5x5x5[1][1]+1, c2d_yz_5x5x5[1][2]), + offset5x5x5(c2d_yz_5x5x5[2][0]+1, c2d_yz_5x5x5[2][1]+1, c2d_yz_5x5x5[2][2]), + offset5x5x5(c2d_yz_5x5x5[3][0]+1, c2d_yz_5x5x5[3][1]+1, c2d_yz_5x5x5[3][2]), + offset5x5x5(c2d_yz_5x5x5[4][0]+1, c2d_yz_5x5x5[4][1]+1, c2d_yz_5x5x5[4][2]), + offset5x5x5(c2d_yz_5x5x5[5][0]+1, c2d_yz_5x5x5[5][1]+1, c2d_yz_5x5x5[5][2]), + offset5x5x5(c2d_yz_5x5x5[6][0]+1, c2d_yz_5x5x5[6][1]+1, c2d_yz_5x5x5[6][2]), + offset5x5x5(c2d_yz_5x5x5[7][0]+1, c2d_yz_5x5x5[7][1]+1, c2d_yz_5x5x5[7][2]), + offset5x5x5(c2d_yz_5x5x5[8][0]+1, c2d_yz_5x5x5[8][1]+1, c2d_yz_5x5x5[8][2]), + offset5x5x5(c2d_yz_5x5x5[9][0]+1, c2d_yz_5x5x5[9][1]+1, c2d_yz_5x5x5[9][2]), + offset5x5x5(c2d_yz_5x5x5[10][0]+1, c2d_yz_5x5x5[10][1]+1, c2d_yz_5x5x5[10][2]), + offset5x5x5(c2d_yz_5x5x5[11][0]+1, c2d_yz_5x5x5[11][1]+1, c2d_yz_5x5x5[11][2]) + }; + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_MMM_Offset_5x5x5(SIZE i) { + static constexpr int offset[8] = {offset5x5x5(c3d_xyz_5x5x5[0][0], c3d_xyz_5x5x5[0][1], c3d_xyz_5x5x5[0][2]), + offset5x5x5(c3d_xyz_5x5x5[1][0], c3d_xyz_5x5x5[1][1], c3d_xyz_5x5x5[1][2]), + offset5x5x5(c3d_xyz_5x5x5[2][0], c3d_xyz_5x5x5[2][1], c3d_xyz_5x5x5[2][2]), + offset5x5x5(c3d_xyz_5x5x5[3][0], c3d_xyz_5x5x5[3][1], c3d_xyz_5x5x5[3][2]), + offset5x5x5(c3d_xyz_5x5x5[4][0], c3d_xyz_5x5x5[4][1], c3d_xyz_5x5x5[4][2]), + offset5x5x5(c3d_xyz_5x5x5[5][0], c3d_xyz_5x5x5[5][1], c3d_xyz_5x5x5[5][2]), + offset5x5x5(c3d_xyz_5x5x5[6][0], c3d_xyz_5x5x5[6][1], c3d_xyz_5x5x5[6][2])}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_LLL_Offset_5x5x5(SIZE i) { + static constexpr int offset[8] = {offset5x5x5(c3d_xyz_5x5x5[0][0]-1, c3d_xyz_5x5x5[0][1]-1, c3d_xyz_5x5x5[0][2]-1), // XY + offset5x5x5(c3d_xyz_5x5x5[1][0]-1, c3d_xyz_5x5x5[1][1]-1, c3d_xyz_5x5x5[1][2]-1), + offset5x5x5(c3d_xyz_5x5x5[2][0]-1, c3d_xyz_5x5x5[2][1]-1, c3d_xyz_5x5x5[2][2]-1), + offset5x5x5(c3d_xyz_5x5x5[3][0]-1, c3d_xyz_5x5x5[3][1]-1, c3d_xyz_5x5x5[3][2]-1), + offset5x5x5(c3d_xyz_5x5x5[4][0]-1, c3d_xyz_5x5x5[4][1]-1, c3d_xyz_5x5x5[4][2]-1), + offset5x5x5(c3d_xyz_5x5x5[5][0]-1, c3d_xyz_5x5x5[5][1]-1, c3d_xyz_5x5x5[5][2]-1), + offset5x5x5(c3d_xyz_5x5x5[6][0]-1, c3d_xyz_5x5x5[6][1]-1, c3d_xyz_5x5x5[6][2]-1), + offset5x5x5(c3d_xyz_5x5x5[7][0]-1, c3d_xyz_5x5x5[7][1]-1, c3d_xyz_5x5x5[7][2]-1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_LLR_Offset_5x5x5(SIZE i) { + static constexpr int offset[8] = {offset5x5x5(c3d_xyz_5x5x5[0][0]-1, c3d_xyz_5x5x5[0][1]-1, c3d_xyz_5x5x5[0][2]+1), // XY + offset5x5x5(c3d_xyz_5x5x5[1][0]-1, c3d_xyz_5x5x5[1][1]-1, c3d_xyz_5x5x5[1][2]+1), + offset5x5x5(c3d_xyz_5x5x5[2][0]-1, c3d_xyz_5x5x5[2][1]-1, c3d_xyz_5x5x5[2][2]+1), + offset5x5x5(c3d_xyz_5x5x5[3][0]-1, c3d_xyz_5x5x5[3][1]-1, c3d_xyz_5x5x5[3][2]+1), + offset5x5x5(c3d_xyz_5x5x5[4][0]-1, c3d_xyz_5x5x5[4][1]-1, c3d_xyz_5x5x5[4][2]+1), + offset5x5x5(c3d_xyz_5x5x5[5][0]-1, c3d_xyz_5x5x5[5][1]-1, c3d_xyz_5x5x5[5][2]+1), + offset5x5x5(c3d_xyz_5x5x5[6][0]-1, c3d_xyz_5x5x5[6][1]-1, c3d_xyz_5x5x5[6][2]+1), + offset5x5x5(c3d_xyz_5x5x5[7][0]-1, c3d_xyz_5x5x5[7][1]-1, c3d_xyz_5x5x5[7][2]+1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_LRL_Offset_5x5x5(SIZE i) { + static constexpr int offset[8] = {offset5x5x5(c3d_xyz_5x5x5[0][0]-1, c3d_xyz_5x5x5[0][1]+1, c3d_xyz_5x5x5[0][2]-1), // XY + offset5x5x5(c3d_xyz_5x5x5[1][0]-1, c3d_xyz_5x5x5[1][1]+1, c3d_xyz_5x5x5[1][2]-1), + offset5x5x5(c3d_xyz_5x5x5[2][0]-1, c3d_xyz_5x5x5[2][1]+1, c3d_xyz_5x5x5[2][2]-1), + offset5x5x5(c3d_xyz_5x5x5[3][0]-1, c3d_xyz_5x5x5[3][1]+1, c3d_xyz_5x5x5[3][2]-1), + offset5x5x5(c3d_xyz_5x5x5[4][0]-1, c3d_xyz_5x5x5[4][1]+1, c3d_xyz_5x5x5[4][2]-1), + offset5x5x5(c3d_xyz_5x5x5[5][0]-1, c3d_xyz_5x5x5[5][1]+1, c3d_xyz_5x5x5[5][2]-1), + offset5x5x5(c3d_xyz_5x5x5[6][0]-1, c3d_xyz_5x5x5[6][1]+1, c3d_xyz_5x5x5[6][2]-1), + offset5x5x5(c3d_xyz_5x5x5[7][0]-1, c3d_xyz_5x5x5[7][1]+1, c3d_xyz_5x5x5[7][2]-1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_LRR_Offset_5x5x5(SIZE i) { + static constexpr int offset[8] = {offset5x5x5(c3d_xyz_5x5x5[0][0]-1, c3d_xyz_5x5x5[0][1]+1, c3d_xyz_5x5x5[0][2]+1), // XY + offset5x5x5(c3d_xyz_5x5x5[1][0]-1, c3d_xyz_5x5x5[1][1]+1, c3d_xyz_5x5x5[1][2]+1), + offset5x5x5(c3d_xyz_5x5x5[2][0]-1, c3d_xyz_5x5x5[2][1]+1, c3d_xyz_5x5x5[2][2]+1), + offset5x5x5(c3d_xyz_5x5x5[3][0]-1, c3d_xyz_5x5x5[3][1]+1, c3d_xyz_5x5x5[3][2]+1), + offset5x5x5(c3d_xyz_5x5x5[4][0]-1, c3d_xyz_5x5x5[4][1]+1, c3d_xyz_5x5x5[4][2]+1), + offset5x5x5(c3d_xyz_5x5x5[5][0]-1, c3d_xyz_5x5x5[5][1]+1, c3d_xyz_5x5x5[5][2]+1), + offset5x5x5(c3d_xyz_5x5x5[6][0]-1, c3d_xyz_5x5x5[6][1]+1, c3d_xyz_5x5x5[6][2]+1), + offset5x5x5(c3d_xyz_5x5x5[7][0]-1, c3d_xyz_5x5x5[7][1]+1, c3d_xyz_5x5x5[7][2]+1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_RLL_Offset_5x5x5(SIZE i) { + static constexpr int offset[8] = {offset5x5x5(c3d_xyz_5x5x5[0][0]+1, c3d_xyz_5x5x5[0][1]-1, c3d_xyz_5x5x5[0][2]-1), // XY + offset5x5x5(c3d_xyz_5x5x5[1][0]+1, c3d_xyz_5x5x5[1][1]-1, c3d_xyz_5x5x5[1][2]-1), + offset5x5x5(c3d_xyz_5x5x5[2][0]+1, c3d_xyz_5x5x5[2][1]-1, c3d_xyz_5x5x5[2][2]-1), + offset5x5x5(c3d_xyz_5x5x5[3][0]+1, c3d_xyz_5x5x5[3][1]-1, c3d_xyz_5x5x5[3][2]-1), + offset5x5x5(c3d_xyz_5x5x5[4][0]+1, c3d_xyz_5x5x5[4][1]-1, c3d_xyz_5x5x5[4][2]-1), + offset5x5x5(c3d_xyz_5x5x5[5][0]+1, c3d_xyz_5x5x5[5][1]-1, c3d_xyz_5x5x5[5][2]-1), + offset5x5x5(c3d_xyz_5x5x5[6][0]+1, c3d_xyz_5x5x5[6][1]-1, c3d_xyz_5x5x5[6][2]-1), + offset5x5x5(c3d_xyz_5x5x5[7][0]+1, c3d_xyz_5x5x5[7][1]-1, c3d_xyz_5x5x5[7][2]-1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_RLR_Offset_5x5x5(SIZE i) { + static constexpr int offset[8] = {offset5x5x5(c3d_xyz_5x5x5[0][0]+1, c3d_xyz_5x5x5[0][1]-1, c3d_xyz_5x5x5[0][2]+1), // XY + offset5x5x5(c3d_xyz_5x5x5[1][0]+1, c3d_xyz_5x5x5[1][1]-1, c3d_xyz_5x5x5[1][2]+1), + offset5x5x5(c3d_xyz_5x5x5[2][0]+1, c3d_xyz_5x5x5[2][1]-1, c3d_xyz_5x5x5[2][2]+1), + offset5x5x5(c3d_xyz_5x5x5[3][0]+1, c3d_xyz_5x5x5[3][1]-1, c3d_xyz_5x5x5[3][2]+1), + offset5x5x5(c3d_xyz_5x5x5[4][0]+1, c3d_xyz_5x5x5[4][1]-1, c3d_xyz_5x5x5[4][2]+1), + offset5x5x5(c3d_xyz_5x5x5[5][0]+1, c3d_xyz_5x5x5[5][1]-1, c3d_xyz_5x5x5[5][2]+1), + offset5x5x5(c3d_xyz_5x5x5[6][0]+1, c3d_xyz_5x5x5[6][1]-1, c3d_xyz_5x5x5[6][2]+1), + offset5x5x5(c3d_xyz_5x5x5[7][0]+1, c3d_xyz_5x5x5[7][1]-1, c3d_xyz_5x5x5[7][2]+1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_RRL_Offset_5x5x5(SIZE i) { + static constexpr int offset[8] = {offset5x5x5(c3d_xyz_5x5x5[0][0]+1, c3d_xyz_5x5x5[0][1]+1, c3d_xyz_5x5x5[0][2]-1), // XY + offset5x5x5(c3d_xyz_5x5x5[1][0]+1, c3d_xyz_5x5x5[1][1]+1, c3d_xyz_5x5x5[1][2]-1), + offset5x5x5(c3d_xyz_5x5x5[2][0]+1, c3d_xyz_5x5x5[2][1]+1, c3d_xyz_5x5x5[2][2]-1), + offset5x5x5(c3d_xyz_5x5x5[3][0]+1, c3d_xyz_5x5x5[3][1]+1, c3d_xyz_5x5x5[3][2]-1), + offset5x5x5(c3d_xyz_5x5x5[4][0]+1, c3d_xyz_5x5x5[4][1]+1, c3d_xyz_5x5x5[4][2]-1), + offset5x5x5(c3d_xyz_5x5x5[5][0]+1, c3d_xyz_5x5x5[5][1]+1, c3d_xyz_5x5x5[5][2]-1), + offset5x5x5(c3d_xyz_5x5x5[6][0]+1, c3d_xyz_5x5x5[6][1]+1, c3d_xyz_5x5x5[6][2]-1), + offset5x5x5(c3d_xyz_5x5x5[7][0]+1, c3d_xyz_5x5x5[7][1]+1, c3d_xyz_5x5x5[7][2]-1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_RRR_Offset_5x5x5(SIZE i) { + static constexpr int offset[8] = {offset5x5x5(c3d_xyz_5x5x5[0][0]+1, c3d_xyz_5x5x5[0][1]+1, c3d_xyz_5x5x5[0][2]+1), // XY + offset5x5x5(c3d_xyz_5x5x5[1][0]+1, c3d_xyz_5x5x5[1][1]+1, c3d_xyz_5x5x5[1][2]+1), + offset5x5x5(c3d_xyz_5x5x5[2][0]+1, c3d_xyz_5x5x5[2][1]+1, c3d_xyz_5x5x5[2][2]+1), + offset5x5x5(c3d_xyz_5x5x5[3][0]+1, c3d_xyz_5x5x5[3][1]+1, c3d_xyz_5x5x5[3][2]+1), + offset5x5x5(c3d_xyz_5x5x5[4][0]+1, c3d_xyz_5x5x5[4][1]+1, c3d_xyz_5x5x5[4][2]+1), + offset5x5x5(c3d_xyz_5x5x5[5][0]+1, c3d_xyz_5x5x5[5][1]+1, c3d_xyz_5x5x5[5][2]+1), + offset5x5x5(c3d_xyz_5x5x5[6][0]+1, c3d_xyz_5x5x5[6][1]+1, c3d_xyz_5x5x5[6][2]+1), + offset5x5x5(c3d_xyz_5x5x5[7][0]+1, c3d_xyz_5x5x5[7][1]+1, c3d_xyz_5x5x5[7][2]+1)}; + + return offset[i]; +} + +MGARDX_EXEC int const *MassTrans_X_Offset_5x5x5(SIZE i) { + static constexpr int zero_offset = 0; + #define OFFSET(Z, Y) \ + { \ + zero_offset, \ + zero_offset, \ + offset5x5x5(Z, Y, coarse_x_5x5x5[0], 5, 5), \ + offset5x5x5(Z, Y, coarse_x_5x5x5[0]+1, 5, 5), \ + offset5x5x5(Z, Y, coarse_x_5x5x5[0]+2, 5, 5), \ + offset5x5x5(Z, Y, coarse_x_5x5x5[0], 3, 5) \ + }, \ + { \ + offset5x5x5(Z, Y, coarse_x_5x5x5[1]-2, 5, 5), \ + offset5x5x5(Z, Y, coarse_x_5x5x5[1]-1, 5, 5), \ + offset5x5x5(Z, Y, coarse_x_5x5x5[1], 5, 5), \ + offset5x5x5(Z, Y, coarse_x_5x5x5[1]+1, 5, 5), \ + offset5x5x5(Z, Y, coarse_x_5x5x5[1]+2, 5, 5), \ + offset5x5x5(Z, Y, coarse_x_5x5x5[1], 3, 5) \ + }, \ + { \ + offset5x5x5(Z, Y, coarse_x_5x5x5[2]-2, 5, 5), \ + offset5x5x5(Z, Y, coarse_x_5x5x5[2]-1, 5, 5), \ + offset5x5x5(Z, Y, coarse_x_5x5x5[2], 5, 5), \ + zero_offset, \ + zero_offset, \ + offset5x5x5(Z, Y, coarse_x_5x5x5[2], 3, 5) \ + } + + static constexpr int offset[75][6] = { + OFFSET(0, 0), OFFSET(0, 1), OFFSET(0, 2), OFFSET(0, 3), OFFSET(0, 4), + OFFSET(1, 0), OFFSET(1, 1), OFFSET(1, 2), OFFSET(1, 3), OFFSET(1, 4), + OFFSET(2, 0), OFFSET(2, 1), OFFSET(2, 2), OFFSET(2, 3), OFFSET(2, 4), + OFFSET(3, 0), OFFSET(3, 1), OFFSET(3, 2), OFFSET(3, 3), OFFSET(3, 4), + OFFSET(4, 0), OFFSET(4, 1), OFFSET(4, 2), OFFSET(4, 3), OFFSET(4, 4) + }; + #undef OFFSET + return offset[i]; +} + +MGARDX_EXEC int const *MassTrans_Y_Offset_5x5x5(SIZE i) { + static constexpr int zero_offset = 0; + #define OFFSET(Z, X) \ + { \ + zero_offset, \ + zero_offset, \ + offset5x5x5(Z, coarse_y_5x5x5[0], X, 3, 5), \ + offset5x5x5(Z, coarse_y_5x5x5[0]+1, X, 3, 5), \ + offset5x5x5(Z, coarse_y_5x5x5[0]+2, X, 3, 5), \ + offset5x5x5(Z, coarse_y_5x5x5[0], X, 3, 3) \ + }, \ + { \ + offset5x5x5(Z, coarse_y_5x5x5[1]-2, X, 3, 5), \ + offset5x5x5(Z, coarse_y_5x5x5[1]-1, X, 3, 5), \ + offset5x5x5(Z, coarse_y_5x5x5[1], X, 3, 5), \ + offset5x5x5(Z, coarse_y_5x5x5[1]+1, X, 3, 5), \ + offset5x5x5(Z, coarse_y_5x5x5[1]+2, X, 3, 5), \ + offset5x5x5(Z, coarse_y_5x5x5[1], X, 3, 3) \ + }, \ + { \ + offset5x5x5(Z, coarse_y_5x5x5[2]-2, X, 3, 5), \ + offset5x5x5(Z, coarse_y_5x5x5[2]-1, X, 3, 5), \ + offset5x5x5(Z, coarse_y_5x5x5[2], X, 3, 5), \ + zero_offset, \ + zero_offset, \ + offset5x5x5(Z, coarse_y_5x5x5[2], X, 3, 3) \ + } + + static constexpr int offset[45][6] = { + OFFSET(0, 0), OFFSET(0, 1), OFFSET(0, 2), + OFFSET(1, 0), OFFSET(1, 1), OFFSET(1, 2), + OFFSET(2, 0), OFFSET(2, 1), OFFSET(2, 2), + OFFSET(3, 0), OFFSET(3, 1), OFFSET(3, 2), + OFFSET(4, 0), OFFSET(4, 1), OFFSET(4, 2) + }; + #undef OFFSET + return offset[i]; +} + +MGARDX_EXEC int const *MassTrans_Z_Offset_5x5x5(SIZE i) { + static constexpr int zero_offset = 0; + #define OFFSET(Y, X) \ + { \ + zero_offset, \ + zero_offset, \ + offset5x5x5(coarse_z_5x5x5[0], Y, X, 3, 3), \ + offset5x5x5(coarse_z_5x5x5[0]+1, Y, X, 3, 3), \ + offset5x5x5(coarse_z_5x5x5[0]+2, Y, X, 3, 3), \ + offset5x5x5(coarse_z_5x5x5[0], Y, X, 3, 3) \ + }, \ + { \ + offset5x5x5(coarse_z_5x5x5[1]-2, Y, X, 3, 3), \ + offset5x5x5(coarse_z_5x5x5[1]-1, Y, X, 3, 3), \ + offset5x5x5(coarse_z_5x5x5[1], Y, X, 3, 3), \ + offset5x5x5(coarse_z_5x5x5[1]+1, Y, X, 3, 3), \ + offset5x5x5(coarse_z_5x5x5[1]+2, Y, X, 3, 3), \ + offset5x5x5(coarse_z_5x5x5[1], Y, X, 3, 3) \ + }, \ + { \ + offset5x5x5(coarse_z_5x5x5[2]-2, Y, X, 3, 3), \ + offset5x5x5(coarse_z_5x5x5[2]-1, Y, X, 3, 3), \ + offset5x5x5(coarse_z_5x5x5[2], Y, X, 3, 3), \ + zero_offset, \ + zero_offset, \ + offset5x5x5(coarse_z_5x5x5[2], Y, X, 3, 3) \ + } + + static constexpr int offset[27][6] = { + OFFSET(0, 0), OFFSET(0, 1), OFFSET(0, 2), + OFFSET(1, 0), OFFSET(1, 1), OFFSET(1, 2), + OFFSET(2, 0), OFFSET(2, 1), OFFSET(2, 2) + }; + #undef OFFSET + return offset[i]; +} + +MGARDX_EXEC int const *TriDiag_X_Offset_5x5x5(SIZE i) { + static constexpr int offset[9][3] = { + {offset5x5x5(0, 0, 0, 3, 3), offset5x5x5(0, 0, 1, 3, 3), offset5x5x5(0, 0, 2, 3, 3)}, + {offset5x5x5(0, 1, 0, 3, 3), offset5x5x5(0, 1, 1, 3, 3), offset5x5x5(0, 1, 2, 3, 3)}, + {offset5x5x5(0, 2, 0, 3, 3), offset5x5x5(0, 2, 1, 3, 3), offset5x5x5(0, 2, 2, 3, 3)}, + + {offset5x5x5(1, 0, 0, 3, 3), offset5x5x5(1, 0, 1, 3, 3), offset5x5x5(1, 0, 2, 3, 3)}, + {offset5x5x5(1, 1, 0, 3, 3), offset5x5x5(1, 1, 1, 3, 3), offset5x5x5(1, 1, 2, 3, 3)}, + {offset5x5x5(1, 2, 0, 3, 3), offset5x5x5(1, 2, 1, 3, 3), offset5x5x5(1, 2, 2, 3, 3)}, + + {offset5x5x5(2, 0, 0, 3, 3), offset5x5x5(2, 0, 1, 3, 3), offset5x5x5(2, 0, 2, 3, 3)}, + {offset5x5x5(2, 1, 0, 3, 3), offset5x5x5(2, 1, 1, 3, 3), offset5x5x5(2, 1, 2, 3, 3)}, + {offset5x5x5(2, 2, 0, 3, 3), offset5x5x5(2, 2, 1, 3, 3), offset5x5x5(2, 2, 2, 3, 3)} + }; + return offset[i]; +} + +MGARDX_EXEC int const *TriDiag_Y_Offset_5x5x5(SIZE i) { + static constexpr int offset[9][3] = { + {offset5x5x5(0, 0, 0, 3, 3), offset5x5x5(0, 1, 0, 3, 3), offset5x5x5(0, 2, 0, 3, 3)}, + {offset5x5x5(0, 0, 1, 3, 3), offset5x5x5(0, 1, 1, 3, 3), offset5x5x5(0, 2, 1, 3, 3)}, + {offset5x5x5(0, 0, 2, 3, 3), offset5x5x5(0, 1, 2, 3, 3), offset5x5x5(0, 2, 2, 3, 3)}, + + {offset5x5x5(1, 0, 0, 3, 3), offset5x5x5(1, 1, 0, 3, 3), offset5x5x5(1, 2, 0, 3, 3)}, + {offset5x5x5(1, 0, 1, 3, 3), offset5x5x5(1, 1, 1, 3, 3), offset5x5x5(1, 2, 1, 3, 3)}, + {offset5x5x5(1, 0, 2, 3, 3), offset5x5x5(1, 1, 2, 3, 3), offset5x5x5(1, 2, 2, 3, 3)}, + + {offset5x5x5(2, 0, 0, 3, 3), offset5x5x5(2, 1, 0, 3, 3), offset5x5x5(2, 2, 0, 3, 3)}, + {offset5x5x5(2, 0, 1, 3, 3), offset5x5x5(2, 1, 1, 3, 3), offset5x5x5(2, 2, 1, 3, 3)}, + {offset5x5x5(2, 0, 2, 3, 3), offset5x5x5(2, 1, 2, 3, 3), offset5x5x5(2, 2, 2, 3, 3)} + }; + return offset[i]; +} + +MGARDX_EXEC int const *TriDiag_Z_Offset_5x5x5(SIZE i) { + static constexpr int offset[9][3] = { + {offset5x5x5(0, 0, 0, 3, 3), offset5x5x5(1, 0, 0, 3, 3), offset5x5x5(2, 0, 0, 3, 3)}, + {offset5x5x5(0, 0, 1, 3, 3), offset5x5x5(1, 0, 1, 3, 3), offset5x5x5(2, 0, 1, 3, 3)}, + {offset5x5x5(0, 0, 2, 3, 3), offset5x5x5(1, 0, 2, 3, 3), offset5x5x5(2, 0, 2, 3, 3)}, + + {offset5x5x5(0, 1, 0, 3, 3), offset5x5x5(1, 1, 0, 3, 3), offset5x5x5(2, 1, 0, 3, 3)}, + {offset5x5x5(0, 1, 1, 3, 3), offset5x5x5(1, 1, 1, 3, 3), offset5x5x5(2, 1, 1, 3, 3)}, + {offset5x5x5(0, 1, 2, 3, 3), offset5x5x5(1, 1, 2, 3, 3), offset5x5x5(2, 1, 2, 3, 3)}, + + {offset5x5x5(0, 2, 0, 3, 3), offset5x5x5(1, 2, 0, 3, 3), offset5x5x5(2, 2, 0, 3, 3)}, + {offset5x5x5(0, 2, 1, 3, 3), offset5x5x5(1, 2, 1, 3, 3), offset5x5x5(2, 2, 1, 3, 3)}, + {offset5x5x5(0, 2, 2, 3, 3), offset5x5x5(1, 2, 2, 3, 3), offset5x5x5(2, 2, 2, 3, 3)} + }; + return offset[i]; +} + +MGARDX_EXEC int Coarse_Offset_5x5x5(SIZE i) { + static constexpr int offset[27] = { + offset5x5x5(0, 0, 0, 5, 5), offset5x5x5(0, 0, 2, 5, 5), offset5x5x5(0, 0, 4, 5, 5), + offset5x5x5(0, 2, 0, 5, 5), offset5x5x5(0, 2, 2, 5, 5), offset5x5x5(0, 2, 4, 5, 5), + offset5x5x5(0, 4, 0, 5, 5), offset5x5x5(0, 4, 2, 5, 5), offset5x5x5(0, 4, 4, 5, 5), + + offset5x5x5(2, 0, 0, 5, 5), offset5x5x5(2, 0, 2, 5, 5), offset5x5x5(2, 0, 4, 5, 5), + offset5x5x5(2, 2, 0, 5, 5), offset5x5x5(2, 2, 2, 5, 5), offset5x5x5(2, 2, 4, 5, 5), + offset5x5x5(2, 4, 0, 5, 5), offset5x5x5(2, 4, 2, 5, 5), offset5x5x5(2, 4, 4, 5, 5), + + offset5x5x5(4, 0, 0, 5, 5), offset5x5x5(4, 0, 2, 5, 5), offset5x5x5(4, 0, 4, 5, 5), + offset5x5x5(4, 2, 0, 5, 5), offset5x5x5(4, 2, 2, 5, 5), offset5x5x5(4, 2, 4, 5, 5), + offset5x5x5(4, 4, 0, 5, 5), offset5x5x5(4, 4, 2, 5, 5), offset5x5x5(4, 4, 4, 5, 5), + + }; + return offset[i]; +} + +// clang-format on +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTable8x8x8.hpp b/include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTable8x8x8.hpp new file mode 100644 index 000000000..9d37f50bb --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTable8x8x8.hpp @@ -0,0 +1,2522 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_INDEX_TABLE_8X8X8_HPP +#define MGARD_X_160_INDEX_TABLE_8X8X8_HPP + +namespace mgard_x_160 { +// clang-format off + +// --- Padded shared-memory layout for the 8x8x8 input plane (sm_v) --- +// The transform stores the 8^3 block in shared memory (sm_v) and then reads it +// with power-of-two strides (coefficient stencils differ by 2/4/8 in y and z), +// which collide on the 32 shared-memory banks (ncu measured avg 2.8-way load +// conflicts, ~65% of shared-load wavefronts wasted). Pad sm_v's y-stride to 9 +// and z-stride to 71 (both coprime with 32) so those accesses scatter across +// banks. Only sm_v is padded; sm_x/sm_y/sm_z keep their layouts. Every sm_v +// access routes through offset8x8x8, so the constexpr offset tables below +// regenerate for the padded layout automatically. +static constexpr int SMV_LDY_8x8x8 = 9; // physical x-row length (was 8) +static constexpr int SMV_LDZ_8x8x8 = 71; // physical z-plane stride (was 64) +static constexpr int SMV_SIZE_8x8x8 = 568; // 7*71 + 7*9 + 7 + 1 (was 512) + +MGARDX_EXEC constexpr int offset8x8x8(SIZE z, SIZE y, SIZE x) { + return z * SMV_LDZ_8x8x8 + y * SMV_LDY_8x8x8 + x; +} + +MGARDX_EXEC constexpr int offset8x8x8(SIZE z, SIZE y, SIZE x, SIZE ld1, SIZE ld2) { + // sm_v is the only array with an 8x8 leading layout; use the padded strides. + if (ld1 == 8 && ld2 == 8) + return z * SMV_LDZ_8x8x8 + y * SMV_LDY_8x8x8 + x; + return z * ld1 * ld2 + y * ld1 + x; +} + +static constexpr int c1d_x_8x8x8[75][3] = { {0, 0, 1}, {0, 0, 3}, {0, 0, 5}, // X + {0, 2, 1}, {0, 2, 3}, {0, 2, 5}, + {0, 4, 1}, {0, 4, 3}, {0, 4, 5}, + {0, 6, 1}, {0, 6, 3}, {0, 6, 5}, + {0, 7, 1}, {0, 7, 3}, {0, 7, 5}, + {2, 0, 1}, {2, 0, 3}, {2, 0, 5}, + {2, 2, 1}, {2, 2, 3}, {2, 2, 5}, + {2, 4, 1}, {2, 4, 3}, {2, 4, 5}, + {2, 6, 1}, {2, 6, 3}, {2, 6, 5}, + {2, 7, 1}, {2, 7, 3}, {2, 7, 5}, + {4, 0, 1}, {4, 0, 3}, {4, 0, 5}, + {4, 2, 1}, {4, 2, 3}, {4, 2, 5}, + {4, 4, 1}, {4, 4, 3}, {4, 4, 5}, + {4, 6, 1}, {4, 6, 3}, {4, 6, 5}, + {4, 7, 1}, {4, 7, 3}, {4, 7, 5}, + {6, 0, 1}, {6, 0, 3}, {6, 0, 5}, + {6, 2, 1}, {6, 2, 3}, {6, 2, 5}, + {6, 4, 1}, {6, 4, 3}, {6, 4, 5}, + {6, 6, 1}, {6, 6, 3}, {6, 6, 5}, + {6, 7, 1}, {6, 7, 3}, {6, 7, 5}, + {7, 0, 1}, {7, 0, 3}, {7, 0, 5}, + {7, 2, 1}, {7, 2, 3}, {7, 2, 5}, + {7, 4, 1}, {7, 4, 3}, {7, 4, 5}, + {7, 6, 1}, {7, 6, 3}, {7, 6, 5}, + {7, 7, 1}, {7, 7, 3}, {7, 7, 5}}; + +static constexpr int c1d_y_8x8x8[75][3] = { {0, 1, 0}, {0, 3, 0}, {0, 5, 0}, // Y + {0, 1, 2}, {0, 3, 2}, {0, 5, 2}, + {0, 1, 4}, {0, 3, 4}, {0, 5, 4}, + {0, 1, 6}, {0, 3, 6}, {0, 5, 6}, + {0, 1, 7}, {0, 3, 7}, {0, 5, 7}, + {2, 1, 0}, {2, 3, 0}, {2, 5, 0}, + {2, 1, 2}, {2, 3, 2}, {2, 5, 2}, + {2, 1, 4}, {2, 3, 4}, {2, 5, 4}, + {2, 1, 6}, {2, 3, 6}, {2, 5, 6}, + {2, 1, 7}, {2, 3, 7}, {2, 5, 7}, + {4, 1, 0}, {4, 3, 0}, {4, 5, 0}, + {4, 1, 2}, {4, 3, 2}, {4, 5, 2}, + {4, 1, 4}, {4, 3, 4}, {4, 5, 4}, + {4, 1, 6}, {4, 3, 6}, {4, 5, 6}, + {4, 1, 7}, {4, 3, 7}, {4, 5, 7}, + {6, 1, 0}, {6, 3, 0}, {6, 5, 0}, + {6, 1, 2}, {6, 3, 2}, {6, 5, 2}, + {6, 1, 4}, {6, 3, 4}, {6, 5, 4}, + {6, 1, 6}, {6, 3, 6}, {6, 5, 6}, + {6, 1, 7}, {6, 3, 7}, {6, 5, 7}, + {7, 1, 0}, {7, 3, 0}, {7, 5, 0}, + {7, 1, 2}, {7, 3, 2}, {7, 5, 2}, + {7, 1, 4}, {7, 3, 4}, {7, 5, 4}, + {7, 1, 6}, {7, 3, 6}, {7, 5, 6}, + {7, 1, 7}, {7, 3, 7}, {7, 5, 7}}; + +static constexpr int c1d_z_8x8x8[75][3] = { {1, 0, 0}, {3, 0, 0}, {5, 0, 0}, // Z + {1, 0, 2}, {3, 0, 2}, {5, 0, 2}, + {1, 0, 4}, {3, 0, 4}, {5, 0, 4}, + {1, 0, 6}, {3, 0, 6}, {5, 0, 6}, + {1, 0, 7}, {3, 0, 7}, {5, 0, 7}, + {1, 2, 0}, {3, 2, 0}, {5, 2, 0}, + {1, 2, 2}, {3, 2, 2}, {5, 2, 2}, + {1, 2, 4}, {3, 2, 4}, {5, 2, 4}, + {1, 2, 6}, {3, 2, 6}, {5, 2, 6}, + {1, 2, 7}, {3, 2, 7}, {5, 2, 7}, + {1, 4, 0}, {3, 4, 0}, {5, 4, 0}, + {1, 4, 2}, {3, 4, 2}, {5, 4, 2}, + {1, 4, 4}, {3, 4, 4}, {5, 4, 4}, + {1, 4, 6}, {3, 4, 6}, {5, 4, 6}, + {1, 4, 7}, {3, 4, 7}, {5, 4, 7}, + {1, 6, 0}, {3, 6, 0}, {5, 6, 0}, + {1, 6, 2}, {3, 6, 2}, {5, 6, 2}, + {1, 6, 4}, {3, 6, 4}, {5, 6, 4}, + {1, 6, 6}, {3, 6, 6}, {5, 6, 6}, + {1, 6, 7}, {3, 6, 7}, {5, 6, 7}, + {1, 7, 0}, {3, 7, 0}, {5, 7, 0}, + {1, 7, 2}, {3, 7, 2}, {5, 7, 2}, + {1, 7, 4}, {3, 7, 4}, {5, 7, 4}, + {1, 7, 6}, {3, 7, 6}, {5, 7, 6}, + {1, 7, 7}, {3, 7, 7}, {5, 7, 7}}; + +static constexpr int c2d_xy_8x8x8[45][3] = {{0, 1, 1}, {0, 1, 3}, {0, 1, 5}, // XY + {0, 3, 1}, {0, 3, 3}, {0, 3, 5}, + {0, 5, 1}, {0, 5, 3}, {0, 5, 5}, + {2, 1, 1}, {2, 1, 3}, {2, 1, 5}, + {2, 3, 1}, {2, 3, 3}, {2, 3, 5}, + {2, 5, 1}, {2, 5, 3}, {2, 5, 5}, + {4, 1, 1}, {4, 1, 3}, {4, 1, 5}, + {4, 3, 1}, {4, 3, 3}, {4, 3, 5}, + {4, 5, 1}, {4, 5, 3}, {4, 5, 5}, + {6, 1, 1}, {6, 1, 3}, {6, 1, 5}, + {6, 3, 1}, {6, 3, 3}, {6, 3, 5}, + {6, 5, 1}, {6, 5, 3}, {6, 5, 5}, + {7, 1, 1}, {7, 1, 3}, {7, 1, 5}, + {7, 3, 1}, {7, 3, 3}, {7, 3, 5}, + {7, 5, 1}, {7, 5, 3}, {7, 5, 5}}; + +static constexpr int c2d_yz_8x8x8[45][3] = { + {1, 1, 0}, {1, 3, 0}, {1, 5, 0}, // YZ + {1, 1, 2}, {1, 3, 2}, {1, 5, 2}, + {1, 1, 4}, {1, 3, 4}, {1, 5, 4}, + {1, 1, 6}, {1, 3, 6}, {1, 5, 6}, + {1, 1, 7}, {1, 3, 7}, {1, 5, 7}, + {3, 1, 0}, {3, 3, 0}, {3, 5, 0}, + {3, 1, 2}, {3, 3, 2}, {3, 5, 2}, + {3, 1, 4}, {3, 3, 4}, {3, 5, 4}, + {3, 1, 6}, {3, 3, 6}, {3, 5, 6}, + {3, 1, 7}, {3, 3, 7}, {3, 5, 7}, + {5, 1, 0}, {5, 3, 0}, {5, 5, 0}, + {5, 1, 2}, {5, 3, 2}, {5, 5, 2}, + {5, 1, 4}, {5, 3, 4}, {5, 5, 4}, + {5, 1, 6}, {5, 3, 6}, {5, 5, 6}, + {5, 1, 7}, {5, 3, 7}, {5, 5, 7}}; + +static constexpr int c2d_xz_8x8x8[45][3] = { + {1, 0, 1}, {3, 0, 1}, {5, 0, 1}, // XZ + {1, 0, 3}, {3, 0, 3}, {5, 0, 3}, + {1, 0, 5}, {3, 0, 5}, {5, 0, 5}, + {1, 2, 1}, {3, 2, 1}, {5, 2, 1}, + {1, 2, 3}, {3, 2, 3}, {5, 2, 3}, + {1, 2, 5}, {3, 2, 5}, {5, 2, 5}, + {1, 4, 1}, {3, 4, 1}, {5, 4, 1}, + {1, 4, 3}, {3, 4, 3}, {5, 4, 3}, + {1, 4, 5}, {3, 4, 5}, {5, 4, 5}, + {1, 6, 1}, {3, 6, 1}, {5, 6, 1}, + {1, 6, 3}, {3, 6, 3}, {5, 6, 3}, + {1, 6, 5}, {3, 6, 5}, {5, 6, 5}, + {1, 7, 1}, {3, 7, 1}, {5, 7, 1}, + {1, 7, 3}, {3, 7, 3}, {5, 7, 3}, + {1, 7, 5}, {3, 7, 5}, {5, 7, 5}}; + +static constexpr int c3d_xyz_8x8x8[27][3] = { + {1, 1, 1}, + {1, 1, 3}, + {1, 1, 5}, + {1, 3, 1}, + {1, 3, 3}, + {1, 3, 5}, + {1, 5, 1}, + {1, 5, 3}, + {1, 5, 5}, + {3, 1, 1}, + {3, 1, 3}, + {3, 1, 5}, + {3, 3, 1}, + {3, 3, 3}, + {3, 3, 5}, + {3, 5, 1}, + {3, 5, 3}, + {3, 5, 5}, + {5, 1, 1}, + {5, 1, 3}, + {5, 1, 5}, + {5, 3, 1}, + {5, 3, 3}, + {5, 3, 5}, + {5, 5, 1}, + {5, 5, 3}, + {5, 5, 5}}; + +static constexpr int coarse_x_8x8x8[5] = {0, 2, 4, 6, 7}; + +static constexpr int coarse_y_8x8x8[5] = {0, 2, 4, 6, 7}; + +static constexpr int coarse_z_8x8x8[5] = {0, 2, 4, 6, 7}; + +MGARDX_EXEC int Coeff1D_M_Offset_8x8x8(SIZE i) { + static constexpr int offset[225] = {offset8x8x8(c1d_x_8x8x8[0][0], c1d_x_8x8x8[0][1], c1d_x_8x8x8[0][2]), // X + offset8x8x8(c1d_x_8x8x8[1][0], c1d_x_8x8x8[1][1], c1d_x_8x8x8[1][2]), + offset8x8x8(c1d_x_8x8x8[2][0], c1d_x_8x8x8[2][1], c1d_x_8x8x8[2][2]), + offset8x8x8(c1d_x_8x8x8[3][0], c1d_x_8x8x8[3][1], c1d_x_8x8x8[3][2]), + offset8x8x8(c1d_x_8x8x8[4][0], c1d_x_8x8x8[4][1], c1d_x_8x8x8[4][2]), + offset8x8x8(c1d_x_8x8x8[5][0], c1d_x_8x8x8[5][1], c1d_x_8x8x8[5][2]), + offset8x8x8(c1d_x_8x8x8[6][0], c1d_x_8x8x8[6][1], c1d_x_8x8x8[6][2]), + offset8x8x8(c1d_x_8x8x8[7][0], c1d_x_8x8x8[7][1], c1d_x_8x8x8[7][2]), + offset8x8x8(c1d_x_8x8x8[8][0], c1d_x_8x8x8[8][1], c1d_x_8x8x8[8][2]), + offset8x8x8(c1d_x_8x8x8[9][0], c1d_x_8x8x8[9][1], c1d_x_8x8x8[9][2]), + offset8x8x8(c1d_x_8x8x8[10][0], c1d_x_8x8x8[10][1], c1d_x_8x8x8[10][2]), + offset8x8x8(c1d_x_8x8x8[11][0], c1d_x_8x8x8[11][1], c1d_x_8x8x8[11][2]), + offset8x8x8(c1d_x_8x8x8[12][0], c1d_x_8x8x8[12][1], c1d_x_8x8x8[12][2]), + offset8x8x8(c1d_x_8x8x8[13][0], c1d_x_8x8x8[13][1], c1d_x_8x8x8[13][2]), + offset8x8x8(c1d_x_8x8x8[14][0], c1d_x_8x8x8[14][1], c1d_x_8x8x8[14][2]), + offset8x8x8(c1d_x_8x8x8[15][0], c1d_x_8x8x8[15][1], c1d_x_8x8x8[15][2]), + offset8x8x8(c1d_x_8x8x8[16][0], c1d_x_8x8x8[16][1], c1d_x_8x8x8[16][2]), + offset8x8x8(c1d_x_8x8x8[17][0], c1d_x_8x8x8[17][1], c1d_x_8x8x8[17][2]), + offset8x8x8(c1d_x_8x8x8[18][0], c1d_x_8x8x8[18][1], c1d_x_8x8x8[18][2]), + offset8x8x8(c1d_x_8x8x8[19][0], c1d_x_8x8x8[19][1], c1d_x_8x8x8[19][2]), + offset8x8x8(c1d_x_8x8x8[20][0], c1d_x_8x8x8[20][1], c1d_x_8x8x8[20][2]), + offset8x8x8(c1d_x_8x8x8[21][0], c1d_x_8x8x8[21][1], c1d_x_8x8x8[21][2]), + offset8x8x8(c1d_x_8x8x8[22][0], c1d_x_8x8x8[22][1], c1d_x_8x8x8[22][2]), + offset8x8x8(c1d_x_8x8x8[23][0], c1d_x_8x8x8[23][1], c1d_x_8x8x8[23][2]), + offset8x8x8(c1d_x_8x8x8[24][0], c1d_x_8x8x8[24][1], c1d_x_8x8x8[24][2]), + offset8x8x8(c1d_x_8x8x8[25][0], c1d_x_8x8x8[25][1], c1d_x_8x8x8[25][2]), + offset8x8x8(c1d_x_8x8x8[26][0], c1d_x_8x8x8[26][1], c1d_x_8x8x8[26][2]), + offset8x8x8(c1d_x_8x8x8[27][0], c1d_x_8x8x8[27][1], c1d_x_8x8x8[27][2]), + offset8x8x8(c1d_x_8x8x8[28][0], c1d_x_8x8x8[28][1], c1d_x_8x8x8[28][2]), + offset8x8x8(c1d_x_8x8x8[29][0], c1d_x_8x8x8[29][1], c1d_x_8x8x8[29][2]), + offset8x8x8(c1d_x_8x8x8[30][0], c1d_x_8x8x8[30][1], c1d_x_8x8x8[30][2]), + offset8x8x8(c1d_x_8x8x8[31][0], c1d_x_8x8x8[31][1], c1d_x_8x8x8[31][2]), + offset8x8x8(c1d_x_8x8x8[32][0], c1d_x_8x8x8[32][1], c1d_x_8x8x8[32][2]), + offset8x8x8(c1d_x_8x8x8[33][0], c1d_x_8x8x8[33][1], c1d_x_8x8x8[33][2]), + offset8x8x8(c1d_x_8x8x8[34][0], c1d_x_8x8x8[34][1], c1d_x_8x8x8[34][2]), + offset8x8x8(c1d_x_8x8x8[35][0], c1d_x_8x8x8[35][1], c1d_x_8x8x8[35][2]), + offset8x8x8(c1d_x_8x8x8[36][0], c1d_x_8x8x8[36][1], c1d_x_8x8x8[36][2]), + offset8x8x8(c1d_x_8x8x8[37][0], c1d_x_8x8x8[37][1], c1d_x_8x8x8[37][2]), + offset8x8x8(c1d_x_8x8x8[38][0], c1d_x_8x8x8[38][1], c1d_x_8x8x8[38][2]), + offset8x8x8(c1d_x_8x8x8[39][0], c1d_x_8x8x8[39][1], c1d_x_8x8x8[39][2]), + offset8x8x8(c1d_x_8x8x8[40][0], c1d_x_8x8x8[40][1], c1d_x_8x8x8[40][2]), + offset8x8x8(c1d_x_8x8x8[41][0], c1d_x_8x8x8[41][1], c1d_x_8x8x8[41][2]), + offset8x8x8(c1d_x_8x8x8[42][0], c1d_x_8x8x8[42][1], c1d_x_8x8x8[42][2]), + offset8x8x8(c1d_x_8x8x8[43][0], c1d_x_8x8x8[43][1], c1d_x_8x8x8[43][2]), + offset8x8x8(c1d_x_8x8x8[44][0], c1d_x_8x8x8[44][1], c1d_x_8x8x8[44][2]), + offset8x8x8(c1d_x_8x8x8[45][0], c1d_x_8x8x8[45][1], c1d_x_8x8x8[45][2]), + offset8x8x8(c1d_x_8x8x8[46][0], c1d_x_8x8x8[46][1], c1d_x_8x8x8[46][2]), + offset8x8x8(c1d_x_8x8x8[47][0], c1d_x_8x8x8[47][1], c1d_x_8x8x8[47][2]), + offset8x8x8(c1d_x_8x8x8[48][0], c1d_x_8x8x8[48][1], c1d_x_8x8x8[48][2]), + offset8x8x8(c1d_x_8x8x8[49][0], c1d_x_8x8x8[49][1], c1d_x_8x8x8[49][2]), + offset8x8x8(c1d_x_8x8x8[50][0], c1d_x_8x8x8[50][1], c1d_x_8x8x8[50][2]), + offset8x8x8(c1d_x_8x8x8[51][0], c1d_x_8x8x8[51][1], c1d_x_8x8x8[51][2]), + offset8x8x8(c1d_x_8x8x8[52][0], c1d_x_8x8x8[52][1], c1d_x_8x8x8[52][2]), + offset8x8x8(c1d_x_8x8x8[53][0], c1d_x_8x8x8[53][1], c1d_x_8x8x8[53][2]), + offset8x8x8(c1d_x_8x8x8[54][0], c1d_x_8x8x8[54][1], c1d_x_8x8x8[54][2]), + offset8x8x8(c1d_x_8x8x8[55][0], c1d_x_8x8x8[55][1], c1d_x_8x8x8[55][2]), + offset8x8x8(c1d_x_8x8x8[56][0], c1d_x_8x8x8[56][1], c1d_x_8x8x8[56][2]), + offset8x8x8(c1d_x_8x8x8[57][0], c1d_x_8x8x8[57][1], c1d_x_8x8x8[57][2]), + offset8x8x8(c1d_x_8x8x8[58][0], c1d_x_8x8x8[58][1], c1d_x_8x8x8[58][2]), + offset8x8x8(c1d_x_8x8x8[59][0], c1d_x_8x8x8[59][1], c1d_x_8x8x8[59][2]), + offset8x8x8(c1d_x_8x8x8[60][0], c1d_x_8x8x8[60][1], c1d_x_8x8x8[60][2]), + offset8x8x8(c1d_x_8x8x8[61][0], c1d_x_8x8x8[61][1], c1d_x_8x8x8[61][2]), + offset8x8x8(c1d_x_8x8x8[62][0], c1d_x_8x8x8[62][1], c1d_x_8x8x8[62][2]), + offset8x8x8(c1d_x_8x8x8[63][0], c1d_x_8x8x8[63][1], c1d_x_8x8x8[63][2]), + offset8x8x8(c1d_x_8x8x8[64][0], c1d_x_8x8x8[64][1], c1d_x_8x8x8[64][2]), + offset8x8x8(c1d_x_8x8x8[65][0], c1d_x_8x8x8[65][1], c1d_x_8x8x8[65][2]), + offset8x8x8(c1d_x_8x8x8[66][0], c1d_x_8x8x8[66][1], c1d_x_8x8x8[66][2]), + offset8x8x8(c1d_x_8x8x8[67][0], c1d_x_8x8x8[67][1], c1d_x_8x8x8[67][2]), + offset8x8x8(c1d_x_8x8x8[68][0], c1d_x_8x8x8[68][1], c1d_x_8x8x8[68][2]), + offset8x8x8(c1d_x_8x8x8[69][0], c1d_x_8x8x8[69][1], c1d_x_8x8x8[69][2]), + offset8x8x8(c1d_x_8x8x8[70][0], c1d_x_8x8x8[70][1], c1d_x_8x8x8[70][2]), + offset8x8x8(c1d_x_8x8x8[71][0], c1d_x_8x8x8[71][1], c1d_x_8x8x8[71][2]), + offset8x8x8(c1d_x_8x8x8[72][0], c1d_x_8x8x8[72][1], c1d_x_8x8x8[72][2]), + offset8x8x8(c1d_x_8x8x8[73][0], c1d_x_8x8x8[73][1], c1d_x_8x8x8[73][2]), + offset8x8x8(c1d_x_8x8x8[74][0], c1d_x_8x8x8[74][1], c1d_x_8x8x8[74][2]), + + offset8x8x8(c1d_y_8x8x8[0][0], c1d_y_8x8x8[0][1], c1d_y_8x8x8[0][2]), // Y + offset8x8x8(c1d_y_8x8x8[1][0], c1d_y_8x8x8[1][1], c1d_y_8x8x8[1][2]), + offset8x8x8(c1d_y_8x8x8[2][0], c1d_y_8x8x8[2][1], c1d_y_8x8x8[2][2]), + offset8x8x8(c1d_y_8x8x8[3][0], c1d_y_8x8x8[3][1], c1d_y_8x8x8[3][2]), + offset8x8x8(c1d_y_8x8x8[4][0], c1d_y_8x8x8[4][1], c1d_y_8x8x8[4][2]), + offset8x8x8(c1d_y_8x8x8[5][0], c1d_y_8x8x8[5][1], c1d_y_8x8x8[5][2]), + offset8x8x8(c1d_y_8x8x8[6][0], c1d_y_8x8x8[6][1], c1d_y_8x8x8[6][2]), + offset8x8x8(c1d_y_8x8x8[7][0], c1d_y_8x8x8[7][1], c1d_y_8x8x8[7][2]), + offset8x8x8(c1d_y_8x8x8[8][0], c1d_y_8x8x8[8][1], c1d_y_8x8x8[8][2]), + offset8x8x8(c1d_y_8x8x8[9][0], c1d_y_8x8x8[9][1], c1d_y_8x8x8[9][2]), + offset8x8x8(c1d_y_8x8x8[10][0], c1d_y_8x8x8[10][1], c1d_y_8x8x8[10][2]), + offset8x8x8(c1d_y_8x8x8[11][0], c1d_y_8x8x8[11][1], c1d_y_8x8x8[11][2]), + offset8x8x8(c1d_y_8x8x8[12][0], c1d_y_8x8x8[12][1], c1d_y_8x8x8[12][2]), + offset8x8x8(c1d_y_8x8x8[13][0], c1d_y_8x8x8[13][1], c1d_y_8x8x8[13][2]), + offset8x8x8(c1d_y_8x8x8[14][0], c1d_y_8x8x8[14][1], c1d_y_8x8x8[14][2]), + offset8x8x8(c1d_y_8x8x8[15][0], c1d_y_8x8x8[15][1], c1d_y_8x8x8[15][2]), + offset8x8x8(c1d_y_8x8x8[16][0], c1d_y_8x8x8[16][1], c1d_y_8x8x8[16][2]), + offset8x8x8(c1d_y_8x8x8[17][0], c1d_y_8x8x8[17][1], c1d_y_8x8x8[17][2]), + offset8x8x8(c1d_y_8x8x8[18][0], c1d_y_8x8x8[18][1], c1d_y_8x8x8[18][2]), + offset8x8x8(c1d_y_8x8x8[19][0], c1d_y_8x8x8[19][1], c1d_y_8x8x8[19][2]), + offset8x8x8(c1d_y_8x8x8[20][0], c1d_y_8x8x8[20][1], c1d_y_8x8x8[20][2]), + offset8x8x8(c1d_y_8x8x8[21][0], c1d_y_8x8x8[21][1], c1d_y_8x8x8[21][2]), + offset8x8x8(c1d_y_8x8x8[22][0], c1d_y_8x8x8[22][1], c1d_y_8x8x8[22][2]), + offset8x8x8(c1d_y_8x8x8[23][0], c1d_y_8x8x8[23][1], c1d_y_8x8x8[23][2]), + offset8x8x8(c1d_y_8x8x8[24][0], c1d_y_8x8x8[24][1], c1d_y_8x8x8[24][2]), + offset8x8x8(c1d_y_8x8x8[25][0], c1d_y_8x8x8[25][1], c1d_y_8x8x8[25][2]), + offset8x8x8(c1d_y_8x8x8[26][0], c1d_y_8x8x8[26][1], c1d_y_8x8x8[26][2]), + offset8x8x8(c1d_y_8x8x8[27][0], c1d_y_8x8x8[27][1], c1d_y_8x8x8[27][2]), + offset8x8x8(c1d_y_8x8x8[28][0], c1d_y_8x8x8[28][1], c1d_y_8x8x8[28][2]), + offset8x8x8(c1d_y_8x8x8[29][0], c1d_y_8x8x8[29][1], c1d_y_8x8x8[29][2]), + offset8x8x8(c1d_y_8x8x8[30][0], c1d_y_8x8x8[30][1], c1d_y_8x8x8[30][2]), + offset8x8x8(c1d_y_8x8x8[31][0], c1d_y_8x8x8[31][1], c1d_y_8x8x8[31][2]), + offset8x8x8(c1d_y_8x8x8[32][0], c1d_y_8x8x8[32][1], c1d_y_8x8x8[32][2]), + offset8x8x8(c1d_y_8x8x8[33][0], c1d_y_8x8x8[33][1], c1d_y_8x8x8[33][2]), + offset8x8x8(c1d_y_8x8x8[34][0], c1d_y_8x8x8[34][1], c1d_y_8x8x8[34][2]), + offset8x8x8(c1d_y_8x8x8[35][0], c1d_y_8x8x8[35][1], c1d_y_8x8x8[35][2]), + offset8x8x8(c1d_y_8x8x8[36][0], c1d_y_8x8x8[36][1], c1d_y_8x8x8[36][2]), + offset8x8x8(c1d_y_8x8x8[37][0], c1d_y_8x8x8[37][1], c1d_y_8x8x8[37][2]), + offset8x8x8(c1d_y_8x8x8[38][0], c1d_y_8x8x8[38][1], c1d_y_8x8x8[38][2]), + offset8x8x8(c1d_y_8x8x8[39][0], c1d_y_8x8x8[39][1], c1d_y_8x8x8[39][2]), + offset8x8x8(c1d_y_8x8x8[40][0], c1d_y_8x8x8[40][1], c1d_y_8x8x8[40][2]), + offset8x8x8(c1d_y_8x8x8[41][0], c1d_y_8x8x8[41][1], c1d_y_8x8x8[41][2]), + offset8x8x8(c1d_y_8x8x8[42][0], c1d_y_8x8x8[42][1], c1d_y_8x8x8[42][2]), + offset8x8x8(c1d_y_8x8x8[43][0], c1d_y_8x8x8[43][1], c1d_y_8x8x8[43][2]), + offset8x8x8(c1d_y_8x8x8[44][0], c1d_y_8x8x8[44][1], c1d_y_8x8x8[44][2]), + offset8x8x8(c1d_y_8x8x8[45][0], c1d_y_8x8x8[45][1], c1d_y_8x8x8[45][2]), + offset8x8x8(c1d_y_8x8x8[46][0], c1d_y_8x8x8[46][1], c1d_y_8x8x8[46][2]), + offset8x8x8(c1d_y_8x8x8[47][0], c1d_y_8x8x8[47][1], c1d_y_8x8x8[47][2]), + offset8x8x8(c1d_y_8x8x8[48][0], c1d_y_8x8x8[48][1], c1d_y_8x8x8[48][2]), + offset8x8x8(c1d_y_8x8x8[49][0], c1d_y_8x8x8[49][1], c1d_y_8x8x8[49][2]), + offset8x8x8(c1d_y_8x8x8[50][0], c1d_y_8x8x8[50][1], c1d_y_8x8x8[50][2]), + offset8x8x8(c1d_y_8x8x8[51][0], c1d_y_8x8x8[51][1], c1d_y_8x8x8[51][2]), + offset8x8x8(c1d_y_8x8x8[52][0], c1d_y_8x8x8[52][1], c1d_y_8x8x8[52][2]), + offset8x8x8(c1d_y_8x8x8[53][0], c1d_y_8x8x8[53][1], c1d_y_8x8x8[53][2]), + offset8x8x8(c1d_y_8x8x8[54][0], c1d_y_8x8x8[54][1], c1d_y_8x8x8[54][2]), + offset8x8x8(c1d_y_8x8x8[55][0], c1d_y_8x8x8[55][1], c1d_y_8x8x8[55][2]), + offset8x8x8(c1d_y_8x8x8[56][0], c1d_y_8x8x8[56][1], c1d_y_8x8x8[56][2]), + offset8x8x8(c1d_y_8x8x8[57][0], c1d_y_8x8x8[57][1], c1d_y_8x8x8[57][2]), + offset8x8x8(c1d_y_8x8x8[58][0], c1d_y_8x8x8[58][1], c1d_y_8x8x8[58][2]), + offset8x8x8(c1d_y_8x8x8[59][0], c1d_y_8x8x8[59][1], c1d_y_8x8x8[59][2]), + offset8x8x8(c1d_y_8x8x8[60][0], c1d_y_8x8x8[60][1], c1d_y_8x8x8[60][2]), + offset8x8x8(c1d_y_8x8x8[61][0], c1d_y_8x8x8[61][1], c1d_y_8x8x8[61][2]), + offset8x8x8(c1d_y_8x8x8[62][0], c1d_y_8x8x8[62][1], c1d_y_8x8x8[62][2]), + offset8x8x8(c1d_y_8x8x8[63][0], c1d_y_8x8x8[63][1], c1d_y_8x8x8[63][2]), + offset8x8x8(c1d_y_8x8x8[64][0], c1d_y_8x8x8[64][1], c1d_y_8x8x8[64][2]), + offset8x8x8(c1d_y_8x8x8[65][0], c1d_y_8x8x8[65][1], c1d_y_8x8x8[65][2]), + offset8x8x8(c1d_y_8x8x8[66][0], c1d_y_8x8x8[66][1], c1d_y_8x8x8[66][2]), + offset8x8x8(c1d_y_8x8x8[67][0], c1d_y_8x8x8[67][1], c1d_y_8x8x8[67][2]), + offset8x8x8(c1d_y_8x8x8[68][0], c1d_y_8x8x8[68][1], c1d_y_8x8x8[68][2]), + offset8x8x8(c1d_y_8x8x8[69][0], c1d_y_8x8x8[69][1], c1d_y_8x8x8[69][2]), + offset8x8x8(c1d_y_8x8x8[70][0], c1d_y_8x8x8[70][1], c1d_y_8x8x8[70][2]), + offset8x8x8(c1d_y_8x8x8[71][0], c1d_y_8x8x8[71][1], c1d_y_8x8x8[71][2]), + offset8x8x8(c1d_y_8x8x8[72][0], c1d_y_8x8x8[72][1], c1d_y_8x8x8[72][2]), + offset8x8x8(c1d_y_8x8x8[73][0], c1d_y_8x8x8[73][1], c1d_y_8x8x8[73][2]), + offset8x8x8(c1d_y_8x8x8[74][0], c1d_y_8x8x8[74][1], c1d_y_8x8x8[74][2]), + + offset8x8x8(c1d_z_8x8x8[0][0], c1d_z_8x8x8[0][1], c1d_z_8x8x8[0][2]), // Z + offset8x8x8(c1d_z_8x8x8[1][0], c1d_z_8x8x8[1][1], c1d_z_8x8x8[1][2]), + offset8x8x8(c1d_z_8x8x8[2][0], c1d_z_8x8x8[2][1], c1d_z_8x8x8[2][2]), + offset8x8x8(c1d_z_8x8x8[3][0], c1d_z_8x8x8[3][1], c1d_z_8x8x8[3][2]), + offset8x8x8(c1d_z_8x8x8[4][0], c1d_z_8x8x8[4][1], c1d_z_8x8x8[4][2]), + offset8x8x8(c1d_z_8x8x8[5][0], c1d_z_8x8x8[5][1], c1d_z_8x8x8[5][2]), + offset8x8x8(c1d_z_8x8x8[6][0], c1d_z_8x8x8[6][1], c1d_z_8x8x8[6][2]), + offset8x8x8(c1d_z_8x8x8[7][0], c1d_z_8x8x8[7][1], c1d_z_8x8x8[7][2]), + offset8x8x8(c1d_z_8x8x8[8][0], c1d_z_8x8x8[8][1], c1d_z_8x8x8[8][2]), + offset8x8x8(c1d_z_8x8x8[9][0], c1d_z_8x8x8[9][1], c1d_z_8x8x8[9][2]), + offset8x8x8(c1d_z_8x8x8[10][0], c1d_z_8x8x8[10][1], c1d_z_8x8x8[10][2]), + offset8x8x8(c1d_z_8x8x8[11][0], c1d_z_8x8x8[11][1], c1d_z_8x8x8[11][2]), + offset8x8x8(c1d_z_8x8x8[12][0], c1d_z_8x8x8[12][1], c1d_z_8x8x8[12][2]), + offset8x8x8(c1d_z_8x8x8[13][0], c1d_z_8x8x8[13][1], c1d_z_8x8x8[13][2]), + offset8x8x8(c1d_z_8x8x8[14][0], c1d_z_8x8x8[14][1], c1d_z_8x8x8[14][2]), + offset8x8x8(c1d_z_8x8x8[15][0], c1d_z_8x8x8[15][1], c1d_z_8x8x8[15][2]), + offset8x8x8(c1d_z_8x8x8[16][0], c1d_z_8x8x8[16][1], c1d_z_8x8x8[16][2]), + offset8x8x8(c1d_z_8x8x8[17][0], c1d_z_8x8x8[17][1], c1d_z_8x8x8[17][2]), + offset8x8x8(c1d_z_8x8x8[18][0], c1d_z_8x8x8[18][1], c1d_z_8x8x8[18][2]), + offset8x8x8(c1d_z_8x8x8[19][0], c1d_z_8x8x8[19][1], c1d_z_8x8x8[19][2]), + offset8x8x8(c1d_z_8x8x8[20][0], c1d_z_8x8x8[20][1], c1d_z_8x8x8[20][2]), + offset8x8x8(c1d_z_8x8x8[21][0], c1d_z_8x8x8[21][1], c1d_z_8x8x8[21][2]), + offset8x8x8(c1d_z_8x8x8[22][0], c1d_z_8x8x8[22][1], c1d_z_8x8x8[22][2]), + offset8x8x8(c1d_z_8x8x8[23][0], c1d_z_8x8x8[23][1], c1d_z_8x8x8[23][2]), + offset8x8x8(c1d_z_8x8x8[24][0], c1d_z_8x8x8[24][1], c1d_z_8x8x8[24][2]), + offset8x8x8(c1d_z_8x8x8[25][0], c1d_z_8x8x8[25][1], c1d_z_8x8x8[25][2]), + offset8x8x8(c1d_z_8x8x8[26][0], c1d_z_8x8x8[26][1], c1d_z_8x8x8[26][2]), + offset8x8x8(c1d_z_8x8x8[27][0], c1d_z_8x8x8[27][1], c1d_z_8x8x8[27][2]), + offset8x8x8(c1d_z_8x8x8[28][0], c1d_z_8x8x8[28][1], c1d_z_8x8x8[28][2]), + offset8x8x8(c1d_z_8x8x8[29][0], c1d_z_8x8x8[29][1], c1d_z_8x8x8[29][2]), + offset8x8x8(c1d_z_8x8x8[30][0], c1d_z_8x8x8[30][1], c1d_z_8x8x8[30][2]), + offset8x8x8(c1d_z_8x8x8[31][0], c1d_z_8x8x8[31][1], c1d_z_8x8x8[31][2]), + offset8x8x8(c1d_z_8x8x8[32][0], c1d_z_8x8x8[32][1], c1d_z_8x8x8[32][2]), + offset8x8x8(c1d_z_8x8x8[33][0], c1d_z_8x8x8[33][1], c1d_z_8x8x8[33][2]), + offset8x8x8(c1d_z_8x8x8[34][0], c1d_z_8x8x8[34][1], c1d_z_8x8x8[34][2]), + offset8x8x8(c1d_z_8x8x8[35][0], c1d_z_8x8x8[35][1], c1d_z_8x8x8[35][2]), + offset8x8x8(c1d_z_8x8x8[36][0], c1d_z_8x8x8[36][1], c1d_z_8x8x8[36][2]), + offset8x8x8(c1d_z_8x8x8[37][0], c1d_z_8x8x8[37][1], c1d_z_8x8x8[37][2]), + offset8x8x8(c1d_z_8x8x8[38][0], c1d_z_8x8x8[38][1], c1d_z_8x8x8[38][2]), + offset8x8x8(c1d_z_8x8x8[39][0], c1d_z_8x8x8[39][1], c1d_z_8x8x8[39][2]), + offset8x8x8(c1d_z_8x8x8[40][0], c1d_z_8x8x8[40][1], c1d_z_8x8x8[40][2]), + offset8x8x8(c1d_z_8x8x8[41][0], c1d_z_8x8x8[41][1], c1d_z_8x8x8[41][2]), + offset8x8x8(c1d_z_8x8x8[42][0], c1d_z_8x8x8[42][1], c1d_z_8x8x8[42][2]), + offset8x8x8(c1d_z_8x8x8[43][0], c1d_z_8x8x8[43][1], c1d_z_8x8x8[43][2]), + offset8x8x8(c1d_z_8x8x8[44][0], c1d_z_8x8x8[44][1], c1d_z_8x8x8[44][2]), + offset8x8x8(c1d_z_8x8x8[45][0], c1d_z_8x8x8[45][1], c1d_z_8x8x8[45][2]), + offset8x8x8(c1d_z_8x8x8[46][0], c1d_z_8x8x8[46][1], c1d_z_8x8x8[46][2]), + offset8x8x8(c1d_z_8x8x8[47][0], c1d_z_8x8x8[47][1], c1d_z_8x8x8[47][2]), + offset8x8x8(c1d_z_8x8x8[48][0], c1d_z_8x8x8[48][1], c1d_z_8x8x8[48][2]), + offset8x8x8(c1d_z_8x8x8[49][0], c1d_z_8x8x8[49][1], c1d_z_8x8x8[49][2]), + offset8x8x8(c1d_z_8x8x8[50][0], c1d_z_8x8x8[50][1], c1d_z_8x8x8[50][2]), + offset8x8x8(c1d_z_8x8x8[51][0], c1d_z_8x8x8[51][1], c1d_z_8x8x8[51][2]), + offset8x8x8(c1d_z_8x8x8[52][0], c1d_z_8x8x8[52][1], c1d_z_8x8x8[52][2]), + offset8x8x8(c1d_z_8x8x8[53][0], c1d_z_8x8x8[53][1], c1d_z_8x8x8[53][2]), + offset8x8x8(c1d_z_8x8x8[54][0], c1d_z_8x8x8[54][1], c1d_z_8x8x8[54][2]), + offset8x8x8(c1d_z_8x8x8[55][0], c1d_z_8x8x8[55][1], c1d_z_8x8x8[55][2]), + offset8x8x8(c1d_z_8x8x8[56][0], c1d_z_8x8x8[56][1], c1d_z_8x8x8[56][2]), + offset8x8x8(c1d_z_8x8x8[57][0], c1d_z_8x8x8[57][1], c1d_z_8x8x8[57][2]), + offset8x8x8(c1d_z_8x8x8[58][0], c1d_z_8x8x8[58][1], c1d_z_8x8x8[58][2]), + offset8x8x8(c1d_z_8x8x8[59][0], c1d_z_8x8x8[59][1], c1d_z_8x8x8[59][2]), + offset8x8x8(c1d_z_8x8x8[60][0], c1d_z_8x8x8[60][1], c1d_z_8x8x8[60][2]), + offset8x8x8(c1d_z_8x8x8[61][0], c1d_z_8x8x8[61][1], c1d_z_8x8x8[61][2]), + offset8x8x8(c1d_z_8x8x8[62][0], c1d_z_8x8x8[62][1], c1d_z_8x8x8[62][2]), + offset8x8x8(c1d_z_8x8x8[63][0], c1d_z_8x8x8[63][1], c1d_z_8x8x8[63][2]), + offset8x8x8(c1d_z_8x8x8[64][0], c1d_z_8x8x8[64][1], c1d_z_8x8x8[64][2]), + offset8x8x8(c1d_z_8x8x8[65][0], c1d_z_8x8x8[65][1], c1d_z_8x8x8[65][2]), + offset8x8x8(c1d_z_8x8x8[66][0], c1d_z_8x8x8[66][1], c1d_z_8x8x8[66][2]), + offset8x8x8(c1d_z_8x8x8[67][0], c1d_z_8x8x8[67][1], c1d_z_8x8x8[67][2]), + offset8x8x8(c1d_z_8x8x8[68][0], c1d_z_8x8x8[68][1], c1d_z_8x8x8[68][2]), + offset8x8x8(c1d_z_8x8x8[69][0], c1d_z_8x8x8[69][1], c1d_z_8x8x8[69][2]), + offset8x8x8(c1d_z_8x8x8[70][0], c1d_z_8x8x8[70][1], c1d_z_8x8x8[70][2]), + offset8x8x8(c1d_z_8x8x8[71][0], c1d_z_8x8x8[71][1], c1d_z_8x8x8[71][2]), + offset8x8x8(c1d_z_8x8x8[72][0], c1d_z_8x8x8[72][1], c1d_z_8x8x8[72][2]), + offset8x8x8(c1d_z_8x8x8[73][0], c1d_z_8x8x8[73][1], c1d_z_8x8x8[73][2]), + offset8x8x8(c1d_z_8x8x8[74][0], c1d_z_8x8x8[74][1], c1d_z_8x8x8[74][2])}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff1D_L_Offset_8x8x8(SIZE i) { + static constexpr int offset[225] = {offset8x8x8(c1d_x_8x8x8[0][0], c1d_x_8x8x8[0][1], c1d_x_8x8x8[0][2]-1), // X + offset8x8x8(c1d_x_8x8x8[1][0], c1d_x_8x8x8[1][1], c1d_x_8x8x8[1][2]-1), + offset8x8x8(c1d_x_8x8x8[2][0], c1d_x_8x8x8[2][1], c1d_x_8x8x8[2][2]-1), + offset8x8x8(c1d_x_8x8x8[3][0], c1d_x_8x8x8[3][1], c1d_x_8x8x8[3][2]-1), + offset8x8x8(c1d_x_8x8x8[4][0], c1d_x_8x8x8[4][1], c1d_x_8x8x8[4][2]-1), + offset8x8x8(c1d_x_8x8x8[5][0], c1d_x_8x8x8[5][1], c1d_x_8x8x8[5][2]-1), + offset8x8x8(c1d_x_8x8x8[6][0], c1d_x_8x8x8[6][1], c1d_x_8x8x8[6][2]-1), + offset8x8x8(c1d_x_8x8x8[7][0], c1d_x_8x8x8[7][1], c1d_x_8x8x8[7][2]-1), + offset8x8x8(c1d_x_8x8x8[8][0], c1d_x_8x8x8[8][1], c1d_x_8x8x8[8][2]-1), + offset8x8x8(c1d_x_8x8x8[9][0], c1d_x_8x8x8[9][1], c1d_x_8x8x8[9][2]-1), + offset8x8x8(c1d_x_8x8x8[10][0], c1d_x_8x8x8[10][1], c1d_x_8x8x8[10][2]-1), + offset8x8x8(c1d_x_8x8x8[11][0], c1d_x_8x8x8[11][1], c1d_x_8x8x8[11][2]-1), + offset8x8x8(c1d_x_8x8x8[12][0], c1d_x_8x8x8[12][1], c1d_x_8x8x8[12][2]-1), + offset8x8x8(c1d_x_8x8x8[13][0], c1d_x_8x8x8[13][1], c1d_x_8x8x8[13][2]-1), + offset8x8x8(c1d_x_8x8x8[14][0], c1d_x_8x8x8[14][1], c1d_x_8x8x8[14][2]-1), + offset8x8x8(c1d_x_8x8x8[15][0], c1d_x_8x8x8[15][1], c1d_x_8x8x8[15][2]-1), + offset8x8x8(c1d_x_8x8x8[16][0], c1d_x_8x8x8[16][1], c1d_x_8x8x8[16][2]-1), + offset8x8x8(c1d_x_8x8x8[17][0], c1d_x_8x8x8[17][1], c1d_x_8x8x8[17][2]-1), + offset8x8x8(c1d_x_8x8x8[18][0], c1d_x_8x8x8[18][1], c1d_x_8x8x8[18][2]-1), + offset8x8x8(c1d_x_8x8x8[19][0], c1d_x_8x8x8[19][1], c1d_x_8x8x8[19][2]-1), + offset8x8x8(c1d_x_8x8x8[20][0], c1d_x_8x8x8[20][1], c1d_x_8x8x8[20][2]-1), + offset8x8x8(c1d_x_8x8x8[21][0], c1d_x_8x8x8[21][1], c1d_x_8x8x8[21][2]-1), + offset8x8x8(c1d_x_8x8x8[22][0], c1d_x_8x8x8[22][1], c1d_x_8x8x8[22][2]-1), + offset8x8x8(c1d_x_8x8x8[23][0], c1d_x_8x8x8[23][1], c1d_x_8x8x8[23][2]-1), + offset8x8x8(c1d_x_8x8x8[24][0], c1d_x_8x8x8[24][1], c1d_x_8x8x8[24][2]-1), + offset8x8x8(c1d_x_8x8x8[25][0], c1d_x_8x8x8[25][1], c1d_x_8x8x8[25][2]-1), + offset8x8x8(c1d_x_8x8x8[26][0], c1d_x_8x8x8[26][1], c1d_x_8x8x8[26][2]-1), + offset8x8x8(c1d_x_8x8x8[27][0], c1d_x_8x8x8[27][1], c1d_x_8x8x8[27][2]-1), + offset8x8x8(c1d_x_8x8x8[28][0], c1d_x_8x8x8[28][1], c1d_x_8x8x8[28][2]-1), + offset8x8x8(c1d_x_8x8x8[29][0], c1d_x_8x8x8[29][1], c1d_x_8x8x8[29][2]-1), + offset8x8x8(c1d_x_8x8x8[30][0], c1d_x_8x8x8[30][1], c1d_x_8x8x8[30][2]-1), + offset8x8x8(c1d_x_8x8x8[31][0], c1d_x_8x8x8[31][1], c1d_x_8x8x8[31][2]-1), + offset8x8x8(c1d_x_8x8x8[32][0], c1d_x_8x8x8[32][1], c1d_x_8x8x8[32][2]-1), + offset8x8x8(c1d_x_8x8x8[33][0], c1d_x_8x8x8[33][1], c1d_x_8x8x8[33][2]-1), + offset8x8x8(c1d_x_8x8x8[34][0], c1d_x_8x8x8[34][1], c1d_x_8x8x8[34][2]-1), + offset8x8x8(c1d_x_8x8x8[35][0], c1d_x_8x8x8[35][1], c1d_x_8x8x8[35][2]-1), + offset8x8x8(c1d_x_8x8x8[36][0], c1d_x_8x8x8[36][1], c1d_x_8x8x8[36][2]-1), + offset8x8x8(c1d_x_8x8x8[37][0], c1d_x_8x8x8[37][1], c1d_x_8x8x8[37][2]-1), + offset8x8x8(c1d_x_8x8x8[38][0], c1d_x_8x8x8[38][1], c1d_x_8x8x8[38][2]-1), + offset8x8x8(c1d_x_8x8x8[39][0], c1d_x_8x8x8[39][1], c1d_x_8x8x8[39][2]-1), + offset8x8x8(c1d_x_8x8x8[40][0], c1d_x_8x8x8[40][1], c1d_x_8x8x8[40][2]-1), + offset8x8x8(c1d_x_8x8x8[41][0], c1d_x_8x8x8[41][1], c1d_x_8x8x8[41][2]-1), + offset8x8x8(c1d_x_8x8x8[42][0], c1d_x_8x8x8[42][1], c1d_x_8x8x8[42][2]-1), + offset8x8x8(c1d_x_8x8x8[43][0], c1d_x_8x8x8[43][1], c1d_x_8x8x8[43][2]-1), + offset8x8x8(c1d_x_8x8x8[44][0], c1d_x_8x8x8[44][1], c1d_x_8x8x8[44][2]-1), + offset8x8x8(c1d_x_8x8x8[45][0], c1d_x_8x8x8[45][1], c1d_x_8x8x8[45][2]-1), + offset8x8x8(c1d_x_8x8x8[46][0], c1d_x_8x8x8[46][1], c1d_x_8x8x8[46][2]-1), + offset8x8x8(c1d_x_8x8x8[47][0], c1d_x_8x8x8[47][1], c1d_x_8x8x8[47][2]-1), + offset8x8x8(c1d_x_8x8x8[48][0], c1d_x_8x8x8[48][1], c1d_x_8x8x8[48][2]-1), + offset8x8x8(c1d_x_8x8x8[49][0], c1d_x_8x8x8[49][1], c1d_x_8x8x8[49][2]-1), + offset8x8x8(c1d_x_8x8x8[50][0], c1d_x_8x8x8[50][1], c1d_x_8x8x8[50][2]-1), + offset8x8x8(c1d_x_8x8x8[51][0], c1d_x_8x8x8[51][1], c1d_x_8x8x8[51][2]-1), + offset8x8x8(c1d_x_8x8x8[52][0], c1d_x_8x8x8[52][1], c1d_x_8x8x8[52][2]-1), + offset8x8x8(c1d_x_8x8x8[53][0], c1d_x_8x8x8[53][1], c1d_x_8x8x8[53][2]-1), + offset8x8x8(c1d_x_8x8x8[54][0], c1d_x_8x8x8[54][1], c1d_x_8x8x8[54][2]-1), + offset8x8x8(c1d_x_8x8x8[55][0], c1d_x_8x8x8[55][1], c1d_x_8x8x8[55][2]-1), + offset8x8x8(c1d_x_8x8x8[56][0], c1d_x_8x8x8[56][1], c1d_x_8x8x8[56][2]-1), + offset8x8x8(c1d_x_8x8x8[57][0], c1d_x_8x8x8[57][1], c1d_x_8x8x8[57][2]-1), + offset8x8x8(c1d_x_8x8x8[58][0], c1d_x_8x8x8[58][1], c1d_x_8x8x8[58][2]-1), + offset8x8x8(c1d_x_8x8x8[59][0], c1d_x_8x8x8[59][1], c1d_x_8x8x8[59][2]-1), + offset8x8x8(c1d_x_8x8x8[60][0], c1d_x_8x8x8[60][1], c1d_x_8x8x8[60][2]-1), + offset8x8x8(c1d_x_8x8x8[61][0], c1d_x_8x8x8[61][1], c1d_x_8x8x8[61][2]-1), + offset8x8x8(c1d_x_8x8x8[62][0], c1d_x_8x8x8[62][1], c1d_x_8x8x8[62][2]-1), + offset8x8x8(c1d_x_8x8x8[63][0], c1d_x_8x8x8[63][1], c1d_x_8x8x8[63][2]-1), + offset8x8x8(c1d_x_8x8x8[64][0], c1d_x_8x8x8[64][1], c1d_x_8x8x8[64][2]-1), + offset8x8x8(c1d_x_8x8x8[65][0], c1d_x_8x8x8[65][1], c1d_x_8x8x8[65][2]-1), + offset8x8x8(c1d_x_8x8x8[66][0], c1d_x_8x8x8[66][1], c1d_x_8x8x8[66][2]-1), + offset8x8x8(c1d_x_8x8x8[67][0], c1d_x_8x8x8[67][1], c1d_x_8x8x8[67][2]-1), + offset8x8x8(c1d_x_8x8x8[68][0], c1d_x_8x8x8[68][1], c1d_x_8x8x8[68][2]-1), + offset8x8x8(c1d_x_8x8x8[69][0], c1d_x_8x8x8[69][1], c1d_x_8x8x8[69][2]-1), + offset8x8x8(c1d_x_8x8x8[70][0], c1d_x_8x8x8[70][1], c1d_x_8x8x8[70][2]-1), + offset8x8x8(c1d_x_8x8x8[71][0], c1d_x_8x8x8[71][1], c1d_x_8x8x8[71][2]-1), + offset8x8x8(c1d_x_8x8x8[72][0], c1d_x_8x8x8[72][1], c1d_x_8x8x8[72][2]-1), + offset8x8x8(c1d_x_8x8x8[73][0], c1d_x_8x8x8[73][1], c1d_x_8x8x8[73][2]-1), + offset8x8x8(c1d_x_8x8x8[74][0], c1d_x_8x8x8[74][1], c1d_x_8x8x8[74][2]-1), + + offset8x8x8(c1d_y_8x8x8[0][0], c1d_y_8x8x8[0][1]-1, c1d_y_8x8x8[0][2]), // Y + offset8x8x8(c1d_y_8x8x8[1][0], c1d_y_8x8x8[1][1]-1, c1d_y_8x8x8[1][2]), + offset8x8x8(c1d_y_8x8x8[2][0], c1d_y_8x8x8[2][1]-1, c1d_y_8x8x8[2][2]), + offset8x8x8(c1d_y_8x8x8[3][0], c1d_y_8x8x8[3][1]-1, c1d_y_8x8x8[3][2]), + offset8x8x8(c1d_y_8x8x8[4][0], c1d_y_8x8x8[4][1]-1, c1d_y_8x8x8[4][2]), + offset8x8x8(c1d_y_8x8x8[5][0], c1d_y_8x8x8[5][1]-1, c1d_y_8x8x8[5][2]), + offset8x8x8(c1d_y_8x8x8[6][0], c1d_y_8x8x8[6][1]-1, c1d_y_8x8x8[6][2]), + offset8x8x8(c1d_y_8x8x8[7][0], c1d_y_8x8x8[7][1]-1, c1d_y_8x8x8[7][2]), + offset8x8x8(c1d_y_8x8x8[8][0], c1d_y_8x8x8[8][1]-1, c1d_y_8x8x8[8][2]), + offset8x8x8(c1d_y_8x8x8[9][0], c1d_y_8x8x8[9][1]-1, c1d_y_8x8x8[9][2]), + offset8x8x8(c1d_y_8x8x8[10][0], c1d_y_8x8x8[10][1]-1, c1d_y_8x8x8[10][2]), + offset8x8x8(c1d_y_8x8x8[11][0], c1d_y_8x8x8[11][1]-1, c1d_y_8x8x8[11][2]), + offset8x8x8(c1d_y_8x8x8[12][0], c1d_y_8x8x8[12][1]-1, c1d_y_8x8x8[12][2]), + offset8x8x8(c1d_y_8x8x8[13][0], c1d_y_8x8x8[13][1]-1, c1d_y_8x8x8[13][2]), + offset8x8x8(c1d_y_8x8x8[14][0], c1d_y_8x8x8[14][1]-1, c1d_y_8x8x8[14][2]), + offset8x8x8(c1d_y_8x8x8[15][0], c1d_y_8x8x8[15][1]-1, c1d_y_8x8x8[15][2]), + offset8x8x8(c1d_y_8x8x8[16][0], c1d_y_8x8x8[16][1]-1, c1d_y_8x8x8[16][2]), + offset8x8x8(c1d_y_8x8x8[17][0], c1d_y_8x8x8[17][1]-1, c1d_y_8x8x8[17][2]), + offset8x8x8(c1d_y_8x8x8[18][0], c1d_y_8x8x8[18][1]-1, c1d_y_8x8x8[18][2]), + offset8x8x8(c1d_y_8x8x8[19][0], c1d_y_8x8x8[19][1]-1, c1d_y_8x8x8[19][2]), + offset8x8x8(c1d_y_8x8x8[20][0], c1d_y_8x8x8[20][1]-1, c1d_y_8x8x8[20][2]), + offset8x8x8(c1d_y_8x8x8[21][0], c1d_y_8x8x8[21][1]-1, c1d_y_8x8x8[21][2]), + offset8x8x8(c1d_y_8x8x8[22][0], c1d_y_8x8x8[22][1]-1, c1d_y_8x8x8[22][2]), + offset8x8x8(c1d_y_8x8x8[23][0], c1d_y_8x8x8[23][1]-1, c1d_y_8x8x8[23][2]), + offset8x8x8(c1d_y_8x8x8[24][0], c1d_y_8x8x8[24][1]-1, c1d_y_8x8x8[24][2]), + offset8x8x8(c1d_y_8x8x8[25][0], c1d_y_8x8x8[25][1]-1, c1d_y_8x8x8[25][2]), + offset8x8x8(c1d_y_8x8x8[26][0], c1d_y_8x8x8[26][1]-1, c1d_y_8x8x8[26][2]), + offset8x8x8(c1d_y_8x8x8[27][0], c1d_y_8x8x8[27][1]-1, c1d_y_8x8x8[27][2]), + offset8x8x8(c1d_y_8x8x8[28][0], c1d_y_8x8x8[28][1]-1, c1d_y_8x8x8[28][2]), + offset8x8x8(c1d_y_8x8x8[29][0], c1d_y_8x8x8[29][1]-1, c1d_y_8x8x8[29][2]), + offset8x8x8(c1d_y_8x8x8[30][0], c1d_y_8x8x8[30][1]-1, c1d_y_8x8x8[30][2]), + offset8x8x8(c1d_y_8x8x8[31][0], c1d_y_8x8x8[31][1]-1, c1d_y_8x8x8[31][2]), + offset8x8x8(c1d_y_8x8x8[32][0], c1d_y_8x8x8[32][1]-1, c1d_y_8x8x8[32][2]), + offset8x8x8(c1d_y_8x8x8[33][0], c1d_y_8x8x8[33][1]-1, c1d_y_8x8x8[33][2]), + offset8x8x8(c1d_y_8x8x8[34][0], c1d_y_8x8x8[34][1]-1, c1d_y_8x8x8[34][2]), + offset8x8x8(c1d_y_8x8x8[35][0], c1d_y_8x8x8[35][1]-1, c1d_y_8x8x8[35][2]), + offset8x8x8(c1d_y_8x8x8[36][0], c1d_y_8x8x8[36][1]-1, c1d_y_8x8x8[36][2]), + offset8x8x8(c1d_y_8x8x8[37][0], c1d_y_8x8x8[37][1]-1, c1d_y_8x8x8[37][2]), + offset8x8x8(c1d_y_8x8x8[38][0], c1d_y_8x8x8[38][1]-1, c1d_y_8x8x8[38][2]), + offset8x8x8(c1d_y_8x8x8[39][0], c1d_y_8x8x8[39][1]-1, c1d_y_8x8x8[39][2]), + offset8x8x8(c1d_y_8x8x8[40][0], c1d_y_8x8x8[40][1]-1, c1d_y_8x8x8[40][2]), + offset8x8x8(c1d_y_8x8x8[41][0], c1d_y_8x8x8[41][1]-1, c1d_y_8x8x8[41][2]), + offset8x8x8(c1d_y_8x8x8[42][0], c1d_y_8x8x8[42][1]-1, c1d_y_8x8x8[42][2]), + offset8x8x8(c1d_y_8x8x8[43][0], c1d_y_8x8x8[43][1]-1, c1d_y_8x8x8[43][2]), + offset8x8x8(c1d_y_8x8x8[44][0], c1d_y_8x8x8[44][1]-1, c1d_y_8x8x8[44][2]), + offset8x8x8(c1d_y_8x8x8[45][0], c1d_y_8x8x8[45][1]-1, c1d_y_8x8x8[45][2]), + offset8x8x8(c1d_y_8x8x8[46][0], c1d_y_8x8x8[46][1]-1, c1d_y_8x8x8[46][2]), + offset8x8x8(c1d_y_8x8x8[47][0], c1d_y_8x8x8[47][1]-1, c1d_y_8x8x8[47][2]), + offset8x8x8(c1d_y_8x8x8[48][0], c1d_y_8x8x8[48][1]-1, c1d_y_8x8x8[48][2]), + offset8x8x8(c1d_y_8x8x8[49][0], c1d_y_8x8x8[49][1]-1, c1d_y_8x8x8[49][2]), + offset8x8x8(c1d_y_8x8x8[50][0], c1d_y_8x8x8[50][1]-1, c1d_y_8x8x8[50][2]), + offset8x8x8(c1d_y_8x8x8[51][0], c1d_y_8x8x8[51][1]-1, c1d_y_8x8x8[51][2]), + offset8x8x8(c1d_y_8x8x8[52][0], c1d_y_8x8x8[52][1]-1, c1d_y_8x8x8[52][2]), + offset8x8x8(c1d_y_8x8x8[53][0], c1d_y_8x8x8[53][1]-1, c1d_y_8x8x8[53][2]), + offset8x8x8(c1d_y_8x8x8[54][0], c1d_y_8x8x8[54][1]-1, c1d_y_8x8x8[54][2]), + offset8x8x8(c1d_y_8x8x8[55][0], c1d_y_8x8x8[55][1]-1, c1d_y_8x8x8[55][2]), + offset8x8x8(c1d_y_8x8x8[56][0], c1d_y_8x8x8[56][1]-1, c1d_y_8x8x8[56][2]), + offset8x8x8(c1d_y_8x8x8[57][0], c1d_y_8x8x8[57][1]-1, c1d_y_8x8x8[57][2]), + offset8x8x8(c1d_y_8x8x8[58][0], c1d_y_8x8x8[58][1]-1, c1d_y_8x8x8[58][2]), + offset8x8x8(c1d_y_8x8x8[59][0], c1d_y_8x8x8[59][1]-1, c1d_y_8x8x8[59][2]), + offset8x8x8(c1d_y_8x8x8[60][0], c1d_y_8x8x8[60][1]-1, c1d_y_8x8x8[60][2]), + offset8x8x8(c1d_y_8x8x8[61][0], c1d_y_8x8x8[61][1]-1, c1d_y_8x8x8[61][2]), + offset8x8x8(c1d_y_8x8x8[62][0], c1d_y_8x8x8[62][1]-1, c1d_y_8x8x8[62][2]), + offset8x8x8(c1d_y_8x8x8[63][0], c1d_y_8x8x8[63][1]-1, c1d_y_8x8x8[63][2]), + offset8x8x8(c1d_y_8x8x8[64][0], c1d_y_8x8x8[64][1]-1, c1d_y_8x8x8[64][2]), + offset8x8x8(c1d_y_8x8x8[65][0], c1d_y_8x8x8[65][1]-1, c1d_y_8x8x8[65][2]), + offset8x8x8(c1d_y_8x8x8[66][0], c1d_y_8x8x8[66][1]-1, c1d_y_8x8x8[66][2]), + offset8x8x8(c1d_y_8x8x8[67][0], c1d_y_8x8x8[67][1]-1, c1d_y_8x8x8[67][2]), + offset8x8x8(c1d_y_8x8x8[68][0], c1d_y_8x8x8[68][1]-1, c1d_y_8x8x8[68][2]), + offset8x8x8(c1d_y_8x8x8[69][0], c1d_y_8x8x8[69][1]-1, c1d_y_8x8x8[69][2]), + offset8x8x8(c1d_y_8x8x8[70][0], c1d_y_8x8x8[70][1]-1, c1d_y_8x8x8[70][2]), + offset8x8x8(c1d_y_8x8x8[71][0], c1d_y_8x8x8[71][1]-1, c1d_y_8x8x8[71][2]), + offset8x8x8(c1d_y_8x8x8[72][0], c1d_y_8x8x8[72][1]-1, c1d_y_8x8x8[72][2]), + offset8x8x8(c1d_y_8x8x8[73][0], c1d_y_8x8x8[73][1]-1, c1d_y_8x8x8[73][2]), + offset8x8x8(c1d_y_8x8x8[74][0], c1d_y_8x8x8[74][1]-1, c1d_y_8x8x8[74][2]), + + offset8x8x8(c1d_z_8x8x8[0][0]-1, c1d_z_8x8x8[0][1], c1d_z_8x8x8[0][2]), // Z + offset8x8x8(c1d_z_8x8x8[1][0]-1, c1d_z_8x8x8[1][1], c1d_z_8x8x8[1][2]), + offset8x8x8(c1d_z_8x8x8[2][0]-1, c1d_z_8x8x8[2][1], c1d_z_8x8x8[2][2]), + offset8x8x8(c1d_z_8x8x8[3][0]-1, c1d_z_8x8x8[3][1], c1d_z_8x8x8[3][2]), + offset8x8x8(c1d_z_8x8x8[4][0]-1, c1d_z_8x8x8[4][1], c1d_z_8x8x8[4][2]), + offset8x8x8(c1d_z_8x8x8[5][0]-1, c1d_z_8x8x8[5][1], c1d_z_8x8x8[5][2]), + offset8x8x8(c1d_z_8x8x8[6][0]-1, c1d_z_8x8x8[6][1], c1d_z_8x8x8[6][2]), + offset8x8x8(c1d_z_8x8x8[7][0]-1, c1d_z_8x8x8[7][1], c1d_z_8x8x8[7][2]), + offset8x8x8(c1d_z_8x8x8[8][0]-1, c1d_z_8x8x8[8][1], c1d_z_8x8x8[8][2]), + offset8x8x8(c1d_z_8x8x8[9][0]-1, c1d_z_8x8x8[9][1], c1d_z_8x8x8[9][2]), + offset8x8x8(c1d_z_8x8x8[10][0]-1, c1d_z_8x8x8[10][1], c1d_z_8x8x8[10][2]), + offset8x8x8(c1d_z_8x8x8[11][0]-1, c1d_z_8x8x8[11][1], c1d_z_8x8x8[11][2]), + offset8x8x8(c1d_z_8x8x8[12][0]-1, c1d_z_8x8x8[12][1], c1d_z_8x8x8[12][2]), + offset8x8x8(c1d_z_8x8x8[13][0]-1, c1d_z_8x8x8[13][1], c1d_z_8x8x8[13][2]), + offset8x8x8(c1d_z_8x8x8[14][0]-1, c1d_z_8x8x8[14][1], c1d_z_8x8x8[14][2]), + offset8x8x8(c1d_z_8x8x8[15][0]-1, c1d_z_8x8x8[15][1], c1d_z_8x8x8[15][2]), + offset8x8x8(c1d_z_8x8x8[16][0]-1, c1d_z_8x8x8[16][1], c1d_z_8x8x8[16][2]), + offset8x8x8(c1d_z_8x8x8[17][0]-1, c1d_z_8x8x8[17][1], c1d_z_8x8x8[17][2]), + offset8x8x8(c1d_z_8x8x8[18][0]-1, c1d_z_8x8x8[18][1], c1d_z_8x8x8[18][2]), + offset8x8x8(c1d_z_8x8x8[19][0]-1, c1d_z_8x8x8[19][1], c1d_z_8x8x8[19][2]), + offset8x8x8(c1d_z_8x8x8[20][0]-1, c1d_z_8x8x8[20][1], c1d_z_8x8x8[20][2]), + offset8x8x8(c1d_z_8x8x8[21][0]-1, c1d_z_8x8x8[21][1], c1d_z_8x8x8[21][2]), + offset8x8x8(c1d_z_8x8x8[22][0]-1, c1d_z_8x8x8[22][1], c1d_z_8x8x8[22][2]), + offset8x8x8(c1d_z_8x8x8[23][0]-1, c1d_z_8x8x8[23][1], c1d_z_8x8x8[23][2]), + offset8x8x8(c1d_z_8x8x8[24][0]-1, c1d_z_8x8x8[24][1], c1d_z_8x8x8[24][2]), + offset8x8x8(c1d_z_8x8x8[25][0]-1, c1d_z_8x8x8[25][1], c1d_z_8x8x8[25][2]), + offset8x8x8(c1d_z_8x8x8[26][0]-1, c1d_z_8x8x8[26][1], c1d_z_8x8x8[26][2]), + offset8x8x8(c1d_z_8x8x8[27][0]-1, c1d_z_8x8x8[27][1], c1d_z_8x8x8[27][2]), + offset8x8x8(c1d_z_8x8x8[28][0]-1, c1d_z_8x8x8[28][1], c1d_z_8x8x8[28][2]), + offset8x8x8(c1d_z_8x8x8[29][0]-1, c1d_z_8x8x8[29][1], c1d_z_8x8x8[29][2]), + offset8x8x8(c1d_z_8x8x8[30][0]-1, c1d_z_8x8x8[30][1], c1d_z_8x8x8[30][2]), + offset8x8x8(c1d_z_8x8x8[31][0]-1, c1d_z_8x8x8[31][1], c1d_z_8x8x8[31][2]), + offset8x8x8(c1d_z_8x8x8[32][0]-1, c1d_z_8x8x8[32][1], c1d_z_8x8x8[32][2]), + offset8x8x8(c1d_z_8x8x8[33][0]-1, c1d_z_8x8x8[33][1], c1d_z_8x8x8[33][2]), + offset8x8x8(c1d_z_8x8x8[34][0]-1, c1d_z_8x8x8[34][1], c1d_z_8x8x8[34][2]), + offset8x8x8(c1d_z_8x8x8[35][0]-1, c1d_z_8x8x8[35][1], c1d_z_8x8x8[35][2]), + offset8x8x8(c1d_z_8x8x8[36][0]-1, c1d_z_8x8x8[36][1], c1d_z_8x8x8[36][2]), + offset8x8x8(c1d_z_8x8x8[37][0]-1, c1d_z_8x8x8[37][1], c1d_z_8x8x8[37][2]), + offset8x8x8(c1d_z_8x8x8[38][0]-1, c1d_z_8x8x8[38][1], c1d_z_8x8x8[38][2]), + offset8x8x8(c1d_z_8x8x8[39][0]-1, c1d_z_8x8x8[39][1], c1d_z_8x8x8[39][2]), + offset8x8x8(c1d_z_8x8x8[40][0]-1, c1d_z_8x8x8[40][1], c1d_z_8x8x8[40][2]), + offset8x8x8(c1d_z_8x8x8[41][0]-1, c1d_z_8x8x8[41][1], c1d_z_8x8x8[41][2]), + offset8x8x8(c1d_z_8x8x8[42][0]-1, c1d_z_8x8x8[42][1], c1d_z_8x8x8[42][2]), + offset8x8x8(c1d_z_8x8x8[43][0]-1, c1d_z_8x8x8[43][1], c1d_z_8x8x8[43][2]), + offset8x8x8(c1d_z_8x8x8[44][0]-1, c1d_z_8x8x8[44][1], c1d_z_8x8x8[44][2]), + offset8x8x8(c1d_z_8x8x8[45][0]-1, c1d_z_8x8x8[45][1], c1d_z_8x8x8[45][2]), + offset8x8x8(c1d_z_8x8x8[46][0]-1, c1d_z_8x8x8[46][1], c1d_z_8x8x8[46][2]), + offset8x8x8(c1d_z_8x8x8[47][0]-1, c1d_z_8x8x8[47][1], c1d_z_8x8x8[47][2]), + offset8x8x8(c1d_z_8x8x8[48][0]-1, c1d_z_8x8x8[48][1], c1d_z_8x8x8[48][2]), + offset8x8x8(c1d_z_8x8x8[49][0]-1, c1d_z_8x8x8[49][1], c1d_z_8x8x8[49][2]), + offset8x8x8(c1d_z_8x8x8[50][0]-1, c1d_z_8x8x8[50][1], c1d_z_8x8x8[50][2]), + offset8x8x8(c1d_z_8x8x8[51][0]-1, c1d_z_8x8x8[51][1], c1d_z_8x8x8[51][2]), + offset8x8x8(c1d_z_8x8x8[52][0]-1, c1d_z_8x8x8[52][1], c1d_z_8x8x8[52][2]), + offset8x8x8(c1d_z_8x8x8[53][0]-1, c1d_z_8x8x8[53][1], c1d_z_8x8x8[53][2]), + offset8x8x8(c1d_z_8x8x8[54][0]-1, c1d_z_8x8x8[54][1], c1d_z_8x8x8[54][2]), + offset8x8x8(c1d_z_8x8x8[55][0]-1, c1d_z_8x8x8[55][1], c1d_z_8x8x8[55][2]), + offset8x8x8(c1d_z_8x8x8[56][0]-1, c1d_z_8x8x8[56][1], c1d_z_8x8x8[56][2]), + offset8x8x8(c1d_z_8x8x8[57][0]-1, c1d_z_8x8x8[57][1], c1d_z_8x8x8[57][2]), + offset8x8x8(c1d_z_8x8x8[58][0]-1, c1d_z_8x8x8[58][1], c1d_z_8x8x8[58][2]), + offset8x8x8(c1d_z_8x8x8[59][0]-1, c1d_z_8x8x8[59][1], c1d_z_8x8x8[59][2]), + offset8x8x8(c1d_z_8x8x8[60][0]-1, c1d_z_8x8x8[60][1], c1d_z_8x8x8[60][2]), + offset8x8x8(c1d_z_8x8x8[61][0]-1, c1d_z_8x8x8[61][1], c1d_z_8x8x8[61][2]), + offset8x8x8(c1d_z_8x8x8[62][0]-1, c1d_z_8x8x8[62][1], c1d_z_8x8x8[62][2]), + offset8x8x8(c1d_z_8x8x8[63][0]-1, c1d_z_8x8x8[63][1], c1d_z_8x8x8[63][2]), + offset8x8x8(c1d_z_8x8x8[64][0]-1, c1d_z_8x8x8[64][1], c1d_z_8x8x8[64][2]), + offset8x8x8(c1d_z_8x8x8[65][0]-1, c1d_z_8x8x8[65][1], c1d_z_8x8x8[65][2]), + offset8x8x8(c1d_z_8x8x8[66][0]-1, c1d_z_8x8x8[66][1], c1d_z_8x8x8[66][2]), + offset8x8x8(c1d_z_8x8x8[67][0]-1, c1d_z_8x8x8[67][1], c1d_z_8x8x8[67][2]), + offset8x8x8(c1d_z_8x8x8[68][0]-1, c1d_z_8x8x8[68][1], c1d_z_8x8x8[68][2]), + offset8x8x8(c1d_z_8x8x8[69][0]-1, c1d_z_8x8x8[69][1], c1d_z_8x8x8[69][2]), + offset8x8x8(c1d_z_8x8x8[70][0]-1, c1d_z_8x8x8[70][1], c1d_z_8x8x8[70][2]), + offset8x8x8(c1d_z_8x8x8[71][0]-1, c1d_z_8x8x8[71][1], c1d_z_8x8x8[71][2]), + offset8x8x8(c1d_z_8x8x8[72][0]-1, c1d_z_8x8x8[72][1], c1d_z_8x8x8[72][2]), + offset8x8x8(c1d_z_8x8x8[73][0]-1, c1d_z_8x8x8[73][1], c1d_z_8x8x8[73][2]), + offset8x8x8(c1d_z_8x8x8[74][0]-1, c1d_z_8x8x8[74][1], c1d_z_8x8x8[74][2])}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff1D_R_Offset_8x8x8(SIZE i) { + static constexpr int offset[225] = {offset8x8x8(c1d_x_8x8x8[0][0], c1d_x_8x8x8[0][1], c1d_x_8x8x8[0][2]+1), // X + offset8x8x8(c1d_x_8x8x8[1][0], c1d_x_8x8x8[1][1], c1d_x_8x8x8[1][2]+1), + offset8x8x8(c1d_x_8x8x8[2][0], c1d_x_8x8x8[2][1], c1d_x_8x8x8[2][2]+1), + offset8x8x8(c1d_x_8x8x8[3][0], c1d_x_8x8x8[3][1], c1d_x_8x8x8[3][2]+1), + offset8x8x8(c1d_x_8x8x8[4][0], c1d_x_8x8x8[4][1], c1d_x_8x8x8[4][2]+1), + offset8x8x8(c1d_x_8x8x8[5][0], c1d_x_8x8x8[5][1], c1d_x_8x8x8[5][2]+1), + offset8x8x8(c1d_x_8x8x8[6][0], c1d_x_8x8x8[6][1], c1d_x_8x8x8[6][2]+1), + offset8x8x8(c1d_x_8x8x8[7][0], c1d_x_8x8x8[7][1], c1d_x_8x8x8[7][2]+1), + offset8x8x8(c1d_x_8x8x8[8][0], c1d_x_8x8x8[8][1], c1d_x_8x8x8[8][2]+1), + offset8x8x8(c1d_x_8x8x8[9][0], c1d_x_8x8x8[9][1], c1d_x_8x8x8[9][2]+1), + offset8x8x8(c1d_x_8x8x8[10][0], c1d_x_8x8x8[10][1], c1d_x_8x8x8[10][2]+1), + offset8x8x8(c1d_x_8x8x8[11][0], c1d_x_8x8x8[11][1], c1d_x_8x8x8[11][2]+1), + offset8x8x8(c1d_x_8x8x8[12][0], c1d_x_8x8x8[12][1], c1d_x_8x8x8[12][2]+1), + offset8x8x8(c1d_x_8x8x8[13][0], c1d_x_8x8x8[13][1], c1d_x_8x8x8[13][2]+1), + offset8x8x8(c1d_x_8x8x8[14][0], c1d_x_8x8x8[14][1], c1d_x_8x8x8[14][2]+1), + offset8x8x8(c1d_x_8x8x8[15][0], c1d_x_8x8x8[15][1], c1d_x_8x8x8[15][2]+1), + offset8x8x8(c1d_x_8x8x8[16][0], c1d_x_8x8x8[16][1], c1d_x_8x8x8[16][2]+1), + offset8x8x8(c1d_x_8x8x8[17][0], c1d_x_8x8x8[17][1], c1d_x_8x8x8[17][2]+1), + offset8x8x8(c1d_x_8x8x8[18][0], c1d_x_8x8x8[18][1], c1d_x_8x8x8[18][2]+1), + offset8x8x8(c1d_x_8x8x8[19][0], c1d_x_8x8x8[19][1], c1d_x_8x8x8[19][2]+1), + offset8x8x8(c1d_x_8x8x8[20][0], c1d_x_8x8x8[20][1], c1d_x_8x8x8[20][2]+1), + offset8x8x8(c1d_x_8x8x8[21][0], c1d_x_8x8x8[21][1], c1d_x_8x8x8[21][2]+1), + offset8x8x8(c1d_x_8x8x8[22][0], c1d_x_8x8x8[22][1], c1d_x_8x8x8[22][2]+1), + offset8x8x8(c1d_x_8x8x8[23][0], c1d_x_8x8x8[23][1], c1d_x_8x8x8[23][2]+1), + offset8x8x8(c1d_x_8x8x8[24][0], c1d_x_8x8x8[24][1], c1d_x_8x8x8[24][2]+1), + offset8x8x8(c1d_x_8x8x8[25][0], c1d_x_8x8x8[25][1], c1d_x_8x8x8[25][2]+1), + offset8x8x8(c1d_x_8x8x8[26][0], c1d_x_8x8x8[26][1], c1d_x_8x8x8[26][2]+1), + offset8x8x8(c1d_x_8x8x8[27][0], c1d_x_8x8x8[27][1], c1d_x_8x8x8[27][2]+1), + offset8x8x8(c1d_x_8x8x8[28][0], c1d_x_8x8x8[28][1], c1d_x_8x8x8[28][2]+1), + offset8x8x8(c1d_x_8x8x8[29][0], c1d_x_8x8x8[29][1], c1d_x_8x8x8[29][2]+1), + offset8x8x8(c1d_x_8x8x8[30][0], c1d_x_8x8x8[30][1], c1d_x_8x8x8[30][2]+1), + offset8x8x8(c1d_x_8x8x8[31][0], c1d_x_8x8x8[31][1], c1d_x_8x8x8[31][2]+1), + offset8x8x8(c1d_x_8x8x8[32][0], c1d_x_8x8x8[32][1], c1d_x_8x8x8[32][2]+1), + offset8x8x8(c1d_x_8x8x8[33][0], c1d_x_8x8x8[33][1], c1d_x_8x8x8[33][2]+1), + offset8x8x8(c1d_x_8x8x8[34][0], c1d_x_8x8x8[34][1], c1d_x_8x8x8[34][2]+1), + offset8x8x8(c1d_x_8x8x8[35][0], c1d_x_8x8x8[35][1], c1d_x_8x8x8[35][2]+1), + offset8x8x8(c1d_x_8x8x8[36][0], c1d_x_8x8x8[36][1], c1d_x_8x8x8[36][2]+1), + offset8x8x8(c1d_x_8x8x8[37][0], c1d_x_8x8x8[37][1], c1d_x_8x8x8[37][2]+1), + offset8x8x8(c1d_x_8x8x8[38][0], c1d_x_8x8x8[38][1], c1d_x_8x8x8[38][2]+1), + offset8x8x8(c1d_x_8x8x8[39][0], c1d_x_8x8x8[39][1], c1d_x_8x8x8[39][2]+1), + offset8x8x8(c1d_x_8x8x8[40][0], c1d_x_8x8x8[40][1], c1d_x_8x8x8[40][2]+1), + offset8x8x8(c1d_x_8x8x8[41][0], c1d_x_8x8x8[41][1], c1d_x_8x8x8[41][2]+1), + offset8x8x8(c1d_x_8x8x8[42][0], c1d_x_8x8x8[42][1], c1d_x_8x8x8[42][2]+1), + offset8x8x8(c1d_x_8x8x8[43][0], c1d_x_8x8x8[43][1], c1d_x_8x8x8[43][2]+1), + offset8x8x8(c1d_x_8x8x8[44][0], c1d_x_8x8x8[44][1], c1d_x_8x8x8[44][2]+1), + offset8x8x8(c1d_x_8x8x8[45][0], c1d_x_8x8x8[45][1], c1d_x_8x8x8[45][2]+1), + offset8x8x8(c1d_x_8x8x8[46][0], c1d_x_8x8x8[46][1], c1d_x_8x8x8[46][2]+1), + offset8x8x8(c1d_x_8x8x8[47][0], c1d_x_8x8x8[47][1], c1d_x_8x8x8[47][2]+1), + offset8x8x8(c1d_x_8x8x8[48][0], c1d_x_8x8x8[48][1], c1d_x_8x8x8[48][2]+1), + offset8x8x8(c1d_x_8x8x8[49][0], c1d_x_8x8x8[49][1], c1d_x_8x8x8[49][2]+1), + offset8x8x8(c1d_x_8x8x8[50][0], c1d_x_8x8x8[50][1], c1d_x_8x8x8[50][2]+1), + offset8x8x8(c1d_x_8x8x8[51][0], c1d_x_8x8x8[51][1], c1d_x_8x8x8[51][2]+1), + offset8x8x8(c1d_x_8x8x8[52][0], c1d_x_8x8x8[52][1], c1d_x_8x8x8[52][2]+1), + offset8x8x8(c1d_x_8x8x8[53][0], c1d_x_8x8x8[53][1], c1d_x_8x8x8[53][2]+1), + offset8x8x8(c1d_x_8x8x8[54][0], c1d_x_8x8x8[54][1], c1d_x_8x8x8[54][2]+1), + offset8x8x8(c1d_x_8x8x8[55][0], c1d_x_8x8x8[55][1], c1d_x_8x8x8[55][2]+1), + offset8x8x8(c1d_x_8x8x8[56][0], c1d_x_8x8x8[56][1], c1d_x_8x8x8[56][2]+1), + offset8x8x8(c1d_x_8x8x8[57][0], c1d_x_8x8x8[57][1], c1d_x_8x8x8[57][2]+1), + offset8x8x8(c1d_x_8x8x8[58][0], c1d_x_8x8x8[58][1], c1d_x_8x8x8[58][2]+1), + offset8x8x8(c1d_x_8x8x8[59][0], c1d_x_8x8x8[59][1], c1d_x_8x8x8[59][2]+1), + offset8x8x8(c1d_x_8x8x8[60][0], c1d_x_8x8x8[60][1], c1d_x_8x8x8[60][2]+1), + offset8x8x8(c1d_x_8x8x8[61][0], c1d_x_8x8x8[61][1], c1d_x_8x8x8[61][2]+1), + offset8x8x8(c1d_x_8x8x8[62][0], c1d_x_8x8x8[62][1], c1d_x_8x8x8[62][2]+1), + offset8x8x8(c1d_x_8x8x8[63][0], c1d_x_8x8x8[63][1], c1d_x_8x8x8[63][2]+1), + offset8x8x8(c1d_x_8x8x8[64][0], c1d_x_8x8x8[64][1], c1d_x_8x8x8[64][2]+1), + offset8x8x8(c1d_x_8x8x8[65][0], c1d_x_8x8x8[65][1], c1d_x_8x8x8[65][2]+1), + offset8x8x8(c1d_x_8x8x8[66][0], c1d_x_8x8x8[66][1], c1d_x_8x8x8[66][2]+1), + offset8x8x8(c1d_x_8x8x8[67][0], c1d_x_8x8x8[67][1], c1d_x_8x8x8[67][2]+1), + offset8x8x8(c1d_x_8x8x8[68][0], c1d_x_8x8x8[68][1], c1d_x_8x8x8[68][2]+1), + offset8x8x8(c1d_x_8x8x8[69][0], c1d_x_8x8x8[69][1], c1d_x_8x8x8[69][2]+1), + offset8x8x8(c1d_x_8x8x8[70][0], c1d_x_8x8x8[70][1], c1d_x_8x8x8[70][2]+1), + offset8x8x8(c1d_x_8x8x8[71][0], c1d_x_8x8x8[71][1], c1d_x_8x8x8[71][2]+1), + offset8x8x8(c1d_x_8x8x8[72][0], c1d_x_8x8x8[72][1], c1d_x_8x8x8[72][2]+1), + offset8x8x8(c1d_x_8x8x8[73][0], c1d_x_8x8x8[73][1], c1d_x_8x8x8[73][2]+1), + offset8x8x8(c1d_x_8x8x8[74][0], c1d_x_8x8x8[74][1], c1d_x_8x8x8[74][2]+1), + + offset8x8x8(c1d_y_8x8x8[0][0], c1d_y_8x8x8[0][1]+1, c1d_y_8x8x8[0][2]), // Y + offset8x8x8(c1d_y_8x8x8[1][0], c1d_y_8x8x8[1][1]+1, c1d_y_8x8x8[1][2]), + offset8x8x8(c1d_y_8x8x8[2][0], c1d_y_8x8x8[2][1]+1, c1d_y_8x8x8[2][2]), + offset8x8x8(c1d_y_8x8x8[3][0], c1d_y_8x8x8[3][1]+1, c1d_y_8x8x8[3][2]), + offset8x8x8(c1d_y_8x8x8[4][0], c1d_y_8x8x8[4][1]+1, c1d_y_8x8x8[4][2]), + offset8x8x8(c1d_y_8x8x8[5][0], c1d_y_8x8x8[5][1]+1, c1d_y_8x8x8[5][2]), + offset8x8x8(c1d_y_8x8x8[6][0], c1d_y_8x8x8[6][1]+1, c1d_y_8x8x8[6][2]), + offset8x8x8(c1d_y_8x8x8[7][0], c1d_y_8x8x8[7][1]+1, c1d_y_8x8x8[7][2]), + offset8x8x8(c1d_y_8x8x8[8][0], c1d_y_8x8x8[8][1]+1, c1d_y_8x8x8[8][2]), + offset8x8x8(c1d_y_8x8x8[9][0], c1d_y_8x8x8[9][1]+1, c1d_y_8x8x8[9][2]), + offset8x8x8(c1d_y_8x8x8[10][0], c1d_y_8x8x8[10][1]+1, c1d_y_8x8x8[10][2]), + offset8x8x8(c1d_y_8x8x8[11][0], c1d_y_8x8x8[11][1]+1, c1d_y_8x8x8[11][2]), + offset8x8x8(c1d_y_8x8x8[12][0], c1d_y_8x8x8[12][1]+1, c1d_y_8x8x8[12][2]), + offset8x8x8(c1d_y_8x8x8[13][0], c1d_y_8x8x8[13][1]+1, c1d_y_8x8x8[13][2]), + offset8x8x8(c1d_y_8x8x8[14][0], c1d_y_8x8x8[14][1]+1, c1d_y_8x8x8[14][2]), + offset8x8x8(c1d_y_8x8x8[15][0], c1d_y_8x8x8[15][1]+1, c1d_y_8x8x8[15][2]), + offset8x8x8(c1d_y_8x8x8[16][0], c1d_y_8x8x8[16][1]+1, c1d_y_8x8x8[16][2]), + offset8x8x8(c1d_y_8x8x8[17][0], c1d_y_8x8x8[17][1]+1, c1d_y_8x8x8[17][2]), + offset8x8x8(c1d_y_8x8x8[18][0], c1d_y_8x8x8[18][1]+1, c1d_y_8x8x8[18][2]), + offset8x8x8(c1d_y_8x8x8[19][0], c1d_y_8x8x8[19][1]+1, c1d_y_8x8x8[19][2]), + offset8x8x8(c1d_y_8x8x8[20][0], c1d_y_8x8x8[20][1]+1, c1d_y_8x8x8[20][2]), + offset8x8x8(c1d_y_8x8x8[21][0], c1d_y_8x8x8[21][1]+1, c1d_y_8x8x8[21][2]), + offset8x8x8(c1d_y_8x8x8[22][0], c1d_y_8x8x8[22][1]+1, c1d_y_8x8x8[22][2]), + offset8x8x8(c1d_y_8x8x8[23][0], c1d_y_8x8x8[23][1]+1, c1d_y_8x8x8[23][2]), + offset8x8x8(c1d_y_8x8x8[24][0], c1d_y_8x8x8[24][1]+1, c1d_y_8x8x8[24][2]), + offset8x8x8(c1d_y_8x8x8[25][0], c1d_y_8x8x8[25][1]+1, c1d_y_8x8x8[25][2]), + offset8x8x8(c1d_y_8x8x8[26][0], c1d_y_8x8x8[26][1]+1, c1d_y_8x8x8[26][2]), + offset8x8x8(c1d_y_8x8x8[27][0], c1d_y_8x8x8[27][1]+1, c1d_y_8x8x8[27][2]), + offset8x8x8(c1d_y_8x8x8[28][0], c1d_y_8x8x8[28][1]+1, c1d_y_8x8x8[28][2]), + offset8x8x8(c1d_y_8x8x8[29][0], c1d_y_8x8x8[29][1]+1, c1d_y_8x8x8[29][2]), + offset8x8x8(c1d_y_8x8x8[30][0], c1d_y_8x8x8[30][1]+1, c1d_y_8x8x8[30][2]), + offset8x8x8(c1d_y_8x8x8[31][0], c1d_y_8x8x8[31][1]+1, c1d_y_8x8x8[31][2]), + offset8x8x8(c1d_y_8x8x8[32][0], c1d_y_8x8x8[32][1]+1, c1d_y_8x8x8[32][2]), + offset8x8x8(c1d_y_8x8x8[33][0], c1d_y_8x8x8[33][1]+1, c1d_y_8x8x8[33][2]), + offset8x8x8(c1d_y_8x8x8[34][0], c1d_y_8x8x8[34][1]+1, c1d_y_8x8x8[34][2]), + offset8x8x8(c1d_y_8x8x8[35][0], c1d_y_8x8x8[35][1]+1, c1d_y_8x8x8[35][2]), + offset8x8x8(c1d_y_8x8x8[36][0], c1d_y_8x8x8[36][1]+1, c1d_y_8x8x8[36][2]), + offset8x8x8(c1d_y_8x8x8[37][0], c1d_y_8x8x8[37][1]+1, c1d_y_8x8x8[37][2]), + offset8x8x8(c1d_y_8x8x8[38][0], c1d_y_8x8x8[38][1]+1, c1d_y_8x8x8[38][2]), + offset8x8x8(c1d_y_8x8x8[39][0], c1d_y_8x8x8[39][1]+1, c1d_y_8x8x8[39][2]), + offset8x8x8(c1d_y_8x8x8[40][0], c1d_y_8x8x8[40][1]+1, c1d_y_8x8x8[40][2]), + offset8x8x8(c1d_y_8x8x8[41][0], c1d_y_8x8x8[41][1]+1, c1d_y_8x8x8[41][2]), + offset8x8x8(c1d_y_8x8x8[42][0], c1d_y_8x8x8[42][1]+1, c1d_y_8x8x8[42][2]), + offset8x8x8(c1d_y_8x8x8[43][0], c1d_y_8x8x8[43][1]+1, c1d_y_8x8x8[43][2]), + offset8x8x8(c1d_y_8x8x8[44][0], c1d_y_8x8x8[44][1]+1, c1d_y_8x8x8[44][2]), + offset8x8x8(c1d_y_8x8x8[45][0], c1d_y_8x8x8[45][1]+1, c1d_y_8x8x8[45][2]), + offset8x8x8(c1d_y_8x8x8[46][0], c1d_y_8x8x8[46][1]+1, c1d_y_8x8x8[46][2]), + offset8x8x8(c1d_y_8x8x8[47][0], c1d_y_8x8x8[47][1]+1, c1d_y_8x8x8[47][2]), + offset8x8x8(c1d_y_8x8x8[48][0], c1d_y_8x8x8[48][1]+1, c1d_y_8x8x8[48][2]), + offset8x8x8(c1d_y_8x8x8[49][0], c1d_y_8x8x8[49][1]+1, c1d_y_8x8x8[49][2]), + offset8x8x8(c1d_y_8x8x8[50][0], c1d_y_8x8x8[50][1]+1, c1d_y_8x8x8[50][2]), + offset8x8x8(c1d_y_8x8x8[51][0], c1d_y_8x8x8[51][1]+1, c1d_y_8x8x8[51][2]), + offset8x8x8(c1d_y_8x8x8[52][0], c1d_y_8x8x8[52][1]+1, c1d_y_8x8x8[52][2]), + offset8x8x8(c1d_y_8x8x8[53][0], c1d_y_8x8x8[53][1]+1, c1d_y_8x8x8[53][2]), + offset8x8x8(c1d_y_8x8x8[54][0], c1d_y_8x8x8[54][1]+1, c1d_y_8x8x8[54][2]), + offset8x8x8(c1d_y_8x8x8[55][0], c1d_y_8x8x8[55][1]+1, c1d_y_8x8x8[55][2]), + offset8x8x8(c1d_y_8x8x8[56][0], c1d_y_8x8x8[56][1]+1, c1d_y_8x8x8[56][2]), + offset8x8x8(c1d_y_8x8x8[57][0], c1d_y_8x8x8[57][1]+1, c1d_y_8x8x8[57][2]), + offset8x8x8(c1d_y_8x8x8[58][0], c1d_y_8x8x8[58][1]+1, c1d_y_8x8x8[58][2]), + offset8x8x8(c1d_y_8x8x8[59][0], c1d_y_8x8x8[59][1]+1, c1d_y_8x8x8[59][2]), + offset8x8x8(c1d_y_8x8x8[60][0], c1d_y_8x8x8[60][1]+1, c1d_y_8x8x8[60][2]), + offset8x8x8(c1d_y_8x8x8[61][0], c1d_y_8x8x8[61][1]+1, c1d_y_8x8x8[61][2]), + offset8x8x8(c1d_y_8x8x8[62][0], c1d_y_8x8x8[62][1]+1, c1d_y_8x8x8[62][2]), + offset8x8x8(c1d_y_8x8x8[63][0], c1d_y_8x8x8[63][1]+1, c1d_y_8x8x8[63][2]), + offset8x8x8(c1d_y_8x8x8[64][0], c1d_y_8x8x8[64][1]+1, c1d_y_8x8x8[64][2]), + offset8x8x8(c1d_y_8x8x8[65][0], c1d_y_8x8x8[65][1]+1, c1d_y_8x8x8[65][2]), + offset8x8x8(c1d_y_8x8x8[66][0], c1d_y_8x8x8[66][1]+1, c1d_y_8x8x8[66][2]), + offset8x8x8(c1d_y_8x8x8[67][0], c1d_y_8x8x8[67][1]+1, c1d_y_8x8x8[67][2]), + offset8x8x8(c1d_y_8x8x8[68][0], c1d_y_8x8x8[68][1]+1, c1d_y_8x8x8[68][2]), + offset8x8x8(c1d_y_8x8x8[69][0], c1d_y_8x8x8[69][1]+1, c1d_y_8x8x8[69][2]), + offset8x8x8(c1d_y_8x8x8[70][0], c1d_y_8x8x8[70][1]+1, c1d_y_8x8x8[70][2]), + offset8x8x8(c1d_y_8x8x8[71][0], c1d_y_8x8x8[71][1]+1, c1d_y_8x8x8[71][2]), + offset8x8x8(c1d_y_8x8x8[72][0], c1d_y_8x8x8[72][1]+1, c1d_y_8x8x8[72][2]), + offset8x8x8(c1d_y_8x8x8[73][0], c1d_y_8x8x8[73][1]+1, c1d_y_8x8x8[73][2]), + offset8x8x8(c1d_y_8x8x8[74][0], c1d_y_8x8x8[74][1]+1, c1d_y_8x8x8[74][2]), + + offset8x8x8(c1d_z_8x8x8[0][0]+1, c1d_z_8x8x8[0][1], c1d_z_8x8x8[0][2]), // Z + offset8x8x8(c1d_z_8x8x8[1][0]+1, c1d_z_8x8x8[1][1], c1d_z_8x8x8[1][2]), + offset8x8x8(c1d_z_8x8x8[2][0]+1, c1d_z_8x8x8[2][1], c1d_z_8x8x8[2][2]), + offset8x8x8(c1d_z_8x8x8[3][0]+1, c1d_z_8x8x8[3][1], c1d_z_8x8x8[3][2]), + offset8x8x8(c1d_z_8x8x8[4][0]+1, c1d_z_8x8x8[4][1], c1d_z_8x8x8[4][2]), + offset8x8x8(c1d_z_8x8x8[5][0]+1, c1d_z_8x8x8[5][1], c1d_z_8x8x8[5][2]), + offset8x8x8(c1d_z_8x8x8[6][0]+1, c1d_z_8x8x8[6][1], c1d_z_8x8x8[6][2]), + offset8x8x8(c1d_z_8x8x8[7][0]+1, c1d_z_8x8x8[7][1], c1d_z_8x8x8[7][2]), + offset8x8x8(c1d_z_8x8x8[8][0]+1, c1d_z_8x8x8[8][1], c1d_z_8x8x8[8][2]), + offset8x8x8(c1d_z_8x8x8[9][0]+1, c1d_z_8x8x8[9][1], c1d_z_8x8x8[9][2]), + offset8x8x8(c1d_z_8x8x8[10][0]+1, c1d_z_8x8x8[10][1], c1d_z_8x8x8[10][2]), + offset8x8x8(c1d_z_8x8x8[11][0]+1, c1d_z_8x8x8[11][1], c1d_z_8x8x8[11][2]), + offset8x8x8(c1d_z_8x8x8[12][0]+1, c1d_z_8x8x8[12][1], c1d_z_8x8x8[12][2]), + offset8x8x8(c1d_z_8x8x8[13][0]+1, c1d_z_8x8x8[13][1], c1d_z_8x8x8[13][2]), + offset8x8x8(c1d_z_8x8x8[14][0]+1, c1d_z_8x8x8[14][1], c1d_z_8x8x8[14][2]), + offset8x8x8(c1d_z_8x8x8[15][0]+1, c1d_z_8x8x8[15][1], c1d_z_8x8x8[15][2]), + offset8x8x8(c1d_z_8x8x8[16][0]+1, c1d_z_8x8x8[16][1], c1d_z_8x8x8[16][2]), + offset8x8x8(c1d_z_8x8x8[17][0]+1, c1d_z_8x8x8[17][1], c1d_z_8x8x8[17][2]), + offset8x8x8(c1d_z_8x8x8[18][0]+1, c1d_z_8x8x8[18][1], c1d_z_8x8x8[18][2]), + offset8x8x8(c1d_z_8x8x8[19][0]+1, c1d_z_8x8x8[19][1], c1d_z_8x8x8[19][2]), + offset8x8x8(c1d_z_8x8x8[20][0]+1, c1d_z_8x8x8[20][1], c1d_z_8x8x8[20][2]), + offset8x8x8(c1d_z_8x8x8[21][0]+1, c1d_z_8x8x8[21][1], c1d_z_8x8x8[21][2]), + offset8x8x8(c1d_z_8x8x8[22][0]+1, c1d_z_8x8x8[22][1], c1d_z_8x8x8[22][2]), + offset8x8x8(c1d_z_8x8x8[23][0]+1, c1d_z_8x8x8[23][1], c1d_z_8x8x8[23][2]), + offset8x8x8(c1d_z_8x8x8[24][0]+1, c1d_z_8x8x8[24][1], c1d_z_8x8x8[24][2]), + offset8x8x8(c1d_z_8x8x8[25][0]+1, c1d_z_8x8x8[25][1], c1d_z_8x8x8[25][2]), + offset8x8x8(c1d_z_8x8x8[26][0]+1, c1d_z_8x8x8[26][1], c1d_z_8x8x8[26][2]), + offset8x8x8(c1d_z_8x8x8[27][0]+1, c1d_z_8x8x8[27][1], c1d_z_8x8x8[27][2]), + offset8x8x8(c1d_z_8x8x8[28][0]+1, c1d_z_8x8x8[28][1], c1d_z_8x8x8[28][2]), + offset8x8x8(c1d_z_8x8x8[29][0]+1, c1d_z_8x8x8[29][1], c1d_z_8x8x8[29][2]), + offset8x8x8(c1d_z_8x8x8[30][0]+1, c1d_z_8x8x8[30][1], c1d_z_8x8x8[30][2]), + offset8x8x8(c1d_z_8x8x8[31][0]+1, c1d_z_8x8x8[31][1], c1d_z_8x8x8[31][2]), + offset8x8x8(c1d_z_8x8x8[32][0]+1, c1d_z_8x8x8[32][1], c1d_z_8x8x8[32][2]), + offset8x8x8(c1d_z_8x8x8[33][0]+1, c1d_z_8x8x8[33][1], c1d_z_8x8x8[33][2]), + offset8x8x8(c1d_z_8x8x8[34][0]+1, c1d_z_8x8x8[34][1], c1d_z_8x8x8[34][2]), + offset8x8x8(c1d_z_8x8x8[35][0]+1, c1d_z_8x8x8[35][1], c1d_z_8x8x8[35][2]), + offset8x8x8(c1d_z_8x8x8[36][0]+1, c1d_z_8x8x8[36][1], c1d_z_8x8x8[36][2]), + offset8x8x8(c1d_z_8x8x8[37][0]+1, c1d_z_8x8x8[37][1], c1d_z_8x8x8[37][2]), + offset8x8x8(c1d_z_8x8x8[38][0]+1, c1d_z_8x8x8[38][1], c1d_z_8x8x8[38][2]), + offset8x8x8(c1d_z_8x8x8[39][0]+1, c1d_z_8x8x8[39][1], c1d_z_8x8x8[39][2]), + offset8x8x8(c1d_z_8x8x8[40][0]+1, c1d_z_8x8x8[40][1], c1d_z_8x8x8[40][2]), + offset8x8x8(c1d_z_8x8x8[41][0]+1, c1d_z_8x8x8[41][1], c1d_z_8x8x8[41][2]), + offset8x8x8(c1d_z_8x8x8[42][0]+1, c1d_z_8x8x8[42][1], c1d_z_8x8x8[42][2]), + offset8x8x8(c1d_z_8x8x8[43][0]+1, c1d_z_8x8x8[43][1], c1d_z_8x8x8[43][2]), + offset8x8x8(c1d_z_8x8x8[44][0]+1, c1d_z_8x8x8[44][1], c1d_z_8x8x8[44][2]), + offset8x8x8(c1d_z_8x8x8[45][0]+1, c1d_z_8x8x8[45][1], c1d_z_8x8x8[45][2]), + offset8x8x8(c1d_z_8x8x8[46][0]+1, c1d_z_8x8x8[46][1], c1d_z_8x8x8[46][2]), + offset8x8x8(c1d_z_8x8x8[47][0]+1, c1d_z_8x8x8[47][1], c1d_z_8x8x8[47][2]), + offset8x8x8(c1d_z_8x8x8[48][0]+1, c1d_z_8x8x8[48][1], c1d_z_8x8x8[48][2]), + offset8x8x8(c1d_z_8x8x8[49][0]+1, c1d_z_8x8x8[49][1], c1d_z_8x8x8[49][2]), + offset8x8x8(c1d_z_8x8x8[50][0]+1, c1d_z_8x8x8[50][1], c1d_z_8x8x8[50][2]), + offset8x8x8(c1d_z_8x8x8[51][0]+1, c1d_z_8x8x8[51][1], c1d_z_8x8x8[51][2]), + offset8x8x8(c1d_z_8x8x8[52][0]+1, c1d_z_8x8x8[52][1], c1d_z_8x8x8[52][2]), + offset8x8x8(c1d_z_8x8x8[53][0]+1, c1d_z_8x8x8[53][1], c1d_z_8x8x8[53][2]), + offset8x8x8(c1d_z_8x8x8[54][0]+1, c1d_z_8x8x8[54][1], c1d_z_8x8x8[54][2]), + offset8x8x8(c1d_z_8x8x8[55][0]+1, c1d_z_8x8x8[55][1], c1d_z_8x8x8[55][2]), + offset8x8x8(c1d_z_8x8x8[56][0]+1, c1d_z_8x8x8[56][1], c1d_z_8x8x8[56][2]), + offset8x8x8(c1d_z_8x8x8[57][0]+1, c1d_z_8x8x8[57][1], c1d_z_8x8x8[57][2]), + offset8x8x8(c1d_z_8x8x8[58][0]+1, c1d_z_8x8x8[58][1], c1d_z_8x8x8[58][2]), + offset8x8x8(c1d_z_8x8x8[59][0]+1, c1d_z_8x8x8[59][1], c1d_z_8x8x8[59][2]), + offset8x8x8(c1d_z_8x8x8[60][0]+1, c1d_z_8x8x8[60][1], c1d_z_8x8x8[60][2]), + offset8x8x8(c1d_z_8x8x8[61][0]+1, c1d_z_8x8x8[61][1], c1d_z_8x8x8[61][2]), + offset8x8x8(c1d_z_8x8x8[62][0]+1, c1d_z_8x8x8[62][1], c1d_z_8x8x8[62][2]), + offset8x8x8(c1d_z_8x8x8[63][0]+1, c1d_z_8x8x8[63][1], c1d_z_8x8x8[63][2]), + offset8x8x8(c1d_z_8x8x8[64][0]+1, c1d_z_8x8x8[64][1], c1d_z_8x8x8[64][2]), + offset8x8x8(c1d_z_8x8x8[65][0]+1, c1d_z_8x8x8[65][1], c1d_z_8x8x8[65][2]), + offset8x8x8(c1d_z_8x8x8[66][0]+1, c1d_z_8x8x8[66][1], c1d_z_8x8x8[66][2]), + offset8x8x8(c1d_z_8x8x8[67][0]+1, c1d_z_8x8x8[67][1], c1d_z_8x8x8[67][2]), + offset8x8x8(c1d_z_8x8x8[68][0]+1, c1d_z_8x8x8[68][1], c1d_z_8x8x8[68][2]), + offset8x8x8(c1d_z_8x8x8[69][0]+1, c1d_z_8x8x8[69][1], c1d_z_8x8x8[69][2]), + offset8x8x8(c1d_z_8x8x8[70][0]+1, c1d_z_8x8x8[70][1], c1d_z_8x8x8[70][2]), + offset8x8x8(c1d_z_8x8x8[71][0]+1, c1d_z_8x8x8[71][1], c1d_z_8x8x8[71][2]), + offset8x8x8(c1d_z_8x8x8[72][0]+1, c1d_z_8x8x8[72][1], c1d_z_8x8x8[72][2]), + offset8x8x8(c1d_z_8x8x8[73][0]+1, c1d_z_8x8x8[73][1], c1d_z_8x8x8[73][2]), + offset8x8x8(c1d_z_8x8x8[74][0]+1, c1d_z_8x8x8[74][1], c1d_z_8x8x8[74][2])}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff2D_MM_Offset_8x8x8(SIZE i) { + static constexpr int offset[135] = {offset8x8x8(c2d_xy_8x8x8[0][0], c2d_xy_8x8x8[0][1], c2d_xy_8x8x8[0][2]), // XY + offset8x8x8(c2d_xy_8x8x8[1][0], c2d_xy_8x8x8[1][1], c2d_xy_8x8x8[1][2]), + offset8x8x8(c2d_xy_8x8x8[2][0], c2d_xy_8x8x8[2][1], c2d_xy_8x8x8[2][2]), + offset8x8x8(c2d_xy_8x8x8[3][0], c2d_xy_8x8x8[3][1], c2d_xy_8x8x8[3][2]), + offset8x8x8(c2d_xy_8x8x8[4][0], c2d_xy_8x8x8[4][1], c2d_xy_8x8x8[4][2]), + offset8x8x8(c2d_xy_8x8x8[5][0], c2d_xy_8x8x8[5][1], c2d_xy_8x8x8[5][2]), + offset8x8x8(c2d_xy_8x8x8[6][0], c2d_xy_8x8x8[6][1], c2d_xy_8x8x8[6][2]), + offset8x8x8(c2d_xy_8x8x8[7][0], c2d_xy_8x8x8[7][1], c2d_xy_8x8x8[7][2]), + offset8x8x8(c2d_xy_8x8x8[8][0], c2d_xy_8x8x8[8][1], c2d_xy_8x8x8[8][2]), + offset8x8x8(c2d_xy_8x8x8[9][0], c2d_xy_8x8x8[9][1], c2d_xy_8x8x8[9][2]), + offset8x8x8(c2d_xy_8x8x8[10][0], c2d_xy_8x8x8[10][1], c2d_xy_8x8x8[10][2]), + offset8x8x8(c2d_xy_8x8x8[11][0], c2d_xy_8x8x8[11][1], c2d_xy_8x8x8[11][2]), + offset8x8x8(c2d_xy_8x8x8[12][0], c2d_xy_8x8x8[12][1], c2d_xy_8x8x8[12][2]), + offset8x8x8(c2d_xy_8x8x8[13][0], c2d_xy_8x8x8[13][1], c2d_xy_8x8x8[13][2]), + offset8x8x8(c2d_xy_8x8x8[14][0], c2d_xy_8x8x8[14][1], c2d_xy_8x8x8[14][2]), + offset8x8x8(c2d_xy_8x8x8[15][0], c2d_xy_8x8x8[15][1], c2d_xy_8x8x8[15][2]), + offset8x8x8(c2d_xy_8x8x8[16][0], c2d_xy_8x8x8[16][1], c2d_xy_8x8x8[16][2]), + offset8x8x8(c2d_xy_8x8x8[17][0], c2d_xy_8x8x8[17][1], c2d_xy_8x8x8[17][2]), + offset8x8x8(c2d_xy_8x8x8[18][0], c2d_xy_8x8x8[18][1], c2d_xy_8x8x8[18][2]), + offset8x8x8(c2d_xy_8x8x8[19][0], c2d_xy_8x8x8[19][1], c2d_xy_8x8x8[19][2]), + offset8x8x8(c2d_xy_8x8x8[20][0], c2d_xy_8x8x8[20][1], c2d_xy_8x8x8[20][2]), + offset8x8x8(c2d_xy_8x8x8[21][0], c2d_xy_8x8x8[21][1], c2d_xy_8x8x8[21][2]), + offset8x8x8(c2d_xy_8x8x8[22][0], c2d_xy_8x8x8[22][1], c2d_xy_8x8x8[22][2]), + offset8x8x8(c2d_xy_8x8x8[23][0], c2d_xy_8x8x8[23][1], c2d_xy_8x8x8[23][2]), + offset8x8x8(c2d_xy_8x8x8[24][0], c2d_xy_8x8x8[24][1], c2d_xy_8x8x8[24][2]), + offset8x8x8(c2d_xy_8x8x8[25][0], c2d_xy_8x8x8[25][1], c2d_xy_8x8x8[25][2]), + offset8x8x8(c2d_xy_8x8x8[26][0], c2d_xy_8x8x8[26][1], c2d_xy_8x8x8[26][2]), + offset8x8x8(c2d_xy_8x8x8[27][0], c2d_xy_8x8x8[27][1], c2d_xy_8x8x8[27][2]), + offset8x8x8(c2d_xy_8x8x8[28][0], c2d_xy_8x8x8[28][1], c2d_xy_8x8x8[28][2]), + offset8x8x8(c2d_xy_8x8x8[29][0], c2d_xy_8x8x8[29][1], c2d_xy_8x8x8[29][2]), + offset8x8x8(c2d_xy_8x8x8[30][0], c2d_xy_8x8x8[30][1], c2d_xy_8x8x8[30][2]), + offset8x8x8(c2d_xy_8x8x8[31][0], c2d_xy_8x8x8[31][1], c2d_xy_8x8x8[31][2]), + offset8x8x8(c2d_xy_8x8x8[32][0], c2d_xy_8x8x8[32][1], c2d_xy_8x8x8[32][2]), + offset8x8x8(c2d_xy_8x8x8[33][0], c2d_xy_8x8x8[33][1], c2d_xy_8x8x8[33][2]), + offset8x8x8(c2d_xy_8x8x8[34][0], c2d_xy_8x8x8[34][1], c2d_xy_8x8x8[34][2]), + offset8x8x8(c2d_xy_8x8x8[35][0], c2d_xy_8x8x8[35][1], c2d_xy_8x8x8[35][2]), + offset8x8x8(c2d_xy_8x8x8[36][0], c2d_xy_8x8x8[36][1], c2d_xy_8x8x8[36][2]), + offset8x8x8(c2d_xy_8x8x8[37][0], c2d_xy_8x8x8[37][1], c2d_xy_8x8x8[37][2]), + offset8x8x8(c2d_xy_8x8x8[38][0], c2d_xy_8x8x8[38][1], c2d_xy_8x8x8[38][2]), + offset8x8x8(c2d_xy_8x8x8[39][0], c2d_xy_8x8x8[39][1], c2d_xy_8x8x8[39][2]), + offset8x8x8(c2d_xy_8x8x8[40][0], c2d_xy_8x8x8[40][1], c2d_xy_8x8x8[40][2]), + offset8x8x8(c2d_xy_8x8x8[41][0], c2d_xy_8x8x8[41][1], c2d_xy_8x8x8[41][2]), + offset8x8x8(c2d_xy_8x8x8[42][0], c2d_xy_8x8x8[42][1], c2d_xy_8x8x8[42][2]), + offset8x8x8(c2d_xy_8x8x8[43][0], c2d_xy_8x8x8[43][1], c2d_xy_8x8x8[43][2]), + offset8x8x8(c2d_xy_8x8x8[44][0], c2d_xy_8x8x8[44][1], c2d_xy_8x8x8[44][2]), + + offset8x8x8(c2d_xz_8x8x8[0][0], c2d_xz_8x8x8[0][1], c2d_xz_8x8x8[0][2]), // XZ + offset8x8x8(c2d_xz_8x8x8[1][0], c2d_xz_8x8x8[1][1], c2d_xz_8x8x8[1][2]), + offset8x8x8(c2d_xz_8x8x8[2][0], c2d_xz_8x8x8[2][1], c2d_xz_8x8x8[2][2]), + offset8x8x8(c2d_xz_8x8x8[3][0], c2d_xz_8x8x8[3][1], c2d_xz_8x8x8[3][2]), + offset8x8x8(c2d_xz_8x8x8[4][0], c2d_xz_8x8x8[4][1], c2d_xz_8x8x8[4][2]), + offset8x8x8(c2d_xz_8x8x8[5][0], c2d_xz_8x8x8[5][1], c2d_xz_8x8x8[5][2]), + offset8x8x8(c2d_xz_8x8x8[6][0], c2d_xz_8x8x8[6][1], c2d_xz_8x8x8[6][2]), + offset8x8x8(c2d_xz_8x8x8[7][0], c2d_xz_8x8x8[7][1], c2d_xz_8x8x8[7][2]), + offset8x8x8(c2d_xz_8x8x8[8][0], c2d_xz_8x8x8[8][1], c2d_xz_8x8x8[8][2]), + offset8x8x8(c2d_xz_8x8x8[9][0], c2d_xz_8x8x8[9][1], c2d_xz_8x8x8[9][2]), + offset8x8x8(c2d_xz_8x8x8[10][0], c2d_xz_8x8x8[10][1], c2d_xz_8x8x8[10][2]), + offset8x8x8(c2d_xz_8x8x8[11][0], c2d_xz_8x8x8[11][1], c2d_xz_8x8x8[11][2]), + offset8x8x8(c2d_xz_8x8x8[12][0], c2d_xz_8x8x8[12][1], c2d_xz_8x8x8[12][2]), + offset8x8x8(c2d_xz_8x8x8[13][0], c2d_xz_8x8x8[13][1], c2d_xz_8x8x8[13][2]), + offset8x8x8(c2d_xz_8x8x8[14][0], c2d_xz_8x8x8[14][1], c2d_xz_8x8x8[14][2]), + offset8x8x8(c2d_xz_8x8x8[15][0], c2d_xz_8x8x8[15][1], c2d_xz_8x8x8[15][2]), + offset8x8x8(c2d_xz_8x8x8[16][0], c2d_xz_8x8x8[16][1], c2d_xz_8x8x8[16][2]), + offset8x8x8(c2d_xz_8x8x8[17][0], c2d_xz_8x8x8[17][1], c2d_xz_8x8x8[17][2]), + offset8x8x8(c2d_xz_8x8x8[18][0], c2d_xz_8x8x8[18][1], c2d_xz_8x8x8[18][2]), + offset8x8x8(c2d_xz_8x8x8[19][0], c2d_xz_8x8x8[19][1], c2d_xz_8x8x8[19][2]), + offset8x8x8(c2d_xz_8x8x8[20][0], c2d_xz_8x8x8[20][1], c2d_xz_8x8x8[20][2]), + offset8x8x8(c2d_xz_8x8x8[21][0], c2d_xz_8x8x8[21][1], c2d_xz_8x8x8[21][2]), + offset8x8x8(c2d_xz_8x8x8[22][0], c2d_xz_8x8x8[22][1], c2d_xz_8x8x8[22][2]), + offset8x8x8(c2d_xz_8x8x8[23][0], c2d_xz_8x8x8[23][1], c2d_xz_8x8x8[23][2]), + offset8x8x8(c2d_xz_8x8x8[24][0], c2d_xz_8x8x8[24][1], c2d_xz_8x8x8[24][2]), + offset8x8x8(c2d_xz_8x8x8[25][0], c2d_xz_8x8x8[25][1], c2d_xz_8x8x8[25][2]), + offset8x8x8(c2d_xz_8x8x8[26][0], c2d_xz_8x8x8[26][1], c2d_xz_8x8x8[26][2]), + offset8x8x8(c2d_xz_8x8x8[27][0], c2d_xz_8x8x8[27][1], c2d_xz_8x8x8[27][2]), + offset8x8x8(c2d_xz_8x8x8[28][0], c2d_xz_8x8x8[28][1], c2d_xz_8x8x8[28][2]), + offset8x8x8(c2d_xz_8x8x8[29][0], c2d_xz_8x8x8[29][1], c2d_xz_8x8x8[29][2]), + offset8x8x8(c2d_xz_8x8x8[30][0], c2d_xz_8x8x8[30][1], c2d_xz_8x8x8[30][2]), + offset8x8x8(c2d_xz_8x8x8[31][0], c2d_xz_8x8x8[31][1], c2d_xz_8x8x8[31][2]), + offset8x8x8(c2d_xz_8x8x8[32][0], c2d_xz_8x8x8[32][1], c2d_xz_8x8x8[32][2]), + offset8x8x8(c2d_xz_8x8x8[33][0], c2d_xz_8x8x8[33][1], c2d_xz_8x8x8[33][2]), + offset8x8x8(c2d_xz_8x8x8[34][0], c2d_xz_8x8x8[34][1], c2d_xz_8x8x8[34][2]), + offset8x8x8(c2d_xz_8x8x8[35][0], c2d_xz_8x8x8[35][1], c2d_xz_8x8x8[35][2]), + offset8x8x8(c2d_xz_8x8x8[36][0], c2d_xz_8x8x8[36][1], c2d_xz_8x8x8[36][2]), + offset8x8x8(c2d_xz_8x8x8[37][0], c2d_xz_8x8x8[37][1], c2d_xz_8x8x8[37][2]), + offset8x8x8(c2d_xz_8x8x8[38][0], c2d_xz_8x8x8[38][1], c2d_xz_8x8x8[38][2]), + offset8x8x8(c2d_xz_8x8x8[39][0], c2d_xz_8x8x8[39][1], c2d_xz_8x8x8[39][2]), + offset8x8x8(c2d_xz_8x8x8[40][0], c2d_xz_8x8x8[40][1], c2d_xz_8x8x8[40][2]), + offset8x8x8(c2d_xz_8x8x8[41][0], c2d_xz_8x8x8[41][1], c2d_xz_8x8x8[41][2]), + offset8x8x8(c2d_xz_8x8x8[42][0], c2d_xz_8x8x8[42][1], c2d_xz_8x8x8[42][2]), + offset8x8x8(c2d_xz_8x8x8[43][0], c2d_xz_8x8x8[43][1], c2d_xz_8x8x8[43][2]), + offset8x8x8(c2d_xz_8x8x8[44][0], c2d_xz_8x8x8[44][1], c2d_xz_8x8x8[44][2]), + + offset8x8x8(c2d_yz_8x8x8[0][0], c2d_yz_8x8x8[0][1], c2d_yz_8x8x8[0][2]), // YZ + offset8x8x8(c2d_yz_8x8x8[1][0], c2d_yz_8x8x8[1][1], c2d_yz_8x8x8[1][2]), + offset8x8x8(c2d_yz_8x8x8[2][0], c2d_yz_8x8x8[2][1], c2d_yz_8x8x8[2][2]), + offset8x8x8(c2d_yz_8x8x8[3][0], c2d_yz_8x8x8[3][1], c2d_yz_8x8x8[3][2]), + offset8x8x8(c2d_yz_8x8x8[4][0], c2d_yz_8x8x8[4][1], c2d_yz_8x8x8[4][2]), + offset8x8x8(c2d_yz_8x8x8[5][0], c2d_yz_8x8x8[5][1], c2d_yz_8x8x8[5][2]), + offset8x8x8(c2d_yz_8x8x8[6][0], c2d_yz_8x8x8[6][1], c2d_yz_8x8x8[6][2]), + offset8x8x8(c2d_yz_8x8x8[7][0], c2d_yz_8x8x8[7][1], c2d_yz_8x8x8[7][2]), + offset8x8x8(c2d_yz_8x8x8[8][0], c2d_yz_8x8x8[8][1], c2d_yz_8x8x8[8][2]), + offset8x8x8(c2d_yz_8x8x8[9][0], c2d_yz_8x8x8[9][1], c2d_yz_8x8x8[9][2]), + offset8x8x8(c2d_yz_8x8x8[10][0], c2d_yz_8x8x8[10][1], c2d_yz_8x8x8[10][2]), + offset8x8x8(c2d_yz_8x8x8[11][0], c2d_yz_8x8x8[11][1], c2d_yz_8x8x8[11][2]), + offset8x8x8(c2d_yz_8x8x8[12][0], c2d_yz_8x8x8[12][1], c2d_yz_8x8x8[12][2]), + offset8x8x8(c2d_yz_8x8x8[13][0], c2d_yz_8x8x8[13][1], c2d_yz_8x8x8[13][2]), + offset8x8x8(c2d_yz_8x8x8[14][0], c2d_yz_8x8x8[14][1], c2d_yz_8x8x8[14][2]), + offset8x8x8(c2d_yz_8x8x8[15][0], c2d_yz_8x8x8[15][1], c2d_yz_8x8x8[15][2]), + offset8x8x8(c2d_yz_8x8x8[16][0], c2d_yz_8x8x8[16][1], c2d_yz_8x8x8[16][2]), + offset8x8x8(c2d_yz_8x8x8[17][0], c2d_yz_8x8x8[17][1], c2d_yz_8x8x8[17][2]), + offset8x8x8(c2d_yz_8x8x8[18][0], c2d_yz_8x8x8[18][1], c2d_yz_8x8x8[18][2]), + offset8x8x8(c2d_yz_8x8x8[19][0], c2d_yz_8x8x8[19][1], c2d_yz_8x8x8[19][2]), + offset8x8x8(c2d_yz_8x8x8[20][0], c2d_yz_8x8x8[20][1], c2d_yz_8x8x8[20][2]), + offset8x8x8(c2d_yz_8x8x8[21][0], c2d_yz_8x8x8[21][1], c2d_yz_8x8x8[21][2]), + offset8x8x8(c2d_yz_8x8x8[22][0], c2d_yz_8x8x8[22][1], c2d_yz_8x8x8[22][2]), + offset8x8x8(c2d_yz_8x8x8[23][0], c2d_yz_8x8x8[23][1], c2d_yz_8x8x8[23][2]), + offset8x8x8(c2d_yz_8x8x8[24][0], c2d_yz_8x8x8[24][1], c2d_yz_8x8x8[24][2]), + offset8x8x8(c2d_yz_8x8x8[25][0], c2d_yz_8x8x8[25][1], c2d_yz_8x8x8[25][2]), + offset8x8x8(c2d_yz_8x8x8[26][0], c2d_yz_8x8x8[26][1], c2d_yz_8x8x8[26][2]), + offset8x8x8(c2d_yz_8x8x8[27][0], c2d_yz_8x8x8[27][1], c2d_yz_8x8x8[27][2]), + offset8x8x8(c2d_yz_8x8x8[28][0], c2d_yz_8x8x8[28][1], c2d_yz_8x8x8[28][2]), + offset8x8x8(c2d_yz_8x8x8[29][0], c2d_yz_8x8x8[29][1], c2d_yz_8x8x8[29][2]), + offset8x8x8(c2d_yz_8x8x8[30][0], c2d_yz_8x8x8[30][1], c2d_yz_8x8x8[30][2]), + offset8x8x8(c2d_yz_8x8x8[31][0], c2d_yz_8x8x8[31][1], c2d_yz_8x8x8[31][2]), + offset8x8x8(c2d_yz_8x8x8[32][0], c2d_yz_8x8x8[32][1], c2d_yz_8x8x8[32][2]), + offset8x8x8(c2d_yz_8x8x8[33][0], c2d_yz_8x8x8[33][1], c2d_yz_8x8x8[33][2]), + offset8x8x8(c2d_yz_8x8x8[34][0], c2d_yz_8x8x8[34][1], c2d_yz_8x8x8[34][2]), + offset8x8x8(c2d_yz_8x8x8[35][0], c2d_yz_8x8x8[35][1], c2d_yz_8x8x8[35][2]), + offset8x8x8(c2d_yz_8x8x8[36][0], c2d_yz_8x8x8[36][1], c2d_yz_8x8x8[36][2]), + offset8x8x8(c2d_yz_8x8x8[37][0], c2d_yz_8x8x8[37][1], c2d_yz_8x8x8[37][2]), + offset8x8x8(c2d_yz_8x8x8[38][0], c2d_yz_8x8x8[38][1], c2d_yz_8x8x8[38][2]), + offset8x8x8(c2d_yz_8x8x8[39][0], c2d_yz_8x8x8[39][1], c2d_yz_8x8x8[39][2]), + offset8x8x8(c2d_yz_8x8x8[40][0], c2d_yz_8x8x8[40][1], c2d_yz_8x8x8[40][2]), + offset8x8x8(c2d_yz_8x8x8[41][0], c2d_yz_8x8x8[41][1], c2d_yz_8x8x8[41][2]), + offset8x8x8(c2d_yz_8x8x8[42][0], c2d_yz_8x8x8[42][1], c2d_yz_8x8x8[42][2]), + offset8x8x8(c2d_yz_8x8x8[43][0], c2d_yz_8x8x8[43][1], c2d_yz_8x8x8[43][2]), + offset8x8x8(c2d_yz_8x8x8[44][0], c2d_yz_8x8x8[44][1], c2d_yz_8x8x8[44][2])}; + return offset[i]; +} + +MGARDX_EXEC int Coeff2D_RL_Offset_8x8x8(SIZE i) { + static constexpr int offset[135] = {offset8x8x8(c2d_xy_8x8x8[0][0], c2d_xy_8x8x8[0][1]+1, c2d_xy_8x8x8[0][2]-1), // XY + offset8x8x8(c2d_xy_8x8x8[1][0], c2d_xy_8x8x8[1][1]+1, c2d_xy_8x8x8[1][2]-1), + offset8x8x8(c2d_xy_8x8x8[2][0], c2d_xy_8x8x8[2][1]+1, c2d_xy_8x8x8[2][2]-1), + offset8x8x8(c2d_xy_8x8x8[3][0], c2d_xy_8x8x8[3][1]+1, c2d_xy_8x8x8[3][2]-1), + offset8x8x8(c2d_xy_8x8x8[4][0], c2d_xy_8x8x8[4][1]+1, c2d_xy_8x8x8[4][2]-1), + offset8x8x8(c2d_xy_8x8x8[5][0], c2d_xy_8x8x8[5][1]+1, c2d_xy_8x8x8[5][2]-1), + offset8x8x8(c2d_xy_8x8x8[6][0], c2d_xy_8x8x8[6][1]+1, c2d_xy_8x8x8[6][2]-1), + offset8x8x8(c2d_xy_8x8x8[7][0], c2d_xy_8x8x8[7][1]+1, c2d_xy_8x8x8[7][2]-1), + offset8x8x8(c2d_xy_8x8x8[8][0], c2d_xy_8x8x8[8][1]+1, c2d_xy_8x8x8[8][2]-1), + offset8x8x8(c2d_xy_8x8x8[9][0], c2d_xy_8x8x8[9][1]+1, c2d_xy_8x8x8[9][2]-1), + offset8x8x8(c2d_xy_8x8x8[10][0], c2d_xy_8x8x8[10][1]+1, c2d_xy_8x8x8[10][2]-1), + offset8x8x8(c2d_xy_8x8x8[11][0], c2d_xy_8x8x8[11][1]+1, c2d_xy_8x8x8[11][2]-1), + offset8x8x8(c2d_xy_8x8x8[12][0], c2d_xy_8x8x8[12][1]+1, c2d_xy_8x8x8[12][2]-1), + offset8x8x8(c2d_xy_8x8x8[13][0], c2d_xy_8x8x8[13][1]+1, c2d_xy_8x8x8[13][2]-1), + offset8x8x8(c2d_xy_8x8x8[14][0], c2d_xy_8x8x8[14][1]+1, c2d_xy_8x8x8[14][2]-1), + offset8x8x8(c2d_xy_8x8x8[15][0], c2d_xy_8x8x8[15][1]+1, c2d_xy_8x8x8[15][2]-1), + offset8x8x8(c2d_xy_8x8x8[16][0], c2d_xy_8x8x8[16][1]+1, c2d_xy_8x8x8[16][2]-1), + offset8x8x8(c2d_xy_8x8x8[17][0], c2d_xy_8x8x8[17][1]+1, c2d_xy_8x8x8[17][2]-1), + offset8x8x8(c2d_xy_8x8x8[18][0], c2d_xy_8x8x8[18][1]+1, c2d_xy_8x8x8[18][2]-1), + offset8x8x8(c2d_xy_8x8x8[19][0], c2d_xy_8x8x8[19][1]+1, c2d_xy_8x8x8[19][2]-1), + offset8x8x8(c2d_xy_8x8x8[20][0], c2d_xy_8x8x8[20][1]+1, c2d_xy_8x8x8[20][2]-1), + offset8x8x8(c2d_xy_8x8x8[21][0], c2d_xy_8x8x8[21][1]+1, c2d_xy_8x8x8[21][2]-1), + offset8x8x8(c2d_xy_8x8x8[22][0], c2d_xy_8x8x8[22][1]+1, c2d_xy_8x8x8[22][2]-1), + offset8x8x8(c2d_xy_8x8x8[23][0], c2d_xy_8x8x8[23][1]+1, c2d_xy_8x8x8[23][2]-1), + offset8x8x8(c2d_xy_8x8x8[24][0], c2d_xy_8x8x8[24][1]+1, c2d_xy_8x8x8[24][2]-1), + offset8x8x8(c2d_xy_8x8x8[25][0], c2d_xy_8x8x8[25][1]+1, c2d_xy_8x8x8[25][2]-1), + offset8x8x8(c2d_xy_8x8x8[26][0], c2d_xy_8x8x8[26][1]+1, c2d_xy_8x8x8[26][2]-1), + offset8x8x8(c2d_xy_8x8x8[27][0], c2d_xy_8x8x8[27][1]+1, c2d_xy_8x8x8[27][2]-1), + offset8x8x8(c2d_xy_8x8x8[28][0], c2d_xy_8x8x8[28][1]+1, c2d_xy_8x8x8[28][2]-1), + offset8x8x8(c2d_xy_8x8x8[29][0], c2d_xy_8x8x8[29][1]+1, c2d_xy_8x8x8[29][2]-1), + offset8x8x8(c2d_xy_8x8x8[30][0], c2d_xy_8x8x8[30][1]+1, c2d_xy_8x8x8[30][2]-1), + offset8x8x8(c2d_xy_8x8x8[31][0], c2d_xy_8x8x8[31][1]+1, c2d_xy_8x8x8[31][2]-1), + offset8x8x8(c2d_xy_8x8x8[32][0], c2d_xy_8x8x8[32][1]+1, c2d_xy_8x8x8[32][2]-1), + offset8x8x8(c2d_xy_8x8x8[33][0], c2d_xy_8x8x8[33][1]+1, c2d_xy_8x8x8[33][2]-1), + offset8x8x8(c2d_xy_8x8x8[34][0], c2d_xy_8x8x8[34][1]+1, c2d_xy_8x8x8[34][2]-1), + offset8x8x8(c2d_xy_8x8x8[35][0], c2d_xy_8x8x8[35][1]+1, c2d_xy_8x8x8[35][2]-1), + offset8x8x8(c2d_xy_8x8x8[36][0], c2d_xy_8x8x8[36][1]+1, c2d_xy_8x8x8[36][2]-1), + offset8x8x8(c2d_xy_8x8x8[37][0], c2d_xy_8x8x8[37][1]+1, c2d_xy_8x8x8[37][2]-1), + offset8x8x8(c2d_xy_8x8x8[38][0], c2d_xy_8x8x8[38][1]+1, c2d_xy_8x8x8[38][2]-1), + offset8x8x8(c2d_xy_8x8x8[39][0], c2d_xy_8x8x8[39][1]+1, c2d_xy_8x8x8[39][2]-1), + offset8x8x8(c2d_xy_8x8x8[40][0], c2d_xy_8x8x8[40][1]+1, c2d_xy_8x8x8[40][2]-1), + offset8x8x8(c2d_xy_8x8x8[41][0], c2d_xy_8x8x8[41][1]+1, c2d_xy_8x8x8[41][2]-1), + offset8x8x8(c2d_xy_8x8x8[42][0], c2d_xy_8x8x8[42][1]+1, c2d_xy_8x8x8[42][2]-1), + offset8x8x8(c2d_xy_8x8x8[43][0], c2d_xy_8x8x8[43][1]+1, c2d_xy_8x8x8[43][2]-1), + offset8x8x8(c2d_xy_8x8x8[44][0], c2d_xy_8x8x8[44][1]+1, c2d_xy_8x8x8[44][2]-1), + + offset8x8x8(c2d_xz_8x8x8[0][0]+1, c2d_xz_8x8x8[0][1], c2d_xz_8x8x8[0][2]-1), // XZ + offset8x8x8(c2d_xz_8x8x8[1][0]+1, c2d_xz_8x8x8[1][1], c2d_xz_8x8x8[1][2]-1), + offset8x8x8(c2d_xz_8x8x8[2][0]+1, c2d_xz_8x8x8[2][1], c2d_xz_8x8x8[2][2]-1), + offset8x8x8(c2d_xz_8x8x8[3][0]+1, c2d_xz_8x8x8[3][1], c2d_xz_8x8x8[3][2]-1), + offset8x8x8(c2d_xz_8x8x8[4][0]+1, c2d_xz_8x8x8[4][1], c2d_xz_8x8x8[4][2]-1), + offset8x8x8(c2d_xz_8x8x8[5][0]+1, c2d_xz_8x8x8[5][1], c2d_xz_8x8x8[5][2]-1), + offset8x8x8(c2d_xz_8x8x8[6][0]+1, c2d_xz_8x8x8[6][1], c2d_xz_8x8x8[6][2]-1), + offset8x8x8(c2d_xz_8x8x8[7][0]+1, c2d_xz_8x8x8[7][1], c2d_xz_8x8x8[7][2]-1), + offset8x8x8(c2d_xz_8x8x8[8][0]+1, c2d_xz_8x8x8[8][1], c2d_xz_8x8x8[8][2]-1), + offset8x8x8(c2d_xz_8x8x8[9][0]+1, c2d_xz_8x8x8[9][1], c2d_xz_8x8x8[9][2]-1), + offset8x8x8(c2d_xz_8x8x8[10][0]+1, c2d_xz_8x8x8[10][1], c2d_xz_8x8x8[10][2]-1), + offset8x8x8(c2d_xz_8x8x8[11][0]+1, c2d_xz_8x8x8[11][1], c2d_xz_8x8x8[11][2]-1), + offset8x8x8(c2d_xz_8x8x8[12][0]+1, c2d_xz_8x8x8[12][1], c2d_xz_8x8x8[12][2]-1), + offset8x8x8(c2d_xz_8x8x8[13][0]+1, c2d_xz_8x8x8[13][1], c2d_xz_8x8x8[13][2]-1), + offset8x8x8(c2d_xz_8x8x8[14][0]+1, c2d_xz_8x8x8[14][1], c2d_xz_8x8x8[14][2]-1), + offset8x8x8(c2d_xz_8x8x8[15][0]+1, c2d_xz_8x8x8[15][1], c2d_xz_8x8x8[15][2]-1), + offset8x8x8(c2d_xz_8x8x8[16][0]+1, c2d_xz_8x8x8[16][1], c2d_xz_8x8x8[16][2]-1), + offset8x8x8(c2d_xz_8x8x8[17][0]+1, c2d_xz_8x8x8[17][1], c2d_xz_8x8x8[17][2]-1), + offset8x8x8(c2d_xz_8x8x8[18][0]+1, c2d_xz_8x8x8[18][1], c2d_xz_8x8x8[18][2]-1), + offset8x8x8(c2d_xz_8x8x8[19][0]+1, c2d_xz_8x8x8[19][1], c2d_xz_8x8x8[19][2]-1), + offset8x8x8(c2d_xz_8x8x8[20][0]+1, c2d_xz_8x8x8[20][1], c2d_xz_8x8x8[20][2]-1), + offset8x8x8(c2d_xz_8x8x8[21][0]+1, c2d_xz_8x8x8[21][1], c2d_xz_8x8x8[21][2]-1), + offset8x8x8(c2d_xz_8x8x8[22][0]+1, c2d_xz_8x8x8[22][1], c2d_xz_8x8x8[22][2]-1), + offset8x8x8(c2d_xz_8x8x8[23][0]+1, c2d_xz_8x8x8[23][1], c2d_xz_8x8x8[23][2]-1), + offset8x8x8(c2d_xz_8x8x8[24][0]+1, c2d_xz_8x8x8[24][1], c2d_xz_8x8x8[24][2]-1), + offset8x8x8(c2d_xz_8x8x8[25][0]+1, c2d_xz_8x8x8[25][1], c2d_xz_8x8x8[25][2]-1), + offset8x8x8(c2d_xz_8x8x8[26][0]+1, c2d_xz_8x8x8[26][1], c2d_xz_8x8x8[26][2]-1), + offset8x8x8(c2d_xz_8x8x8[27][0]+1, c2d_xz_8x8x8[27][1], c2d_xz_8x8x8[27][2]-1), + offset8x8x8(c2d_xz_8x8x8[28][0]+1, c2d_xz_8x8x8[28][1], c2d_xz_8x8x8[28][2]-1), + offset8x8x8(c2d_xz_8x8x8[29][0]+1, c2d_xz_8x8x8[29][1], c2d_xz_8x8x8[29][2]-1), + offset8x8x8(c2d_xz_8x8x8[30][0]+1, c2d_xz_8x8x8[30][1], c2d_xz_8x8x8[30][2]-1), + offset8x8x8(c2d_xz_8x8x8[31][0]+1, c2d_xz_8x8x8[31][1], c2d_xz_8x8x8[31][2]-1), + offset8x8x8(c2d_xz_8x8x8[32][0]+1, c2d_xz_8x8x8[32][1], c2d_xz_8x8x8[32][2]-1), + offset8x8x8(c2d_xz_8x8x8[33][0]+1, c2d_xz_8x8x8[33][1], c2d_xz_8x8x8[33][2]-1), + offset8x8x8(c2d_xz_8x8x8[34][0]+1, c2d_xz_8x8x8[34][1], c2d_xz_8x8x8[34][2]-1), + offset8x8x8(c2d_xz_8x8x8[35][0]+1, c2d_xz_8x8x8[35][1], c2d_xz_8x8x8[35][2]-1), + offset8x8x8(c2d_xz_8x8x8[36][0]+1, c2d_xz_8x8x8[36][1], c2d_xz_8x8x8[36][2]-1), + offset8x8x8(c2d_xz_8x8x8[37][0]+1, c2d_xz_8x8x8[37][1], c2d_xz_8x8x8[37][2]-1), + offset8x8x8(c2d_xz_8x8x8[38][0]+1, c2d_xz_8x8x8[38][1], c2d_xz_8x8x8[38][2]-1), + offset8x8x8(c2d_xz_8x8x8[39][0]+1, c2d_xz_8x8x8[39][1], c2d_xz_8x8x8[39][2]-1), + offset8x8x8(c2d_xz_8x8x8[40][0]+1, c2d_xz_8x8x8[40][1], c2d_xz_8x8x8[40][2]-1), + offset8x8x8(c2d_xz_8x8x8[41][0]+1, c2d_xz_8x8x8[41][1], c2d_xz_8x8x8[41][2]-1), + offset8x8x8(c2d_xz_8x8x8[42][0]+1, c2d_xz_8x8x8[42][1], c2d_xz_8x8x8[42][2]-1), + offset8x8x8(c2d_xz_8x8x8[43][0]+1, c2d_xz_8x8x8[43][1], c2d_xz_8x8x8[43][2]-1), + offset8x8x8(c2d_xz_8x8x8[44][0]+1, c2d_xz_8x8x8[44][1], c2d_xz_8x8x8[44][2]-1), + + offset8x8x8(c2d_yz_8x8x8[0][0]+1, c2d_yz_8x8x8[0][1]-1, c2d_yz_8x8x8[0][2]), // YZ + offset8x8x8(c2d_yz_8x8x8[1][0]+1, c2d_yz_8x8x8[1][1]-1, c2d_yz_8x8x8[1][2]), + offset8x8x8(c2d_yz_8x8x8[2][0]+1, c2d_yz_8x8x8[2][1]-1, c2d_yz_8x8x8[2][2]), + offset8x8x8(c2d_yz_8x8x8[3][0]+1, c2d_yz_8x8x8[3][1]-1, c2d_yz_8x8x8[3][2]), + offset8x8x8(c2d_yz_8x8x8[4][0]+1, c2d_yz_8x8x8[4][1]-1, c2d_yz_8x8x8[4][2]), + offset8x8x8(c2d_yz_8x8x8[5][0]+1, c2d_yz_8x8x8[5][1]-1, c2d_yz_8x8x8[5][2]), + offset8x8x8(c2d_yz_8x8x8[6][0]+1, c2d_yz_8x8x8[6][1]-1, c2d_yz_8x8x8[6][2]), + offset8x8x8(c2d_yz_8x8x8[7][0]+1, c2d_yz_8x8x8[7][1]-1, c2d_yz_8x8x8[7][2]), + offset8x8x8(c2d_yz_8x8x8[8][0]+1, c2d_yz_8x8x8[8][1]-1, c2d_yz_8x8x8[8][2]), + offset8x8x8(c2d_yz_8x8x8[9][0]+1, c2d_yz_8x8x8[9][1]-1, c2d_yz_8x8x8[9][2]), + offset8x8x8(c2d_yz_8x8x8[10][0]+1, c2d_yz_8x8x8[10][1]-1, c2d_yz_8x8x8[10][2]), + offset8x8x8(c2d_yz_8x8x8[11][0]+1, c2d_yz_8x8x8[11][1]-1, c2d_yz_8x8x8[11][2]), + offset8x8x8(c2d_yz_8x8x8[12][0]+1, c2d_yz_8x8x8[12][1]-1, c2d_yz_8x8x8[12][2]), + offset8x8x8(c2d_yz_8x8x8[13][0]+1, c2d_yz_8x8x8[13][1]-1, c2d_yz_8x8x8[13][2]), + offset8x8x8(c2d_yz_8x8x8[14][0]+1, c2d_yz_8x8x8[14][1]-1, c2d_yz_8x8x8[14][2]), + offset8x8x8(c2d_yz_8x8x8[15][0]+1, c2d_yz_8x8x8[15][1]-1, c2d_yz_8x8x8[15][2]), + offset8x8x8(c2d_yz_8x8x8[16][0]+1, c2d_yz_8x8x8[16][1]-1, c2d_yz_8x8x8[16][2]), + offset8x8x8(c2d_yz_8x8x8[17][0]+1, c2d_yz_8x8x8[17][1]-1, c2d_yz_8x8x8[17][2]), + offset8x8x8(c2d_yz_8x8x8[18][0]+1, c2d_yz_8x8x8[18][1]-1, c2d_yz_8x8x8[18][2]), + offset8x8x8(c2d_yz_8x8x8[19][0]+1, c2d_yz_8x8x8[19][1]-1, c2d_yz_8x8x8[19][2]), + offset8x8x8(c2d_yz_8x8x8[20][0]+1, c2d_yz_8x8x8[20][1]-1, c2d_yz_8x8x8[20][2]), + offset8x8x8(c2d_yz_8x8x8[21][0]+1, c2d_yz_8x8x8[21][1]-1, c2d_yz_8x8x8[21][2]), + offset8x8x8(c2d_yz_8x8x8[22][0]+1, c2d_yz_8x8x8[22][1]-1, c2d_yz_8x8x8[22][2]), + offset8x8x8(c2d_yz_8x8x8[23][0]+1, c2d_yz_8x8x8[23][1]-1, c2d_yz_8x8x8[23][2]), + offset8x8x8(c2d_yz_8x8x8[24][0]+1, c2d_yz_8x8x8[24][1]-1, c2d_yz_8x8x8[24][2]), + offset8x8x8(c2d_yz_8x8x8[25][0]+1, c2d_yz_8x8x8[25][1]-1, c2d_yz_8x8x8[25][2]), + offset8x8x8(c2d_yz_8x8x8[26][0]+1, c2d_yz_8x8x8[26][1]-1, c2d_yz_8x8x8[26][2]), + offset8x8x8(c2d_yz_8x8x8[27][0]+1, c2d_yz_8x8x8[27][1]-1, c2d_yz_8x8x8[27][2]), + offset8x8x8(c2d_yz_8x8x8[28][0]+1, c2d_yz_8x8x8[28][1]-1, c2d_yz_8x8x8[28][2]), + offset8x8x8(c2d_yz_8x8x8[29][0]+1, c2d_yz_8x8x8[29][1]-1, c2d_yz_8x8x8[29][2]), + offset8x8x8(c2d_yz_8x8x8[30][0]+1, c2d_yz_8x8x8[30][1]-1, c2d_yz_8x8x8[30][2]), + offset8x8x8(c2d_yz_8x8x8[31][0]+1, c2d_yz_8x8x8[31][1]-1, c2d_yz_8x8x8[31][2]), + offset8x8x8(c2d_yz_8x8x8[32][0]+1, c2d_yz_8x8x8[32][1]-1, c2d_yz_8x8x8[32][2]), + offset8x8x8(c2d_yz_8x8x8[33][0]+1, c2d_yz_8x8x8[33][1]-1, c2d_yz_8x8x8[33][2]), + offset8x8x8(c2d_yz_8x8x8[34][0]+1, c2d_yz_8x8x8[34][1]-1, c2d_yz_8x8x8[34][2]), + offset8x8x8(c2d_yz_8x8x8[35][0]+1, c2d_yz_8x8x8[35][1]-1, c2d_yz_8x8x8[35][2]), + offset8x8x8(c2d_yz_8x8x8[36][0]+1, c2d_yz_8x8x8[36][1]-1, c2d_yz_8x8x8[36][2]), + offset8x8x8(c2d_yz_8x8x8[37][0]+1, c2d_yz_8x8x8[37][1]-1, c2d_yz_8x8x8[37][2]), + offset8x8x8(c2d_yz_8x8x8[38][0]+1, c2d_yz_8x8x8[38][1]-1, c2d_yz_8x8x8[38][2]), + offset8x8x8(c2d_yz_8x8x8[39][0]+1, c2d_yz_8x8x8[39][1]-1, c2d_yz_8x8x8[39][2]), + offset8x8x8(c2d_yz_8x8x8[40][0]+1, c2d_yz_8x8x8[40][1]-1, c2d_yz_8x8x8[40][2]), + offset8x8x8(c2d_yz_8x8x8[41][0]+1, c2d_yz_8x8x8[41][1]-1, c2d_yz_8x8x8[41][2]), + offset8x8x8(c2d_yz_8x8x8[42][0]+1, c2d_yz_8x8x8[42][1]-1, c2d_yz_8x8x8[42][2]), + offset8x8x8(c2d_yz_8x8x8[43][0]+1, c2d_yz_8x8x8[43][1]-1, c2d_yz_8x8x8[43][2]), + offset8x8x8(c2d_yz_8x8x8[44][0]+1, c2d_yz_8x8x8[44][1]-1, c2d_yz_8x8x8[44][2])}; + return offset[i]; +} + +MGARDX_EXEC int Coeff2D_LR_Offset_8x8x8(SIZE i) { + static constexpr int offset[135] = {offset8x8x8(c2d_xy_8x8x8[0][0], c2d_xy_8x8x8[0][1]-1, c2d_xy_8x8x8[0][2]+1), // XY + offset8x8x8(c2d_xy_8x8x8[1][0], c2d_xy_8x8x8[1][1]-1, c2d_xy_8x8x8[1][2]+1), + offset8x8x8(c2d_xy_8x8x8[2][0], c2d_xy_8x8x8[2][1]-1, c2d_xy_8x8x8[2][2]+1), + offset8x8x8(c2d_xy_8x8x8[3][0], c2d_xy_8x8x8[3][1]-1, c2d_xy_8x8x8[3][2]+1), + offset8x8x8(c2d_xy_8x8x8[4][0], c2d_xy_8x8x8[4][1]-1, c2d_xy_8x8x8[4][2]+1), + offset8x8x8(c2d_xy_8x8x8[5][0], c2d_xy_8x8x8[5][1]-1, c2d_xy_8x8x8[5][2]+1), + offset8x8x8(c2d_xy_8x8x8[6][0], c2d_xy_8x8x8[6][1]-1, c2d_xy_8x8x8[6][2]+1), + offset8x8x8(c2d_xy_8x8x8[7][0], c2d_xy_8x8x8[7][1]-1, c2d_xy_8x8x8[7][2]+1), + offset8x8x8(c2d_xy_8x8x8[8][0], c2d_xy_8x8x8[8][1]-1, c2d_xy_8x8x8[8][2]+1), + offset8x8x8(c2d_xy_8x8x8[9][0], c2d_xy_8x8x8[9][1]-1, c2d_xy_8x8x8[9][2]+1), + offset8x8x8(c2d_xy_8x8x8[10][0], c2d_xy_8x8x8[10][1]-1, c2d_xy_8x8x8[10][2]+1), + offset8x8x8(c2d_xy_8x8x8[11][0], c2d_xy_8x8x8[11][1]-1, c2d_xy_8x8x8[11][2]+1), + offset8x8x8(c2d_xy_8x8x8[12][0], c2d_xy_8x8x8[12][1]-1, c2d_xy_8x8x8[12][2]+1), + offset8x8x8(c2d_xy_8x8x8[13][0], c2d_xy_8x8x8[13][1]-1, c2d_xy_8x8x8[13][2]+1), + offset8x8x8(c2d_xy_8x8x8[14][0], c2d_xy_8x8x8[14][1]-1, c2d_xy_8x8x8[14][2]+1), + offset8x8x8(c2d_xy_8x8x8[15][0], c2d_xy_8x8x8[15][1]-1, c2d_xy_8x8x8[15][2]+1), + offset8x8x8(c2d_xy_8x8x8[16][0], c2d_xy_8x8x8[16][1]-1, c2d_xy_8x8x8[16][2]+1), + offset8x8x8(c2d_xy_8x8x8[17][0], c2d_xy_8x8x8[17][1]-1, c2d_xy_8x8x8[17][2]+1), + offset8x8x8(c2d_xy_8x8x8[18][0], c2d_xy_8x8x8[18][1]-1, c2d_xy_8x8x8[18][2]+1), + offset8x8x8(c2d_xy_8x8x8[19][0], c2d_xy_8x8x8[19][1]-1, c2d_xy_8x8x8[19][2]+1), + offset8x8x8(c2d_xy_8x8x8[20][0], c2d_xy_8x8x8[20][1]-1, c2d_xy_8x8x8[20][2]+1), + offset8x8x8(c2d_xy_8x8x8[21][0], c2d_xy_8x8x8[21][1]-1, c2d_xy_8x8x8[21][2]+1), + offset8x8x8(c2d_xy_8x8x8[22][0], c2d_xy_8x8x8[22][1]-1, c2d_xy_8x8x8[22][2]+1), + offset8x8x8(c2d_xy_8x8x8[23][0], c2d_xy_8x8x8[23][1]-1, c2d_xy_8x8x8[23][2]+1), + offset8x8x8(c2d_xy_8x8x8[24][0], c2d_xy_8x8x8[24][1]-1, c2d_xy_8x8x8[24][2]+1), + offset8x8x8(c2d_xy_8x8x8[25][0], c2d_xy_8x8x8[25][1]-1, c2d_xy_8x8x8[25][2]+1), + offset8x8x8(c2d_xy_8x8x8[26][0], c2d_xy_8x8x8[26][1]-1, c2d_xy_8x8x8[26][2]+1), + offset8x8x8(c2d_xy_8x8x8[27][0], c2d_xy_8x8x8[27][1]-1, c2d_xy_8x8x8[27][2]+1), + offset8x8x8(c2d_xy_8x8x8[28][0], c2d_xy_8x8x8[28][1]-1, c2d_xy_8x8x8[28][2]+1), + offset8x8x8(c2d_xy_8x8x8[29][0], c2d_xy_8x8x8[29][1]-1, c2d_xy_8x8x8[29][2]+1), + offset8x8x8(c2d_xy_8x8x8[30][0], c2d_xy_8x8x8[30][1]-1, c2d_xy_8x8x8[30][2]+1), + offset8x8x8(c2d_xy_8x8x8[31][0], c2d_xy_8x8x8[31][1]-1, c2d_xy_8x8x8[31][2]+1), + offset8x8x8(c2d_xy_8x8x8[32][0], c2d_xy_8x8x8[32][1]-1, c2d_xy_8x8x8[32][2]+1), + offset8x8x8(c2d_xy_8x8x8[33][0], c2d_xy_8x8x8[33][1]-1, c2d_xy_8x8x8[33][2]+1), + offset8x8x8(c2d_xy_8x8x8[34][0], c2d_xy_8x8x8[34][1]-1, c2d_xy_8x8x8[34][2]+1), + offset8x8x8(c2d_xy_8x8x8[35][0], c2d_xy_8x8x8[35][1]-1, c2d_xy_8x8x8[35][2]+1), + offset8x8x8(c2d_xy_8x8x8[36][0], c2d_xy_8x8x8[36][1]-1, c2d_xy_8x8x8[36][2]+1), + offset8x8x8(c2d_xy_8x8x8[37][0], c2d_xy_8x8x8[37][1]-1, c2d_xy_8x8x8[37][2]+1), + offset8x8x8(c2d_xy_8x8x8[38][0], c2d_xy_8x8x8[38][1]-1, c2d_xy_8x8x8[38][2]+1), + offset8x8x8(c2d_xy_8x8x8[39][0], c2d_xy_8x8x8[39][1]-1, c2d_xy_8x8x8[39][2]+1), + offset8x8x8(c2d_xy_8x8x8[40][0], c2d_xy_8x8x8[40][1]-1, c2d_xy_8x8x8[40][2]+1), + offset8x8x8(c2d_xy_8x8x8[41][0], c2d_xy_8x8x8[41][1]-1, c2d_xy_8x8x8[41][2]+1), + offset8x8x8(c2d_xy_8x8x8[42][0], c2d_xy_8x8x8[42][1]-1, c2d_xy_8x8x8[42][2]+1), + offset8x8x8(c2d_xy_8x8x8[43][0], c2d_xy_8x8x8[43][1]-1, c2d_xy_8x8x8[43][2]+1), + offset8x8x8(c2d_xy_8x8x8[44][0], c2d_xy_8x8x8[44][1]-1, c2d_xy_8x8x8[44][2]+1), + + offset8x8x8(c2d_xz_8x8x8[0][0]-1, c2d_xz_8x8x8[0][1], c2d_xz_8x8x8[0][2]+1), // XZ + offset8x8x8(c2d_xz_8x8x8[1][0]-1, c2d_xz_8x8x8[1][1], c2d_xz_8x8x8[1][2]+1), + offset8x8x8(c2d_xz_8x8x8[2][0]-1, c2d_xz_8x8x8[2][1], c2d_xz_8x8x8[2][2]+1), + offset8x8x8(c2d_xz_8x8x8[3][0]-1, c2d_xz_8x8x8[3][1], c2d_xz_8x8x8[3][2]+1), + offset8x8x8(c2d_xz_8x8x8[4][0]-1, c2d_xz_8x8x8[4][1], c2d_xz_8x8x8[4][2]+1), + offset8x8x8(c2d_xz_8x8x8[5][0]-1, c2d_xz_8x8x8[5][1], c2d_xz_8x8x8[5][2]+1), + offset8x8x8(c2d_xz_8x8x8[6][0]-1, c2d_xz_8x8x8[6][1], c2d_xz_8x8x8[6][2]+1), + offset8x8x8(c2d_xz_8x8x8[7][0]-1, c2d_xz_8x8x8[7][1], c2d_xz_8x8x8[7][2]+1), + offset8x8x8(c2d_xz_8x8x8[8][0]-1, c2d_xz_8x8x8[8][1], c2d_xz_8x8x8[8][2]+1), + offset8x8x8(c2d_xz_8x8x8[9][0]-1, c2d_xz_8x8x8[9][1], c2d_xz_8x8x8[9][2]+1), + offset8x8x8(c2d_xz_8x8x8[10][0]-1, c2d_xz_8x8x8[10][1], c2d_xz_8x8x8[10][2]+1), + offset8x8x8(c2d_xz_8x8x8[11][0]-1, c2d_xz_8x8x8[11][1], c2d_xz_8x8x8[11][2]+1), + offset8x8x8(c2d_xz_8x8x8[12][0]-1, c2d_xz_8x8x8[12][1], c2d_xz_8x8x8[12][2]+1), + offset8x8x8(c2d_xz_8x8x8[13][0]-1, c2d_xz_8x8x8[13][1], c2d_xz_8x8x8[13][2]+1), + offset8x8x8(c2d_xz_8x8x8[14][0]-1, c2d_xz_8x8x8[14][1], c2d_xz_8x8x8[14][2]+1), + offset8x8x8(c2d_xz_8x8x8[15][0]-1, c2d_xz_8x8x8[15][1], c2d_xz_8x8x8[15][2]+1), + offset8x8x8(c2d_xz_8x8x8[16][0]-1, c2d_xz_8x8x8[16][1], c2d_xz_8x8x8[16][2]+1), + offset8x8x8(c2d_xz_8x8x8[17][0]-1, c2d_xz_8x8x8[17][1], c2d_xz_8x8x8[17][2]+1), + offset8x8x8(c2d_xz_8x8x8[18][0]-1, c2d_xz_8x8x8[18][1], c2d_xz_8x8x8[18][2]+1), + offset8x8x8(c2d_xz_8x8x8[19][0]-1, c2d_xz_8x8x8[19][1], c2d_xz_8x8x8[19][2]+1), + offset8x8x8(c2d_xz_8x8x8[20][0]-1, c2d_xz_8x8x8[20][1], c2d_xz_8x8x8[20][2]+1), + offset8x8x8(c2d_xz_8x8x8[21][0]-1, c2d_xz_8x8x8[21][1], c2d_xz_8x8x8[21][2]+1), + offset8x8x8(c2d_xz_8x8x8[22][0]-1, c2d_xz_8x8x8[22][1], c2d_xz_8x8x8[22][2]+1), + offset8x8x8(c2d_xz_8x8x8[23][0]-1, c2d_xz_8x8x8[23][1], c2d_xz_8x8x8[23][2]+1), + offset8x8x8(c2d_xz_8x8x8[24][0]-1, c2d_xz_8x8x8[24][1], c2d_xz_8x8x8[24][2]+1), + offset8x8x8(c2d_xz_8x8x8[25][0]-1, c2d_xz_8x8x8[25][1], c2d_xz_8x8x8[25][2]+1), + offset8x8x8(c2d_xz_8x8x8[26][0]-1, c2d_xz_8x8x8[26][1], c2d_xz_8x8x8[26][2]+1), + offset8x8x8(c2d_xz_8x8x8[27][0]-1, c2d_xz_8x8x8[27][1], c2d_xz_8x8x8[27][2]+1), + offset8x8x8(c2d_xz_8x8x8[28][0]-1, c2d_xz_8x8x8[28][1], c2d_xz_8x8x8[28][2]+1), + offset8x8x8(c2d_xz_8x8x8[29][0]-1, c2d_xz_8x8x8[29][1], c2d_xz_8x8x8[29][2]+1), + offset8x8x8(c2d_xz_8x8x8[30][0]-1, c2d_xz_8x8x8[30][1], c2d_xz_8x8x8[30][2]+1), + offset8x8x8(c2d_xz_8x8x8[31][0]-1, c2d_xz_8x8x8[31][1], c2d_xz_8x8x8[31][2]+1), + offset8x8x8(c2d_xz_8x8x8[32][0]-1, c2d_xz_8x8x8[32][1], c2d_xz_8x8x8[32][2]+1), + offset8x8x8(c2d_xz_8x8x8[33][0]-1, c2d_xz_8x8x8[33][1], c2d_xz_8x8x8[33][2]+1), + offset8x8x8(c2d_xz_8x8x8[34][0]-1, c2d_xz_8x8x8[34][1], c2d_xz_8x8x8[34][2]+1), + offset8x8x8(c2d_xz_8x8x8[35][0]-1, c2d_xz_8x8x8[35][1], c2d_xz_8x8x8[35][2]+1), + offset8x8x8(c2d_xz_8x8x8[36][0]-1, c2d_xz_8x8x8[36][1], c2d_xz_8x8x8[36][2]+1), + offset8x8x8(c2d_xz_8x8x8[37][0]-1, c2d_xz_8x8x8[37][1], c2d_xz_8x8x8[37][2]+1), + offset8x8x8(c2d_xz_8x8x8[38][0]-1, c2d_xz_8x8x8[38][1], c2d_xz_8x8x8[38][2]+1), + offset8x8x8(c2d_xz_8x8x8[39][0]-1, c2d_xz_8x8x8[39][1], c2d_xz_8x8x8[39][2]+1), + offset8x8x8(c2d_xz_8x8x8[40][0]-1, c2d_xz_8x8x8[40][1], c2d_xz_8x8x8[40][2]+1), + offset8x8x8(c2d_xz_8x8x8[41][0]-1, c2d_xz_8x8x8[41][1], c2d_xz_8x8x8[41][2]+1), + offset8x8x8(c2d_xz_8x8x8[42][0]-1, c2d_xz_8x8x8[42][1], c2d_xz_8x8x8[42][2]+1), + offset8x8x8(c2d_xz_8x8x8[43][0]-1, c2d_xz_8x8x8[43][1], c2d_xz_8x8x8[43][2]+1), + offset8x8x8(c2d_xz_8x8x8[44][0]-1, c2d_xz_8x8x8[44][1], c2d_xz_8x8x8[44][2]+1), + + offset8x8x8(c2d_yz_8x8x8[0][0]-1, c2d_yz_8x8x8[0][1]+1, c2d_yz_8x8x8[0][2]), // YZ + offset8x8x8(c2d_yz_8x8x8[1][0]-1, c2d_yz_8x8x8[1][1]+1, c2d_yz_8x8x8[1][2]), + offset8x8x8(c2d_yz_8x8x8[2][0]-1, c2d_yz_8x8x8[2][1]+1, c2d_yz_8x8x8[2][2]), + offset8x8x8(c2d_yz_8x8x8[3][0]-1, c2d_yz_8x8x8[3][1]+1, c2d_yz_8x8x8[3][2]), + offset8x8x8(c2d_yz_8x8x8[4][0]-1, c2d_yz_8x8x8[4][1]+1, c2d_yz_8x8x8[4][2]), + offset8x8x8(c2d_yz_8x8x8[5][0]-1, c2d_yz_8x8x8[5][1]+1, c2d_yz_8x8x8[5][2]), + offset8x8x8(c2d_yz_8x8x8[6][0]-1, c2d_yz_8x8x8[6][1]+1, c2d_yz_8x8x8[6][2]), + offset8x8x8(c2d_yz_8x8x8[7][0]-1, c2d_yz_8x8x8[7][1]+1, c2d_yz_8x8x8[7][2]), + offset8x8x8(c2d_yz_8x8x8[8][0]-1, c2d_yz_8x8x8[8][1]+1, c2d_yz_8x8x8[8][2]), + offset8x8x8(c2d_yz_8x8x8[9][0]-1, c2d_yz_8x8x8[9][1]+1, c2d_yz_8x8x8[9][2]), + offset8x8x8(c2d_yz_8x8x8[10][0]-1, c2d_yz_8x8x8[10][1]+1, c2d_yz_8x8x8[10][2]), + offset8x8x8(c2d_yz_8x8x8[11][0]-1, c2d_yz_8x8x8[11][1]+1, c2d_yz_8x8x8[11][2]), + offset8x8x8(c2d_yz_8x8x8[12][0]-1, c2d_yz_8x8x8[12][1]+1, c2d_yz_8x8x8[12][2]), + offset8x8x8(c2d_yz_8x8x8[13][0]-1, c2d_yz_8x8x8[13][1]+1, c2d_yz_8x8x8[13][2]), + offset8x8x8(c2d_yz_8x8x8[14][0]-1, c2d_yz_8x8x8[14][1]+1, c2d_yz_8x8x8[14][2]), + offset8x8x8(c2d_yz_8x8x8[15][0]-1, c2d_yz_8x8x8[15][1]+1, c2d_yz_8x8x8[15][2]), + offset8x8x8(c2d_yz_8x8x8[16][0]-1, c2d_yz_8x8x8[16][1]+1, c2d_yz_8x8x8[16][2]), + offset8x8x8(c2d_yz_8x8x8[17][0]-1, c2d_yz_8x8x8[17][1]+1, c2d_yz_8x8x8[17][2]), + offset8x8x8(c2d_yz_8x8x8[18][0]-1, c2d_yz_8x8x8[18][1]+1, c2d_yz_8x8x8[18][2]), + offset8x8x8(c2d_yz_8x8x8[19][0]-1, c2d_yz_8x8x8[19][1]+1, c2d_yz_8x8x8[19][2]), + offset8x8x8(c2d_yz_8x8x8[20][0]-1, c2d_yz_8x8x8[20][1]+1, c2d_yz_8x8x8[20][2]), + offset8x8x8(c2d_yz_8x8x8[21][0]-1, c2d_yz_8x8x8[21][1]+1, c2d_yz_8x8x8[21][2]), + offset8x8x8(c2d_yz_8x8x8[22][0]-1, c2d_yz_8x8x8[22][1]+1, c2d_yz_8x8x8[22][2]), + offset8x8x8(c2d_yz_8x8x8[23][0]-1, c2d_yz_8x8x8[23][1]+1, c2d_yz_8x8x8[23][2]), + offset8x8x8(c2d_yz_8x8x8[24][0]-1, c2d_yz_8x8x8[24][1]+1, c2d_yz_8x8x8[24][2]), + offset8x8x8(c2d_yz_8x8x8[25][0]-1, c2d_yz_8x8x8[25][1]+1, c2d_yz_8x8x8[25][2]), + offset8x8x8(c2d_yz_8x8x8[26][0]-1, c2d_yz_8x8x8[26][1]+1, c2d_yz_8x8x8[26][2]), + offset8x8x8(c2d_yz_8x8x8[27][0]-1, c2d_yz_8x8x8[27][1]+1, c2d_yz_8x8x8[27][2]), + offset8x8x8(c2d_yz_8x8x8[28][0]-1, c2d_yz_8x8x8[28][1]+1, c2d_yz_8x8x8[28][2]), + offset8x8x8(c2d_yz_8x8x8[29][0]-1, c2d_yz_8x8x8[29][1]+1, c2d_yz_8x8x8[29][2]), + offset8x8x8(c2d_yz_8x8x8[30][0]-1, c2d_yz_8x8x8[30][1]+1, c2d_yz_8x8x8[30][2]), + offset8x8x8(c2d_yz_8x8x8[31][0]-1, c2d_yz_8x8x8[31][1]+1, c2d_yz_8x8x8[31][2]), + offset8x8x8(c2d_yz_8x8x8[32][0]-1, c2d_yz_8x8x8[32][1]+1, c2d_yz_8x8x8[32][2]), + offset8x8x8(c2d_yz_8x8x8[33][0]-1, c2d_yz_8x8x8[33][1]+1, c2d_yz_8x8x8[33][2]), + offset8x8x8(c2d_yz_8x8x8[34][0]-1, c2d_yz_8x8x8[34][1]+1, c2d_yz_8x8x8[34][2]), + offset8x8x8(c2d_yz_8x8x8[35][0]-1, c2d_yz_8x8x8[35][1]+1, c2d_yz_8x8x8[35][2]), + offset8x8x8(c2d_yz_8x8x8[36][0]-1, c2d_yz_8x8x8[36][1]+1, c2d_yz_8x8x8[36][2]), + offset8x8x8(c2d_yz_8x8x8[37][0]-1, c2d_yz_8x8x8[37][1]+1, c2d_yz_8x8x8[37][2]), + offset8x8x8(c2d_yz_8x8x8[38][0]-1, c2d_yz_8x8x8[38][1]+1, c2d_yz_8x8x8[38][2]), + offset8x8x8(c2d_yz_8x8x8[39][0]-1, c2d_yz_8x8x8[39][1]+1, c2d_yz_8x8x8[39][2]), + offset8x8x8(c2d_yz_8x8x8[40][0]-1, c2d_yz_8x8x8[40][1]+1, c2d_yz_8x8x8[40][2]), + offset8x8x8(c2d_yz_8x8x8[41][0]-1, c2d_yz_8x8x8[41][1]+1, c2d_yz_8x8x8[41][2]), + offset8x8x8(c2d_yz_8x8x8[42][0]-1, c2d_yz_8x8x8[42][1]+1, c2d_yz_8x8x8[42][2]), + offset8x8x8(c2d_yz_8x8x8[43][0]-1, c2d_yz_8x8x8[43][1]+1, c2d_yz_8x8x8[43][2]), + offset8x8x8(c2d_yz_8x8x8[44][0]-1, c2d_yz_8x8x8[44][1]+1, c2d_yz_8x8x8[44][2])}; + return offset[i]; +} + +MGARDX_EXEC int Coeff2D_RR_Offset_8x8x8(SIZE i) { + static constexpr int offset[135] = {offset8x8x8(c2d_xy_8x8x8[0][0], c2d_xy_8x8x8[0][1]+1, c2d_xy_8x8x8[0][2]+1), // XY + offset8x8x8(c2d_xy_8x8x8[1][0], c2d_xy_8x8x8[1][1]+1, c2d_xy_8x8x8[1][2]+1), + offset8x8x8(c2d_xy_8x8x8[2][0], c2d_xy_8x8x8[2][1]+1, c2d_xy_8x8x8[2][2]+1), + offset8x8x8(c2d_xy_8x8x8[3][0], c2d_xy_8x8x8[3][1]+1, c2d_xy_8x8x8[3][2]+1), + offset8x8x8(c2d_xy_8x8x8[4][0], c2d_xy_8x8x8[4][1]+1, c2d_xy_8x8x8[4][2]+1), + offset8x8x8(c2d_xy_8x8x8[5][0], c2d_xy_8x8x8[5][1]+1, c2d_xy_8x8x8[5][2]+1), + offset8x8x8(c2d_xy_8x8x8[6][0], c2d_xy_8x8x8[6][1]+1, c2d_xy_8x8x8[6][2]+1), + offset8x8x8(c2d_xy_8x8x8[7][0], c2d_xy_8x8x8[7][1]+1, c2d_xy_8x8x8[7][2]+1), + offset8x8x8(c2d_xy_8x8x8[8][0], c2d_xy_8x8x8[8][1]+1, c2d_xy_8x8x8[8][2]+1), + offset8x8x8(c2d_xy_8x8x8[9][0], c2d_xy_8x8x8[9][1]+1, c2d_xy_8x8x8[9][2]+1), + offset8x8x8(c2d_xy_8x8x8[10][0], c2d_xy_8x8x8[10][1]+1, c2d_xy_8x8x8[10][2]+1), + offset8x8x8(c2d_xy_8x8x8[11][0], c2d_xy_8x8x8[11][1]+1, c2d_xy_8x8x8[11][2]+1), + offset8x8x8(c2d_xy_8x8x8[12][0], c2d_xy_8x8x8[12][1]+1, c2d_xy_8x8x8[12][2]+1), + offset8x8x8(c2d_xy_8x8x8[13][0], c2d_xy_8x8x8[13][1]+1, c2d_xy_8x8x8[13][2]+1), + offset8x8x8(c2d_xy_8x8x8[14][0], c2d_xy_8x8x8[14][1]+1, c2d_xy_8x8x8[14][2]+1), + offset8x8x8(c2d_xy_8x8x8[15][0], c2d_xy_8x8x8[15][1]+1, c2d_xy_8x8x8[15][2]+1), + offset8x8x8(c2d_xy_8x8x8[16][0], c2d_xy_8x8x8[16][1]+1, c2d_xy_8x8x8[16][2]+1), + offset8x8x8(c2d_xy_8x8x8[17][0], c2d_xy_8x8x8[17][1]+1, c2d_xy_8x8x8[17][2]+1), + offset8x8x8(c2d_xy_8x8x8[18][0], c2d_xy_8x8x8[18][1]+1, c2d_xy_8x8x8[18][2]+1), + offset8x8x8(c2d_xy_8x8x8[19][0], c2d_xy_8x8x8[19][1]+1, c2d_xy_8x8x8[19][2]+1), + offset8x8x8(c2d_xy_8x8x8[20][0], c2d_xy_8x8x8[20][1]+1, c2d_xy_8x8x8[20][2]+1), + offset8x8x8(c2d_xy_8x8x8[21][0], c2d_xy_8x8x8[21][1]+1, c2d_xy_8x8x8[21][2]+1), + offset8x8x8(c2d_xy_8x8x8[22][0], c2d_xy_8x8x8[22][1]+1, c2d_xy_8x8x8[22][2]+1), + offset8x8x8(c2d_xy_8x8x8[23][0], c2d_xy_8x8x8[23][1]+1, c2d_xy_8x8x8[23][2]+1), + offset8x8x8(c2d_xy_8x8x8[24][0], c2d_xy_8x8x8[24][1]+1, c2d_xy_8x8x8[24][2]+1), + offset8x8x8(c2d_xy_8x8x8[25][0], c2d_xy_8x8x8[25][1]+1, c2d_xy_8x8x8[25][2]+1), + offset8x8x8(c2d_xy_8x8x8[26][0], c2d_xy_8x8x8[26][1]+1, c2d_xy_8x8x8[26][2]+1), + offset8x8x8(c2d_xy_8x8x8[27][0], c2d_xy_8x8x8[27][1]+1, c2d_xy_8x8x8[27][2]+1), + offset8x8x8(c2d_xy_8x8x8[28][0], c2d_xy_8x8x8[28][1]+1, c2d_xy_8x8x8[28][2]+1), + offset8x8x8(c2d_xy_8x8x8[29][0], c2d_xy_8x8x8[29][1]+1, c2d_xy_8x8x8[29][2]+1), + offset8x8x8(c2d_xy_8x8x8[30][0], c2d_xy_8x8x8[30][1]+1, c2d_xy_8x8x8[30][2]+1), + offset8x8x8(c2d_xy_8x8x8[31][0], c2d_xy_8x8x8[31][1]+1, c2d_xy_8x8x8[31][2]+1), + offset8x8x8(c2d_xy_8x8x8[32][0], c2d_xy_8x8x8[32][1]+1, c2d_xy_8x8x8[32][2]+1), + offset8x8x8(c2d_xy_8x8x8[33][0], c2d_xy_8x8x8[33][1]+1, c2d_xy_8x8x8[33][2]+1), + offset8x8x8(c2d_xy_8x8x8[34][0], c2d_xy_8x8x8[34][1]+1, c2d_xy_8x8x8[34][2]+1), + offset8x8x8(c2d_xy_8x8x8[35][0], c2d_xy_8x8x8[35][1]+1, c2d_xy_8x8x8[35][2]+1), + offset8x8x8(c2d_xy_8x8x8[36][0], c2d_xy_8x8x8[36][1]+1, c2d_xy_8x8x8[36][2]+1), + offset8x8x8(c2d_xy_8x8x8[37][0], c2d_xy_8x8x8[37][1]+1, c2d_xy_8x8x8[37][2]+1), + offset8x8x8(c2d_xy_8x8x8[38][0], c2d_xy_8x8x8[38][1]+1, c2d_xy_8x8x8[38][2]+1), + offset8x8x8(c2d_xy_8x8x8[39][0], c2d_xy_8x8x8[39][1]+1, c2d_xy_8x8x8[39][2]+1), + offset8x8x8(c2d_xy_8x8x8[40][0], c2d_xy_8x8x8[40][1]+1, c2d_xy_8x8x8[40][2]+1), + offset8x8x8(c2d_xy_8x8x8[41][0], c2d_xy_8x8x8[41][1]+1, c2d_xy_8x8x8[41][2]+1), + offset8x8x8(c2d_xy_8x8x8[42][0], c2d_xy_8x8x8[42][1]+1, c2d_xy_8x8x8[42][2]+1), + offset8x8x8(c2d_xy_8x8x8[43][0], c2d_xy_8x8x8[43][1]+1, c2d_xy_8x8x8[43][2]+1), + offset8x8x8(c2d_xy_8x8x8[44][0], c2d_xy_8x8x8[44][1]+1, c2d_xy_8x8x8[44][2]+1), + + offset8x8x8(c2d_xz_8x8x8[0][0]+1, c2d_xz_8x8x8[0][1], c2d_xz_8x8x8[0][2]+1), // XZ + offset8x8x8(c2d_xz_8x8x8[1][0]+1, c2d_xz_8x8x8[1][1], c2d_xz_8x8x8[1][2]+1), + offset8x8x8(c2d_xz_8x8x8[2][0]+1, c2d_xz_8x8x8[2][1], c2d_xz_8x8x8[2][2]+1), + offset8x8x8(c2d_xz_8x8x8[3][0]+1, c2d_xz_8x8x8[3][1], c2d_xz_8x8x8[3][2]+1), + offset8x8x8(c2d_xz_8x8x8[4][0]+1, c2d_xz_8x8x8[4][1], c2d_xz_8x8x8[4][2]+1), + offset8x8x8(c2d_xz_8x8x8[5][0]+1, c2d_xz_8x8x8[5][1], c2d_xz_8x8x8[5][2]+1), + offset8x8x8(c2d_xz_8x8x8[6][0]+1, c2d_xz_8x8x8[6][1], c2d_xz_8x8x8[6][2]+1), + offset8x8x8(c2d_xz_8x8x8[7][0]+1, c2d_xz_8x8x8[7][1], c2d_xz_8x8x8[7][2]+1), + offset8x8x8(c2d_xz_8x8x8[8][0]+1, c2d_xz_8x8x8[8][1], c2d_xz_8x8x8[8][2]+1), + offset8x8x8(c2d_xz_8x8x8[9][0]+1, c2d_xz_8x8x8[9][1], c2d_xz_8x8x8[9][2]+1), + offset8x8x8(c2d_xz_8x8x8[10][0]+1, c2d_xz_8x8x8[10][1], c2d_xz_8x8x8[10][2]+1), + offset8x8x8(c2d_xz_8x8x8[11][0]+1, c2d_xz_8x8x8[11][1], c2d_xz_8x8x8[11][2]+1), + offset8x8x8(c2d_xz_8x8x8[12][0]+1, c2d_xz_8x8x8[12][1], c2d_xz_8x8x8[12][2]+1), + offset8x8x8(c2d_xz_8x8x8[13][0]+1, c2d_xz_8x8x8[13][1], c2d_xz_8x8x8[13][2]+1), + offset8x8x8(c2d_xz_8x8x8[14][0]+1, c2d_xz_8x8x8[14][1], c2d_xz_8x8x8[14][2]+1), + offset8x8x8(c2d_xz_8x8x8[15][0]+1, c2d_xz_8x8x8[15][1], c2d_xz_8x8x8[15][2]+1), + offset8x8x8(c2d_xz_8x8x8[16][0]+1, c2d_xz_8x8x8[16][1], c2d_xz_8x8x8[16][2]+1), + offset8x8x8(c2d_xz_8x8x8[17][0]+1, c2d_xz_8x8x8[17][1], c2d_xz_8x8x8[17][2]+1), + offset8x8x8(c2d_xz_8x8x8[18][0]+1, c2d_xz_8x8x8[18][1], c2d_xz_8x8x8[18][2]+1), + offset8x8x8(c2d_xz_8x8x8[19][0]+1, c2d_xz_8x8x8[19][1], c2d_xz_8x8x8[19][2]+1), + offset8x8x8(c2d_xz_8x8x8[20][0]+1, c2d_xz_8x8x8[20][1], c2d_xz_8x8x8[20][2]+1), + offset8x8x8(c2d_xz_8x8x8[21][0]+1, c2d_xz_8x8x8[21][1], c2d_xz_8x8x8[21][2]+1), + offset8x8x8(c2d_xz_8x8x8[22][0]+1, c2d_xz_8x8x8[22][1], c2d_xz_8x8x8[22][2]+1), + offset8x8x8(c2d_xz_8x8x8[23][0]+1, c2d_xz_8x8x8[23][1], c2d_xz_8x8x8[23][2]+1), + offset8x8x8(c2d_xz_8x8x8[24][0]+1, c2d_xz_8x8x8[24][1], c2d_xz_8x8x8[24][2]+1), + offset8x8x8(c2d_xz_8x8x8[25][0]+1, c2d_xz_8x8x8[25][1], c2d_xz_8x8x8[25][2]+1), + offset8x8x8(c2d_xz_8x8x8[26][0]+1, c2d_xz_8x8x8[26][1], c2d_xz_8x8x8[26][2]+1), + offset8x8x8(c2d_xz_8x8x8[27][0]+1, c2d_xz_8x8x8[27][1], c2d_xz_8x8x8[27][2]+1), + offset8x8x8(c2d_xz_8x8x8[28][0]+1, c2d_xz_8x8x8[28][1], c2d_xz_8x8x8[28][2]+1), + offset8x8x8(c2d_xz_8x8x8[29][0]+1, c2d_xz_8x8x8[29][1], c2d_xz_8x8x8[29][2]+1), + offset8x8x8(c2d_xz_8x8x8[30][0]+1, c2d_xz_8x8x8[30][1], c2d_xz_8x8x8[30][2]+1), + offset8x8x8(c2d_xz_8x8x8[31][0]+1, c2d_xz_8x8x8[31][1], c2d_xz_8x8x8[31][2]+1), + offset8x8x8(c2d_xz_8x8x8[32][0]+1, c2d_xz_8x8x8[32][1], c2d_xz_8x8x8[32][2]+1), + offset8x8x8(c2d_xz_8x8x8[33][0]+1, c2d_xz_8x8x8[33][1], c2d_xz_8x8x8[33][2]+1), + offset8x8x8(c2d_xz_8x8x8[34][0]+1, c2d_xz_8x8x8[34][1], c2d_xz_8x8x8[34][2]+1), + offset8x8x8(c2d_xz_8x8x8[35][0]+1, c2d_xz_8x8x8[35][1], c2d_xz_8x8x8[35][2]+1), + offset8x8x8(c2d_xz_8x8x8[36][0]+1, c2d_xz_8x8x8[36][1], c2d_xz_8x8x8[36][2]+1), + offset8x8x8(c2d_xz_8x8x8[37][0]+1, c2d_xz_8x8x8[37][1], c2d_xz_8x8x8[37][2]+1), + offset8x8x8(c2d_xz_8x8x8[38][0]+1, c2d_xz_8x8x8[38][1], c2d_xz_8x8x8[38][2]+1), + offset8x8x8(c2d_xz_8x8x8[39][0]+1, c2d_xz_8x8x8[39][1], c2d_xz_8x8x8[39][2]+1), + offset8x8x8(c2d_xz_8x8x8[40][0]+1, c2d_xz_8x8x8[40][1], c2d_xz_8x8x8[40][2]+1), + offset8x8x8(c2d_xz_8x8x8[41][0]+1, c2d_xz_8x8x8[41][1], c2d_xz_8x8x8[41][2]+1), + offset8x8x8(c2d_xz_8x8x8[42][0]+1, c2d_xz_8x8x8[42][1], c2d_xz_8x8x8[42][2]+1), + offset8x8x8(c2d_xz_8x8x8[43][0]+1, c2d_xz_8x8x8[43][1], c2d_xz_8x8x8[43][2]+1), + offset8x8x8(c2d_xz_8x8x8[44][0]+1, c2d_xz_8x8x8[44][1], c2d_xz_8x8x8[44][2]+1), + + offset8x8x8(c2d_yz_8x8x8[0][0]+1, c2d_yz_8x8x8[0][1]+1, c2d_yz_8x8x8[0][2]), // YZ + offset8x8x8(c2d_yz_8x8x8[1][0]+1, c2d_yz_8x8x8[1][1]+1, c2d_yz_8x8x8[1][2]), + offset8x8x8(c2d_yz_8x8x8[2][0]+1, c2d_yz_8x8x8[2][1]+1, c2d_yz_8x8x8[2][2]), + offset8x8x8(c2d_yz_8x8x8[3][0]+1, c2d_yz_8x8x8[3][1]+1, c2d_yz_8x8x8[3][2]), + offset8x8x8(c2d_yz_8x8x8[4][0]+1, c2d_yz_8x8x8[4][1]+1, c2d_yz_8x8x8[4][2]), + offset8x8x8(c2d_yz_8x8x8[5][0]+1, c2d_yz_8x8x8[5][1]+1, c2d_yz_8x8x8[5][2]), + offset8x8x8(c2d_yz_8x8x8[6][0]+1, c2d_yz_8x8x8[6][1]+1, c2d_yz_8x8x8[6][2]), + offset8x8x8(c2d_yz_8x8x8[7][0]+1, c2d_yz_8x8x8[7][1]+1, c2d_yz_8x8x8[7][2]), + offset8x8x8(c2d_yz_8x8x8[8][0]+1, c2d_yz_8x8x8[8][1]+1, c2d_yz_8x8x8[8][2]), + offset8x8x8(c2d_yz_8x8x8[9][0]+1, c2d_yz_8x8x8[9][1]+1, c2d_yz_8x8x8[9][2]), + offset8x8x8(c2d_yz_8x8x8[10][0]+1, c2d_yz_8x8x8[10][1]+1, c2d_yz_8x8x8[10][2]), + offset8x8x8(c2d_yz_8x8x8[11][0]+1, c2d_yz_8x8x8[11][1]+1, c2d_yz_8x8x8[11][2]), + offset8x8x8(c2d_yz_8x8x8[12][0]+1, c2d_yz_8x8x8[12][1]+1, c2d_yz_8x8x8[12][2]), + offset8x8x8(c2d_yz_8x8x8[13][0]+1, c2d_yz_8x8x8[13][1]+1, c2d_yz_8x8x8[13][2]), + offset8x8x8(c2d_yz_8x8x8[14][0]+1, c2d_yz_8x8x8[14][1]+1, c2d_yz_8x8x8[14][2]), + offset8x8x8(c2d_yz_8x8x8[15][0]+1, c2d_yz_8x8x8[15][1]+1, c2d_yz_8x8x8[15][2]), + offset8x8x8(c2d_yz_8x8x8[16][0]+1, c2d_yz_8x8x8[16][1]+1, c2d_yz_8x8x8[16][2]), + offset8x8x8(c2d_yz_8x8x8[17][0]+1, c2d_yz_8x8x8[17][1]+1, c2d_yz_8x8x8[17][2]), + offset8x8x8(c2d_yz_8x8x8[18][0]+1, c2d_yz_8x8x8[18][1]+1, c2d_yz_8x8x8[18][2]), + offset8x8x8(c2d_yz_8x8x8[19][0]+1, c2d_yz_8x8x8[19][1]+1, c2d_yz_8x8x8[19][2]), + offset8x8x8(c2d_yz_8x8x8[20][0]+1, c2d_yz_8x8x8[20][1]+1, c2d_yz_8x8x8[20][2]), + offset8x8x8(c2d_yz_8x8x8[21][0]+1, c2d_yz_8x8x8[21][1]+1, c2d_yz_8x8x8[21][2]), + offset8x8x8(c2d_yz_8x8x8[22][0]+1, c2d_yz_8x8x8[22][1]+1, c2d_yz_8x8x8[22][2]), + offset8x8x8(c2d_yz_8x8x8[23][0]+1, c2d_yz_8x8x8[23][1]+1, c2d_yz_8x8x8[23][2]), + offset8x8x8(c2d_yz_8x8x8[24][0]+1, c2d_yz_8x8x8[24][1]+1, c2d_yz_8x8x8[24][2]), + offset8x8x8(c2d_yz_8x8x8[25][0]+1, c2d_yz_8x8x8[25][1]+1, c2d_yz_8x8x8[25][2]), + offset8x8x8(c2d_yz_8x8x8[26][0]+1, c2d_yz_8x8x8[26][1]+1, c2d_yz_8x8x8[26][2]), + offset8x8x8(c2d_yz_8x8x8[27][0]+1, c2d_yz_8x8x8[27][1]+1, c2d_yz_8x8x8[27][2]), + offset8x8x8(c2d_yz_8x8x8[28][0]+1, c2d_yz_8x8x8[28][1]+1, c2d_yz_8x8x8[28][2]), + offset8x8x8(c2d_yz_8x8x8[29][0]+1, c2d_yz_8x8x8[29][1]+1, c2d_yz_8x8x8[29][2]), + offset8x8x8(c2d_yz_8x8x8[30][0]+1, c2d_yz_8x8x8[30][1]+1, c2d_yz_8x8x8[30][2]), + offset8x8x8(c2d_yz_8x8x8[31][0]+1, c2d_yz_8x8x8[31][1]+1, c2d_yz_8x8x8[31][2]), + offset8x8x8(c2d_yz_8x8x8[32][0]+1, c2d_yz_8x8x8[32][1]+1, c2d_yz_8x8x8[32][2]), + offset8x8x8(c2d_yz_8x8x8[33][0]+1, c2d_yz_8x8x8[33][1]+1, c2d_yz_8x8x8[33][2]), + offset8x8x8(c2d_yz_8x8x8[34][0]+1, c2d_yz_8x8x8[34][1]+1, c2d_yz_8x8x8[34][2]), + offset8x8x8(c2d_yz_8x8x8[35][0]+1, c2d_yz_8x8x8[35][1]+1, c2d_yz_8x8x8[35][2]), + offset8x8x8(c2d_yz_8x8x8[36][0]+1, c2d_yz_8x8x8[36][1]+1, c2d_yz_8x8x8[36][2]), + offset8x8x8(c2d_yz_8x8x8[37][0]+1, c2d_yz_8x8x8[37][1]+1, c2d_yz_8x8x8[37][2]), + offset8x8x8(c2d_yz_8x8x8[38][0]+1, c2d_yz_8x8x8[38][1]+1, c2d_yz_8x8x8[38][2]), + offset8x8x8(c2d_yz_8x8x8[39][0]+1, c2d_yz_8x8x8[39][1]+1, c2d_yz_8x8x8[39][2]), + offset8x8x8(c2d_yz_8x8x8[40][0]+1, c2d_yz_8x8x8[40][1]+1, c2d_yz_8x8x8[40][2]), + offset8x8x8(c2d_yz_8x8x8[41][0]+1, c2d_yz_8x8x8[41][1]+1, c2d_yz_8x8x8[41][2]), + offset8x8x8(c2d_yz_8x8x8[42][0]+1, c2d_yz_8x8x8[42][1]+1, c2d_yz_8x8x8[42][2]), + offset8x8x8(c2d_yz_8x8x8[43][0]+1, c2d_yz_8x8x8[43][1]+1, c2d_yz_8x8x8[43][2]), + offset8x8x8(c2d_yz_8x8x8[44][0]+1, c2d_yz_8x8x8[44][1]+1, c2d_yz_8x8x8[44][2])}; + return offset[i]; +} + +MGARDX_EXEC int Coeff2D_LL_Offset_8x8x8(SIZE i) { + static constexpr int offset[135] = {offset8x8x8(c2d_xy_8x8x8[0][0], c2d_xy_8x8x8[0][1]-1, c2d_xy_8x8x8[0][2]-1), // XY + offset8x8x8(c2d_xy_8x8x8[1][0], c2d_xy_8x8x8[1][1]-1, c2d_xy_8x8x8[1][2]-1), + offset8x8x8(c2d_xy_8x8x8[2][0], c2d_xy_8x8x8[2][1]-1, c2d_xy_8x8x8[2][2]-1), + offset8x8x8(c2d_xy_8x8x8[3][0], c2d_xy_8x8x8[3][1]-1, c2d_xy_8x8x8[3][2]-1), + offset8x8x8(c2d_xy_8x8x8[4][0], c2d_xy_8x8x8[4][1]-1, c2d_xy_8x8x8[4][2]-1), + offset8x8x8(c2d_xy_8x8x8[5][0], c2d_xy_8x8x8[5][1]-1, c2d_xy_8x8x8[5][2]-1), + offset8x8x8(c2d_xy_8x8x8[6][0], c2d_xy_8x8x8[6][1]-1, c2d_xy_8x8x8[6][2]-1), + offset8x8x8(c2d_xy_8x8x8[7][0], c2d_xy_8x8x8[7][1]-1, c2d_xy_8x8x8[7][2]-1), + offset8x8x8(c2d_xy_8x8x8[8][0], c2d_xy_8x8x8[8][1]-1, c2d_xy_8x8x8[8][2]-1), + offset8x8x8(c2d_xy_8x8x8[9][0], c2d_xy_8x8x8[9][1]-1, c2d_xy_8x8x8[9][2]-1), + offset8x8x8(c2d_xy_8x8x8[10][0], c2d_xy_8x8x8[10][1]-1, c2d_xy_8x8x8[10][2]-1), + offset8x8x8(c2d_xy_8x8x8[11][0], c2d_xy_8x8x8[11][1]-1, c2d_xy_8x8x8[11][2]-1), + offset8x8x8(c2d_xy_8x8x8[12][0], c2d_xy_8x8x8[12][1]-1, c2d_xy_8x8x8[12][2]-1), + offset8x8x8(c2d_xy_8x8x8[13][0], c2d_xy_8x8x8[13][1]-1, c2d_xy_8x8x8[13][2]-1), + offset8x8x8(c2d_xy_8x8x8[14][0], c2d_xy_8x8x8[14][1]-1, c2d_xy_8x8x8[14][2]-1), + offset8x8x8(c2d_xy_8x8x8[15][0], c2d_xy_8x8x8[15][1]-1, c2d_xy_8x8x8[15][2]-1), + offset8x8x8(c2d_xy_8x8x8[16][0], c2d_xy_8x8x8[16][1]-1, c2d_xy_8x8x8[16][2]-1), + offset8x8x8(c2d_xy_8x8x8[17][0], c2d_xy_8x8x8[17][1]-1, c2d_xy_8x8x8[17][2]-1), + offset8x8x8(c2d_xy_8x8x8[18][0], c2d_xy_8x8x8[18][1]-1, c2d_xy_8x8x8[18][2]-1), + offset8x8x8(c2d_xy_8x8x8[19][0], c2d_xy_8x8x8[19][1]-1, c2d_xy_8x8x8[19][2]-1), + offset8x8x8(c2d_xy_8x8x8[20][0], c2d_xy_8x8x8[20][1]-1, c2d_xy_8x8x8[20][2]-1), + offset8x8x8(c2d_xy_8x8x8[21][0], c2d_xy_8x8x8[21][1]-1, c2d_xy_8x8x8[21][2]-1), + offset8x8x8(c2d_xy_8x8x8[22][0], c2d_xy_8x8x8[22][1]-1, c2d_xy_8x8x8[22][2]-1), + offset8x8x8(c2d_xy_8x8x8[23][0], c2d_xy_8x8x8[23][1]-1, c2d_xy_8x8x8[23][2]-1), + offset8x8x8(c2d_xy_8x8x8[24][0], c2d_xy_8x8x8[24][1]-1, c2d_xy_8x8x8[24][2]-1), + offset8x8x8(c2d_xy_8x8x8[25][0], c2d_xy_8x8x8[25][1]-1, c2d_xy_8x8x8[25][2]-1), + offset8x8x8(c2d_xy_8x8x8[26][0], c2d_xy_8x8x8[26][1]-1, c2d_xy_8x8x8[26][2]-1), + offset8x8x8(c2d_xy_8x8x8[27][0], c2d_xy_8x8x8[27][1]-1, c2d_xy_8x8x8[27][2]-1), + offset8x8x8(c2d_xy_8x8x8[28][0], c2d_xy_8x8x8[28][1]-1, c2d_xy_8x8x8[28][2]-1), + offset8x8x8(c2d_xy_8x8x8[29][0], c2d_xy_8x8x8[29][1]-1, c2d_xy_8x8x8[29][2]-1), + offset8x8x8(c2d_xy_8x8x8[30][0], c2d_xy_8x8x8[30][1]-1, c2d_xy_8x8x8[30][2]-1), + offset8x8x8(c2d_xy_8x8x8[31][0], c2d_xy_8x8x8[31][1]-1, c2d_xy_8x8x8[31][2]-1), + offset8x8x8(c2d_xy_8x8x8[32][0], c2d_xy_8x8x8[32][1]-1, c2d_xy_8x8x8[32][2]-1), + offset8x8x8(c2d_xy_8x8x8[33][0], c2d_xy_8x8x8[33][1]-1, c2d_xy_8x8x8[33][2]-1), + offset8x8x8(c2d_xy_8x8x8[34][0], c2d_xy_8x8x8[34][1]-1, c2d_xy_8x8x8[34][2]-1), + offset8x8x8(c2d_xy_8x8x8[35][0], c2d_xy_8x8x8[35][1]-1, c2d_xy_8x8x8[35][2]-1), + offset8x8x8(c2d_xy_8x8x8[36][0], c2d_xy_8x8x8[36][1]-1, c2d_xy_8x8x8[36][2]-1), + offset8x8x8(c2d_xy_8x8x8[37][0], c2d_xy_8x8x8[37][1]-1, c2d_xy_8x8x8[37][2]-1), + offset8x8x8(c2d_xy_8x8x8[38][0], c2d_xy_8x8x8[38][1]-1, c2d_xy_8x8x8[38][2]-1), + offset8x8x8(c2d_xy_8x8x8[39][0], c2d_xy_8x8x8[39][1]-1, c2d_xy_8x8x8[39][2]-1), + offset8x8x8(c2d_xy_8x8x8[40][0], c2d_xy_8x8x8[40][1]-1, c2d_xy_8x8x8[40][2]-1), + offset8x8x8(c2d_xy_8x8x8[41][0], c2d_xy_8x8x8[41][1]-1, c2d_xy_8x8x8[41][2]-1), + offset8x8x8(c2d_xy_8x8x8[42][0], c2d_xy_8x8x8[42][1]-1, c2d_xy_8x8x8[42][2]-1), + offset8x8x8(c2d_xy_8x8x8[43][0], c2d_xy_8x8x8[43][1]-1, c2d_xy_8x8x8[43][2]-1), + offset8x8x8(c2d_xy_8x8x8[44][0], c2d_xy_8x8x8[44][1]-1, c2d_xy_8x8x8[44][2]-1), + + offset8x8x8(c2d_xz_8x8x8[0][0]-1, c2d_xz_8x8x8[0][1], c2d_xz_8x8x8[0][2]-1), // XZ + offset8x8x8(c2d_xz_8x8x8[1][0]-1, c2d_xz_8x8x8[1][1], c2d_xz_8x8x8[1][2]-1), + offset8x8x8(c2d_xz_8x8x8[2][0]-1, c2d_xz_8x8x8[2][1], c2d_xz_8x8x8[2][2]-1), + offset8x8x8(c2d_xz_8x8x8[3][0]-1, c2d_xz_8x8x8[3][1], c2d_xz_8x8x8[3][2]-1), + offset8x8x8(c2d_xz_8x8x8[4][0]-1, c2d_xz_8x8x8[4][1], c2d_xz_8x8x8[4][2]-1), + offset8x8x8(c2d_xz_8x8x8[5][0]-1, c2d_xz_8x8x8[5][1], c2d_xz_8x8x8[5][2]-1), + offset8x8x8(c2d_xz_8x8x8[6][0]-1, c2d_xz_8x8x8[6][1], c2d_xz_8x8x8[6][2]-1), + offset8x8x8(c2d_xz_8x8x8[7][0]-1, c2d_xz_8x8x8[7][1], c2d_xz_8x8x8[7][2]-1), + offset8x8x8(c2d_xz_8x8x8[8][0]-1, c2d_xz_8x8x8[8][1], c2d_xz_8x8x8[8][2]-1), + offset8x8x8(c2d_xz_8x8x8[9][0]-1, c2d_xz_8x8x8[9][1], c2d_xz_8x8x8[9][2]-1), + offset8x8x8(c2d_xz_8x8x8[10][0]-1, c2d_xz_8x8x8[10][1], c2d_xz_8x8x8[10][2]-1), + offset8x8x8(c2d_xz_8x8x8[11][0]-1, c2d_xz_8x8x8[11][1], c2d_xz_8x8x8[11][2]-1), + offset8x8x8(c2d_xz_8x8x8[12][0]-1, c2d_xz_8x8x8[12][1], c2d_xz_8x8x8[12][2]-1), + offset8x8x8(c2d_xz_8x8x8[13][0]-1, c2d_xz_8x8x8[13][1], c2d_xz_8x8x8[13][2]-1), + offset8x8x8(c2d_xz_8x8x8[14][0]-1, c2d_xz_8x8x8[14][1], c2d_xz_8x8x8[14][2]-1), + offset8x8x8(c2d_xz_8x8x8[15][0]-1, c2d_xz_8x8x8[15][1], c2d_xz_8x8x8[15][2]-1), + offset8x8x8(c2d_xz_8x8x8[16][0]-1, c2d_xz_8x8x8[16][1], c2d_xz_8x8x8[16][2]-1), + offset8x8x8(c2d_xz_8x8x8[17][0]-1, c2d_xz_8x8x8[17][1], c2d_xz_8x8x8[17][2]-1), + offset8x8x8(c2d_xz_8x8x8[18][0]-1, c2d_xz_8x8x8[18][1], c2d_xz_8x8x8[18][2]-1), + offset8x8x8(c2d_xz_8x8x8[19][0]-1, c2d_xz_8x8x8[19][1], c2d_xz_8x8x8[19][2]-1), + offset8x8x8(c2d_xz_8x8x8[20][0]-1, c2d_xz_8x8x8[20][1], c2d_xz_8x8x8[20][2]-1), + offset8x8x8(c2d_xz_8x8x8[21][0]-1, c2d_xz_8x8x8[21][1], c2d_xz_8x8x8[21][2]-1), + offset8x8x8(c2d_xz_8x8x8[22][0]-1, c2d_xz_8x8x8[22][1], c2d_xz_8x8x8[22][2]-1), + offset8x8x8(c2d_xz_8x8x8[23][0]-1, c2d_xz_8x8x8[23][1], c2d_xz_8x8x8[23][2]-1), + offset8x8x8(c2d_xz_8x8x8[24][0]-1, c2d_xz_8x8x8[24][1], c2d_xz_8x8x8[24][2]-1), + offset8x8x8(c2d_xz_8x8x8[25][0]-1, c2d_xz_8x8x8[25][1], c2d_xz_8x8x8[25][2]-1), + offset8x8x8(c2d_xz_8x8x8[26][0]-1, c2d_xz_8x8x8[26][1], c2d_xz_8x8x8[26][2]-1), + offset8x8x8(c2d_xz_8x8x8[27][0]-1, c2d_xz_8x8x8[27][1], c2d_xz_8x8x8[27][2]-1), + offset8x8x8(c2d_xz_8x8x8[28][0]-1, c2d_xz_8x8x8[28][1], c2d_xz_8x8x8[28][2]-1), + offset8x8x8(c2d_xz_8x8x8[29][0]-1, c2d_xz_8x8x8[29][1], c2d_xz_8x8x8[29][2]-1), + offset8x8x8(c2d_xz_8x8x8[30][0]-1, c2d_xz_8x8x8[30][1], c2d_xz_8x8x8[30][2]-1), + offset8x8x8(c2d_xz_8x8x8[31][0]-1, c2d_xz_8x8x8[31][1], c2d_xz_8x8x8[31][2]-1), + offset8x8x8(c2d_xz_8x8x8[32][0]-1, c2d_xz_8x8x8[32][1], c2d_xz_8x8x8[32][2]-1), + offset8x8x8(c2d_xz_8x8x8[33][0]-1, c2d_xz_8x8x8[33][1], c2d_xz_8x8x8[33][2]-1), + offset8x8x8(c2d_xz_8x8x8[34][0]-1, c2d_xz_8x8x8[34][1], c2d_xz_8x8x8[34][2]-1), + offset8x8x8(c2d_xz_8x8x8[35][0]-1, c2d_xz_8x8x8[35][1], c2d_xz_8x8x8[35][2]-1), + offset8x8x8(c2d_xz_8x8x8[36][0]-1, c2d_xz_8x8x8[36][1], c2d_xz_8x8x8[36][2]-1), + offset8x8x8(c2d_xz_8x8x8[37][0]-1, c2d_xz_8x8x8[37][1], c2d_xz_8x8x8[37][2]-1), + offset8x8x8(c2d_xz_8x8x8[38][0]-1, c2d_xz_8x8x8[38][1], c2d_xz_8x8x8[38][2]-1), + offset8x8x8(c2d_xz_8x8x8[39][0]-1, c2d_xz_8x8x8[39][1], c2d_xz_8x8x8[39][2]-1), + offset8x8x8(c2d_xz_8x8x8[40][0]-1, c2d_xz_8x8x8[40][1], c2d_xz_8x8x8[40][2]-1), + offset8x8x8(c2d_xz_8x8x8[41][0]-1, c2d_xz_8x8x8[41][1], c2d_xz_8x8x8[41][2]-1), + offset8x8x8(c2d_xz_8x8x8[42][0]-1, c2d_xz_8x8x8[42][1], c2d_xz_8x8x8[42][2]-1), + offset8x8x8(c2d_xz_8x8x8[43][0]-1, c2d_xz_8x8x8[43][1], c2d_xz_8x8x8[43][2]-1), + offset8x8x8(c2d_xz_8x8x8[44][0]-1, c2d_xz_8x8x8[44][1], c2d_xz_8x8x8[44][2]-1), + + offset8x8x8(c2d_yz_8x8x8[0][0]-1, c2d_yz_8x8x8[0][1]-1, c2d_yz_8x8x8[0][2]), // YZ + offset8x8x8(c2d_yz_8x8x8[1][0]-1, c2d_yz_8x8x8[1][1]-1, c2d_yz_8x8x8[1][2]), + offset8x8x8(c2d_yz_8x8x8[2][0]-1, c2d_yz_8x8x8[2][1]-1, c2d_yz_8x8x8[2][2]), + offset8x8x8(c2d_yz_8x8x8[3][0]-1, c2d_yz_8x8x8[3][1]-1, c2d_yz_8x8x8[3][2]), + offset8x8x8(c2d_yz_8x8x8[4][0]-1, c2d_yz_8x8x8[4][1]-1, c2d_yz_8x8x8[4][2]), + offset8x8x8(c2d_yz_8x8x8[5][0]-1, c2d_yz_8x8x8[5][1]-1, c2d_yz_8x8x8[5][2]), + offset8x8x8(c2d_yz_8x8x8[6][0]-1, c2d_yz_8x8x8[6][1]-1, c2d_yz_8x8x8[6][2]), + offset8x8x8(c2d_yz_8x8x8[7][0]-1, c2d_yz_8x8x8[7][1]-1, c2d_yz_8x8x8[7][2]), + offset8x8x8(c2d_yz_8x8x8[8][0]-1, c2d_yz_8x8x8[8][1]-1, c2d_yz_8x8x8[8][2]), + offset8x8x8(c2d_yz_8x8x8[9][0]-1, c2d_yz_8x8x8[9][1]-1, c2d_yz_8x8x8[9][2]), + offset8x8x8(c2d_yz_8x8x8[10][0]-1, c2d_yz_8x8x8[10][1]-1, c2d_yz_8x8x8[10][2]), + offset8x8x8(c2d_yz_8x8x8[11][0]-1, c2d_yz_8x8x8[11][1]-1, c2d_yz_8x8x8[11][2]), + offset8x8x8(c2d_yz_8x8x8[12][0]-1, c2d_yz_8x8x8[12][1]-1, c2d_yz_8x8x8[12][2]), + offset8x8x8(c2d_yz_8x8x8[13][0]-1, c2d_yz_8x8x8[13][1]-1, c2d_yz_8x8x8[13][2]), + offset8x8x8(c2d_yz_8x8x8[14][0]-1, c2d_yz_8x8x8[14][1]-1, c2d_yz_8x8x8[14][2]), + offset8x8x8(c2d_yz_8x8x8[15][0]-1, c2d_yz_8x8x8[15][1]-1, c2d_yz_8x8x8[15][2]), + offset8x8x8(c2d_yz_8x8x8[16][0]-1, c2d_yz_8x8x8[16][1]-1, c2d_yz_8x8x8[16][2]), + offset8x8x8(c2d_yz_8x8x8[17][0]-1, c2d_yz_8x8x8[17][1]-1, c2d_yz_8x8x8[17][2]), + offset8x8x8(c2d_yz_8x8x8[18][0]-1, c2d_yz_8x8x8[18][1]-1, c2d_yz_8x8x8[18][2]), + offset8x8x8(c2d_yz_8x8x8[19][0]-1, c2d_yz_8x8x8[19][1]-1, c2d_yz_8x8x8[19][2]), + offset8x8x8(c2d_yz_8x8x8[20][0]-1, c2d_yz_8x8x8[20][1]-1, c2d_yz_8x8x8[20][2]), + offset8x8x8(c2d_yz_8x8x8[21][0]-1, c2d_yz_8x8x8[21][1]-1, c2d_yz_8x8x8[21][2]), + offset8x8x8(c2d_yz_8x8x8[22][0]-1, c2d_yz_8x8x8[22][1]-1, c2d_yz_8x8x8[22][2]), + offset8x8x8(c2d_yz_8x8x8[23][0]-1, c2d_yz_8x8x8[23][1]-1, c2d_yz_8x8x8[23][2]), + offset8x8x8(c2d_yz_8x8x8[24][0]-1, c2d_yz_8x8x8[24][1]-1, c2d_yz_8x8x8[24][2]), + offset8x8x8(c2d_yz_8x8x8[25][0]-1, c2d_yz_8x8x8[25][1]-1, c2d_yz_8x8x8[25][2]), + offset8x8x8(c2d_yz_8x8x8[26][0]-1, c2d_yz_8x8x8[26][1]-1, c2d_yz_8x8x8[26][2]), + offset8x8x8(c2d_yz_8x8x8[27][0]-1, c2d_yz_8x8x8[27][1]-1, c2d_yz_8x8x8[27][2]), + offset8x8x8(c2d_yz_8x8x8[28][0]-1, c2d_yz_8x8x8[28][1]-1, c2d_yz_8x8x8[28][2]), + offset8x8x8(c2d_yz_8x8x8[29][0]-1, c2d_yz_8x8x8[29][1]-1, c2d_yz_8x8x8[29][2]), + offset8x8x8(c2d_yz_8x8x8[30][0]-1, c2d_yz_8x8x8[30][1]-1, c2d_yz_8x8x8[30][2]), + offset8x8x8(c2d_yz_8x8x8[31][0]-1, c2d_yz_8x8x8[31][1]-1, c2d_yz_8x8x8[31][2]), + offset8x8x8(c2d_yz_8x8x8[32][0]-1, c2d_yz_8x8x8[32][1]-1, c2d_yz_8x8x8[32][2]), + offset8x8x8(c2d_yz_8x8x8[33][0]-1, c2d_yz_8x8x8[33][1]-1, c2d_yz_8x8x8[33][2]), + offset8x8x8(c2d_yz_8x8x8[34][0]-1, c2d_yz_8x8x8[34][1]-1, c2d_yz_8x8x8[34][2]), + offset8x8x8(c2d_yz_8x8x8[35][0]-1, c2d_yz_8x8x8[35][1]-1, c2d_yz_8x8x8[35][2]), + offset8x8x8(c2d_yz_8x8x8[36][0]-1, c2d_yz_8x8x8[36][1]-1, c2d_yz_8x8x8[36][2]), + offset8x8x8(c2d_yz_8x8x8[37][0]-1, c2d_yz_8x8x8[37][1]-1, c2d_yz_8x8x8[37][2]), + offset8x8x8(c2d_yz_8x8x8[38][0]-1, c2d_yz_8x8x8[38][1]-1, c2d_yz_8x8x8[38][2]), + offset8x8x8(c2d_yz_8x8x8[39][0]-1, c2d_yz_8x8x8[39][1]-1, c2d_yz_8x8x8[39][2]), + offset8x8x8(c2d_yz_8x8x8[40][0]-1, c2d_yz_8x8x8[40][1]-1, c2d_yz_8x8x8[40][2]), + offset8x8x8(c2d_yz_8x8x8[41][0]-1, c2d_yz_8x8x8[41][1]-1, c2d_yz_8x8x8[41][2]), + offset8x8x8(c2d_yz_8x8x8[42][0]-1, c2d_yz_8x8x8[42][1]-1, c2d_yz_8x8x8[42][2]), + offset8x8x8(c2d_yz_8x8x8[43][0]-1, c2d_yz_8x8x8[43][1]-1, c2d_yz_8x8x8[43][2]), + offset8x8x8(c2d_yz_8x8x8[44][0]-1, c2d_yz_8x8x8[44][1]-1, c2d_yz_8x8x8[44][2])}; + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_MMM_Offset_8x8x8(SIZE i) { + static constexpr int offset[27] = {offset8x8x8(c3d_xyz_8x8x8[0][0], c3d_xyz_8x8x8[0][1], c3d_xyz_8x8x8[0][2]), // XY + offset8x8x8(c3d_xyz_8x8x8[1][0], c3d_xyz_8x8x8[1][1], c3d_xyz_8x8x8[1][2]), + offset8x8x8(c3d_xyz_8x8x8[2][0], c3d_xyz_8x8x8[2][1], c3d_xyz_8x8x8[2][2]), + offset8x8x8(c3d_xyz_8x8x8[3][0], c3d_xyz_8x8x8[3][1], c3d_xyz_8x8x8[3][2]), + offset8x8x8(c3d_xyz_8x8x8[4][0], c3d_xyz_8x8x8[4][1], c3d_xyz_8x8x8[4][2]), + offset8x8x8(c3d_xyz_8x8x8[5][0], c3d_xyz_8x8x8[5][1], c3d_xyz_8x8x8[5][2]), + offset8x8x8(c3d_xyz_8x8x8[6][0], c3d_xyz_8x8x8[6][1], c3d_xyz_8x8x8[6][2]), + offset8x8x8(c3d_xyz_8x8x8[7][0], c3d_xyz_8x8x8[7][1], c3d_xyz_8x8x8[7][2]), + offset8x8x8(c3d_xyz_8x8x8[8][0], c3d_xyz_8x8x8[8][1], c3d_xyz_8x8x8[8][2]), + offset8x8x8(c3d_xyz_8x8x8[9][0], c3d_xyz_8x8x8[9][1], c3d_xyz_8x8x8[9][2]), + offset8x8x8(c3d_xyz_8x8x8[10][0], c3d_xyz_8x8x8[10][1], c3d_xyz_8x8x8[10][2]), + offset8x8x8(c3d_xyz_8x8x8[11][0], c3d_xyz_8x8x8[11][1], c3d_xyz_8x8x8[11][2]), + offset8x8x8(c3d_xyz_8x8x8[12][0], c3d_xyz_8x8x8[12][1], c3d_xyz_8x8x8[12][2]), + offset8x8x8(c3d_xyz_8x8x8[13][0], c3d_xyz_8x8x8[13][1], c3d_xyz_8x8x8[13][2]), + offset8x8x8(c3d_xyz_8x8x8[14][0], c3d_xyz_8x8x8[14][1], c3d_xyz_8x8x8[14][2]), + offset8x8x8(c3d_xyz_8x8x8[15][0], c3d_xyz_8x8x8[15][1], c3d_xyz_8x8x8[15][2]), + offset8x8x8(c3d_xyz_8x8x8[16][0], c3d_xyz_8x8x8[16][1], c3d_xyz_8x8x8[16][2]), + offset8x8x8(c3d_xyz_8x8x8[17][0], c3d_xyz_8x8x8[17][1], c3d_xyz_8x8x8[17][2]), + offset8x8x8(c3d_xyz_8x8x8[18][0], c3d_xyz_8x8x8[18][1], c3d_xyz_8x8x8[18][2]), + offset8x8x8(c3d_xyz_8x8x8[19][0], c3d_xyz_8x8x8[19][1], c3d_xyz_8x8x8[19][2]), + offset8x8x8(c3d_xyz_8x8x8[20][0], c3d_xyz_8x8x8[20][1], c3d_xyz_8x8x8[20][2]), + offset8x8x8(c3d_xyz_8x8x8[21][0], c3d_xyz_8x8x8[21][1], c3d_xyz_8x8x8[21][2]), + offset8x8x8(c3d_xyz_8x8x8[22][0], c3d_xyz_8x8x8[22][1], c3d_xyz_8x8x8[22][2]), + offset8x8x8(c3d_xyz_8x8x8[23][0], c3d_xyz_8x8x8[23][1], c3d_xyz_8x8x8[23][2]), + offset8x8x8(c3d_xyz_8x8x8[24][0], c3d_xyz_8x8x8[24][1], c3d_xyz_8x8x8[24][2]), + offset8x8x8(c3d_xyz_8x8x8[25][0], c3d_xyz_8x8x8[25][1], c3d_xyz_8x8x8[25][2]), + offset8x8x8(c3d_xyz_8x8x8[26][0], c3d_xyz_8x8x8[26][1], c3d_xyz_8x8x8[26][2])}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_LLL_Offset_8x8x8(SIZE i) { + static constexpr int offset[27] = { offset8x8x8(c3d_xyz_8x8x8[0][0]-1, c3d_xyz_8x8x8[0][1]-1, c3d_xyz_8x8x8[0][2]-1), // XY + offset8x8x8(c3d_xyz_8x8x8[1][0]-1, c3d_xyz_8x8x8[1][1]-1, c3d_xyz_8x8x8[1][2]-1), + offset8x8x8(c3d_xyz_8x8x8[2][0]-1, c3d_xyz_8x8x8[2][1]-1, c3d_xyz_8x8x8[2][2]-1), + offset8x8x8(c3d_xyz_8x8x8[3][0]-1, c3d_xyz_8x8x8[3][1]-1, c3d_xyz_8x8x8[3][2]-1), + offset8x8x8(c3d_xyz_8x8x8[4][0]-1, c3d_xyz_8x8x8[4][1]-1, c3d_xyz_8x8x8[4][2]-1), + offset8x8x8(c3d_xyz_8x8x8[5][0]-1, c3d_xyz_8x8x8[5][1]-1, c3d_xyz_8x8x8[5][2]-1), + offset8x8x8(c3d_xyz_8x8x8[6][0]-1, c3d_xyz_8x8x8[6][1]-1, c3d_xyz_8x8x8[6][2]-1), + offset8x8x8(c3d_xyz_8x8x8[7][0]-1, c3d_xyz_8x8x8[7][1]-1, c3d_xyz_8x8x8[7][2]-1), + offset8x8x8(c3d_xyz_8x8x8[8][0]-1, c3d_xyz_8x8x8[8][1]-1, c3d_xyz_8x8x8[8][2]-1), + offset8x8x8(c3d_xyz_8x8x8[9][0]-1, c3d_xyz_8x8x8[9][1]-1, c3d_xyz_8x8x8[9][2]-1), + offset8x8x8(c3d_xyz_8x8x8[10][0]-1, c3d_xyz_8x8x8[10][1]-1, c3d_xyz_8x8x8[10][2]-1), + offset8x8x8(c3d_xyz_8x8x8[11][0]-1, c3d_xyz_8x8x8[11][1]-1, c3d_xyz_8x8x8[11][2]-1), + offset8x8x8(c3d_xyz_8x8x8[12][0]-1, c3d_xyz_8x8x8[12][1]-1, c3d_xyz_8x8x8[12][2]-1), + offset8x8x8(c3d_xyz_8x8x8[13][0]-1, c3d_xyz_8x8x8[13][1]-1, c3d_xyz_8x8x8[13][2]-1), + offset8x8x8(c3d_xyz_8x8x8[14][0]-1, c3d_xyz_8x8x8[14][1]-1, c3d_xyz_8x8x8[14][2]-1), + offset8x8x8(c3d_xyz_8x8x8[15][0]-1, c3d_xyz_8x8x8[15][1]-1, c3d_xyz_8x8x8[15][2]-1), + offset8x8x8(c3d_xyz_8x8x8[16][0]-1, c3d_xyz_8x8x8[16][1]-1, c3d_xyz_8x8x8[16][2]-1), + offset8x8x8(c3d_xyz_8x8x8[17][0]-1, c3d_xyz_8x8x8[17][1]-1, c3d_xyz_8x8x8[17][2]-1), + offset8x8x8(c3d_xyz_8x8x8[18][0]-1, c3d_xyz_8x8x8[18][1]-1, c3d_xyz_8x8x8[18][2]-1), + offset8x8x8(c3d_xyz_8x8x8[19][0]-1, c3d_xyz_8x8x8[19][1]-1, c3d_xyz_8x8x8[19][2]-1), + offset8x8x8(c3d_xyz_8x8x8[20][0]-1, c3d_xyz_8x8x8[20][1]-1, c3d_xyz_8x8x8[20][2]-1), + offset8x8x8(c3d_xyz_8x8x8[21][0]-1, c3d_xyz_8x8x8[21][1]-1, c3d_xyz_8x8x8[21][2]-1), + offset8x8x8(c3d_xyz_8x8x8[22][0]-1, c3d_xyz_8x8x8[22][1]-1, c3d_xyz_8x8x8[22][2]-1), + offset8x8x8(c3d_xyz_8x8x8[23][0]-1, c3d_xyz_8x8x8[23][1]-1, c3d_xyz_8x8x8[23][2]-1), + offset8x8x8(c3d_xyz_8x8x8[24][0]-1, c3d_xyz_8x8x8[24][1]-1, c3d_xyz_8x8x8[24][2]-1), + offset8x8x8(c3d_xyz_8x8x8[25][0]-1, c3d_xyz_8x8x8[25][1]-1, c3d_xyz_8x8x8[25][2]-1), + offset8x8x8(c3d_xyz_8x8x8[26][0]-1, c3d_xyz_8x8x8[26][1]-1, c3d_xyz_8x8x8[26][2]-1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_LLR_Offset_8x8x8(SIZE i) { + static constexpr int offset[27] = { offset8x8x8(c3d_xyz_8x8x8[0][0]-1, c3d_xyz_8x8x8[0][1]-1, c3d_xyz_8x8x8[0][2]+1), // XY + offset8x8x8(c3d_xyz_8x8x8[1][0]-1, c3d_xyz_8x8x8[1][1]-1, c3d_xyz_8x8x8[1][2]+1), + offset8x8x8(c3d_xyz_8x8x8[2][0]-1, c3d_xyz_8x8x8[2][1]-1, c3d_xyz_8x8x8[2][2]+1), + offset8x8x8(c3d_xyz_8x8x8[3][0]-1, c3d_xyz_8x8x8[3][1]-1, c3d_xyz_8x8x8[3][2]+1), + offset8x8x8(c3d_xyz_8x8x8[4][0]-1, c3d_xyz_8x8x8[4][1]-1, c3d_xyz_8x8x8[4][2]+1), + offset8x8x8(c3d_xyz_8x8x8[5][0]-1, c3d_xyz_8x8x8[5][1]-1, c3d_xyz_8x8x8[5][2]+1), + offset8x8x8(c3d_xyz_8x8x8[6][0]-1, c3d_xyz_8x8x8[6][1]-1, c3d_xyz_8x8x8[6][2]+1), + offset8x8x8(c3d_xyz_8x8x8[7][0]-1, c3d_xyz_8x8x8[7][1]-1, c3d_xyz_8x8x8[7][2]+1), + offset8x8x8(c3d_xyz_8x8x8[8][0]-1, c3d_xyz_8x8x8[8][1]-1, c3d_xyz_8x8x8[8][2]+1), + offset8x8x8(c3d_xyz_8x8x8[9][0]-1, c3d_xyz_8x8x8[9][1]-1, c3d_xyz_8x8x8[9][2]+1), + offset8x8x8(c3d_xyz_8x8x8[10][0]-1, c3d_xyz_8x8x8[10][1]-1, c3d_xyz_8x8x8[10][2]+1), + offset8x8x8(c3d_xyz_8x8x8[11][0]-1, c3d_xyz_8x8x8[11][1]-1, c3d_xyz_8x8x8[11][2]+1), + offset8x8x8(c3d_xyz_8x8x8[12][0]-1, c3d_xyz_8x8x8[12][1]-1, c3d_xyz_8x8x8[12][2]+1), + offset8x8x8(c3d_xyz_8x8x8[13][0]-1, c3d_xyz_8x8x8[13][1]-1, c3d_xyz_8x8x8[13][2]+1), + offset8x8x8(c3d_xyz_8x8x8[14][0]-1, c3d_xyz_8x8x8[14][1]-1, c3d_xyz_8x8x8[14][2]+1), + offset8x8x8(c3d_xyz_8x8x8[15][0]-1, c3d_xyz_8x8x8[15][1]-1, c3d_xyz_8x8x8[15][2]+1), + offset8x8x8(c3d_xyz_8x8x8[16][0]-1, c3d_xyz_8x8x8[16][1]-1, c3d_xyz_8x8x8[16][2]+1), + offset8x8x8(c3d_xyz_8x8x8[17][0]-1, c3d_xyz_8x8x8[17][1]-1, c3d_xyz_8x8x8[17][2]+1), + offset8x8x8(c3d_xyz_8x8x8[18][0]-1, c3d_xyz_8x8x8[18][1]-1, c3d_xyz_8x8x8[18][2]+1), + offset8x8x8(c3d_xyz_8x8x8[19][0]-1, c3d_xyz_8x8x8[19][1]-1, c3d_xyz_8x8x8[19][2]+1), + offset8x8x8(c3d_xyz_8x8x8[20][0]-1, c3d_xyz_8x8x8[20][1]-1, c3d_xyz_8x8x8[20][2]+1), + offset8x8x8(c3d_xyz_8x8x8[21][0]-1, c3d_xyz_8x8x8[21][1]-1, c3d_xyz_8x8x8[21][2]+1), + offset8x8x8(c3d_xyz_8x8x8[22][0]-1, c3d_xyz_8x8x8[22][1]-1, c3d_xyz_8x8x8[22][2]+1), + offset8x8x8(c3d_xyz_8x8x8[23][0]-1, c3d_xyz_8x8x8[23][1]-1, c3d_xyz_8x8x8[23][2]+1), + offset8x8x8(c3d_xyz_8x8x8[24][0]-1, c3d_xyz_8x8x8[24][1]-1, c3d_xyz_8x8x8[24][2]+1), + offset8x8x8(c3d_xyz_8x8x8[25][0]-1, c3d_xyz_8x8x8[25][1]-1, c3d_xyz_8x8x8[25][2]+1), + offset8x8x8(c3d_xyz_8x8x8[26][0]-1, c3d_xyz_8x8x8[26][1]-1, c3d_xyz_8x8x8[26][2]+1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_LRL_Offset_8x8x8(SIZE i) { + static constexpr int offset[27] = { offset8x8x8(c3d_xyz_8x8x8[0][0]-1, c3d_xyz_8x8x8[0][1]+1, c3d_xyz_8x8x8[0][2]-1), // XY + offset8x8x8(c3d_xyz_8x8x8[1][0]-1, c3d_xyz_8x8x8[1][1]+1, c3d_xyz_8x8x8[1][2]-1), + offset8x8x8(c3d_xyz_8x8x8[2][0]-1, c3d_xyz_8x8x8[2][1]+1, c3d_xyz_8x8x8[2][2]-1), + offset8x8x8(c3d_xyz_8x8x8[3][0]-1, c3d_xyz_8x8x8[3][1]+1, c3d_xyz_8x8x8[3][2]-1), + offset8x8x8(c3d_xyz_8x8x8[4][0]-1, c3d_xyz_8x8x8[4][1]+1, c3d_xyz_8x8x8[4][2]-1), + offset8x8x8(c3d_xyz_8x8x8[5][0]-1, c3d_xyz_8x8x8[5][1]+1, c3d_xyz_8x8x8[5][2]-1), + offset8x8x8(c3d_xyz_8x8x8[6][0]-1, c3d_xyz_8x8x8[6][1]+1, c3d_xyz_8x8x8[6][2]-1), + offset8x8x8(c3d_xyz_8x8x8[7][0]-1, c3d_xyz_8x8x8[7][1]+1, c3d_xyz_8x8x8[7][2]-1), + offset8x8x8(c3d_xyz_8x8x8[8][0]-1, c3d_xyz_8x8x8[8][1]+1, c3d_xyz_8x8x8[8][2]-1), + offset8x8x8(c3d_xyz_8x8x8[9][0]-1, c3d_xyz_8x8x8[9][1]+1, c3d_xyz_8x8x8[9][2]-1), + offset8x8x8(c3d_xyz_8x8x8[10][0]-1, c3d_xyz_8x8x8[10][1]+1, c3d_xyz_8x8x8[10][2]-1), + offset8x8x8(c3d_xyz_8x8x8[11][0]-1, c3d_xyz_8x8x8[11][1]+1, c3d_xyz_8x8x8[11][2]-1), + offset8x8x8(c3d_xyz_8x8x8[12][0]-1, c3d_xyz_8x8x8[12][1]+1, c3d_xyz_8x8x8[12][2]-1), + offset8x8x8(c3d_xyz_8x8x8[13][0]-1, c3d_xyz_8x8x8[13][1]+1, c3d_xyz_8x8x8[13][2]-1), + offset8x8x8(c3d_xyz_8x8x8[14][0]-1, c3d_xyz_8x8x8[14][1]+1, c3d_xyz_8x8x8[14][2]-1), + offset8x8x8(c3d_xyz_8x8x8[15][0]-1, c3d_xyz_8x8x8[15][1]+1, c3d_xyz_8x8x8[15][2]-1), + offset8x8x8(c3d_xyz_8x8x8[16][0]-1, c3d_xyz_8x8x8[16][1]+1, c3d_xyz_8x8x8[16][2]-1), + offset8x8x8(c3d_xyz_8x8x8[17][0]-1, c3d_xyz_8x8x8[17][1]+1, c3d_xyz_8x8x8[17][2]-1), + offset8x8x8(c3d_xyz_8x8x8[18][0]-1, c3d_xyz_8x8x8[18][1]+1, c3d_xyz_8x8x8[18][2]-1), + offset8x8x8(c3d_xyz_8x8x8[19][0]-1, c3d_xyz_8x8x8[19][1]+1, c3d_xyz_8x8x8[19][2]-1), + offset8x8x8(c3d_xyz_8x8x8[20][0]-1, c3d_xyz_8x8x8[20][1]+1, c3d_xyz_8x8x8[20][2]-1), + offset8x8x8(c3d_xyz_8x8x8[21][0]-1, c3d_xyz_8x8x8[21][1]+1, c3d_xyz_8x8x8[21][2]-1), + offset8x8x8(c3d_xyz_8x8x8[22][0]-1, c3d_xyz_8x8x8[22][1]+1, c3d_xyz_8x8x8[22][2]-1), + offset8x8x8(c3d_xyz_8x8x8[23][0]-1, c3d_xyz_8x8x8[23][1]+1, c3d_xyz_8x8x8[23][2]-1), + offset8x8x8(c3d_xyz_8x8x8[24][0]-1, c3d_xyz_8x8x8[24][1]+1, c3d_xyz_8x8x8[24][2]-1), + offset8x8x8(c3d_xyz_8x8x8[25][0]-1, c3d_xyz_8x8x8[25][1]+1, c3d_xyz_8x8x8[25][2]-1), + offset8x8x8(c3d_xyz_8x8x8[26][0]-1, c3d_xyz_8x8x8[26][1]+1, c3d_xyz_8x8x8[26][2]-1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_LRR_Offset_8x8x8(SIZE i) { + static constexpr int offset[27] = { offset8x8x8(c3d_xyz_8x8x8[0][0]-1, c3d_xyz_8x8x8[0][1]+1, c3d_xyz_8x8x8[0][2]+1), // XY + offset8x8x8(c3d_xyz_8x8x8[1][0]-1, c3d_xyz_8x8x8[1][1]+1, c3d_xyz_8x8x8[1][2]+1), + offset8x8x8(c3d_xyz_8x8x8[2][0]-1, c3d_xyz_8x8x8[2][1]+1, c3d_xyz_8x8x8[2][2]+1), + offset8x8x8(c3d_xyz_8x8x8[3][0]-1, c3d_xyz_8x8x8[3][1]+1, c3d_xyz_8x8x8[3][2]+1), + offset8x8x8(c3d_xyz_8x8x8[4][0]-1, c3d_xyz_8x8x8[4][1]+1, c3d_xyz_8x8x8[4][2]+1), + offset8x8x8(c3d_xyz_8x8x8[5][0]-1, c3d_xyz_8x8x8[5][1]+1, c3d_xyz_8x8x8[5][2]+1), + offset8x8x8(c3d_xyz_8x8x8[6][0]-1, c3d_xyz_8x8x8[6][1]+1, c3d_xyz_8x8x8[6][2]+1), + offset8x8x8(c3d_xyz_8x8x8[7][0]-1, c3d_xyz_8x8x8[7][1]+1, c3d_xyz_8x8x8[7][2]+1), + offset8x8x8(c3d_xyz_8x8x8[8][0]-1, c3d_xyz_8x8x8[8][1]+1, c3d_xyz_8x8x8[8][2]+1), + offset8x8x8(c3d_xyz_8x8x8[9][0]-1, c3d_xyz_8x8x8[9][1]+1, c3d_xyz_8x8x8[9][2]+1), + offset8x8x8(c3d_xyz_8x8x8[10][0]-1, c3d_xyz_8x8x8[10][1]+1, c3d_xyz_8x8x8[10][2]+1), + offset8x8x8(c3d_xyz_8x8x8[11][0]-1, c3d_xyz_8x8x8[11][1]+1, c3d_xyz_8x8x8[11][2]+1), + offset8x8x8(c3d_xyz_8x8x8[12][0]-1, c3d_xyz_8x8x8[12][1]+1, c3d_xyz_8x8x8[12][2]+1), + offset8x8x8(c3d_xyz_8x8x8[13][0]-1, c3d_xyz_8x8x8[13][1]+1, c3d_xyz_8x8x8[13][2]+1), + offset8x8x8(c3d_xyz_8x8x8[14][0]-1, c3d_xyz_8x8x8[14][1]+1, c3d_xyz_8x8x8[14][2]+1), + offset8x8x8(c3d_xyz_8x8x8[15][0]-1, c3d_xyz_8x8x8[15][1]+1, c3d_xyz_8x8x8[15][2]+1), + offset8x8x8(c3d_xyz_8x8x8[16][0]-1, c3d_xyz_8x8x8[16][1]+1, c3d_xyz_8x8x8[16][2]+1), + offset8x8x8(c3d_xyz_8x8x8[17][0]-1, c3d_xyz_8x8x8[17][1]+1, c3d_xyz_8x8x8[17][2]+1), + offset8x8x8(c3d_xyz_8x8x8[18][0]-1, c3d_xyz_8x8x8[18][1]+1, c3d_xyz_8x8x8[18][2]+1), + offset8x8x8(c3d_xyz_8x8x8[19][0]-1, c3d_xyz_8x8x8[19][1]+1, c3d_xyz_8x8x8[19][2]+1), + offset8x8x8(c3d_xyz_8x8x8[20][0]-1, c3d_xyz_8x8x8[20][1]+1, c3d_xyz_8x8x8[20][2]+1), + offset8x8x8(c3d_xyz_8x8x8[21][0]-1, c3d_xyz_8x8x8[21][1]+1, c3d_xyz_8x8x8[21][2]+1), + offset8x8x8(c3d_xyz_8x8x8[22][0]-1, c3d_xyz_8x8x8[22][1]+1, c3d_xyz_8x8x8[22][2]+1), + offset8x8x8(c3d_xyz_8x8x8[23][0]-1, c3d_xyz_8x8x8[23][1]+1, c3d_xyz_8x8x8[23][2]+1), + offset8x8x8(c3d_xyz_8x8x8[24][0]-1, c3d_xyz_8x8x8[24][1]+1, c3d_xyz_8x8x8[24][2]+1), + offset8x8x8(c3d_xyz_8x8x8[25][0]-1, c3d_xyz_8x8x8[25][1]+1, c3d_xyz_8x8x8[25][2]+1), + offset8x8x8(c3d_xyz_8x8x8[26][0]-1, c3d_xyz_8x8x8[26][1]+1, c3d_xyz_8x8x8[26][2]+1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_RLL_Offset_8x8x8(SIZE i) { + static constexpr int offset[27] = { offset8x8x8(c3d_xyz_8x8x8[0][0]+1, c3d_xyz_8x8x8[0][1]-1, c3d_xyz_8x8x8[0][2]-1), // XY + offset8x8x8(c3d_xyz_8x8x8[1][0]+1, c3d_xyz_8x8x8[1][1]-1, c3d_xyz_8x8x8[1][2]-1), + offset8x8x8(c3d_xyz_8x8x8[2][0]+1, c3d_xyz_8x8x8[2][1]-1, c3d_xyz_8x8x8[2][2]-1), + offset8x8x8(c3d_xyz_8x8x8[3][0]+1, c3d_xyz_8x8x8[3][1]-1, c3d_xyz_8x8x8[3][2]-1), + offset8x8x8(c3d_xyz_8x8x8[4][0]+1, c3d_xyz_8x8x8[4][1]-1, c3d_xyz_8x8x8[4][2]-1), + offset8x8x8(c3d_xyz_8x8x8[5][0]+1, c3d_xyz_8x8x8[5][1]-1, c3d_xyz_8x8x8[5][2]-1), + offset8x8x8(c3d_xyz_8x8x8[6][0]+1, c3d_xyz_8x8x8[6][1]-1, c3d_xyz_8x8x8[6][2]-1), + offset8x8x8(c3d_xyz_8x8x8[7][0]+1, c3d_xyz_8x8x8[7][1]-1, c3d_xyz_8x8x8[7][2]-1), + offset8x8x8(c3d_xyz_8x8x8[8][0]+1, c3d_xyz_8x8x8[8][1]-1, c3d_xyz_8x8x8[8][2]-1), + offset8x8x8(c3d_xyz_8x8x8[9][0]+1, c3d_xyz_8x8x8[9][1]-1, c3d_xyz_8x8x8[9][2]-1), + offset8x8x8(c3d_xyz_8x8x8[10][0]+1, c3d_xyz_8x8x8[10][1]-1, c3d_xyz_8x8x8[10][2]-1), + offset8x8x8(c3d_xyz_8x8x8[11][0]+1, c3d_xyz_8x8x8[11][1]-1, c3d_xyz_8x8x8[11][2]-1), + offset8x8x8(c3d_xyz_8x8x8[12][0]+1, c3d_xyz_8x8x8[12][1]-1, c3d_xyz_8x8x8[12][2]-1), + offset8x8x8(c3d_xyz_8x8x8[13][0]+1, c3d_xyz_8x8x8[13][1]-1, c3d_xyz_8x8x8[13][2]-1), + offset8x8x8(c3d_xyz_8x8x8[14][0]+1, c3d_xyz_8x8x8[14][1]-1, c3d_xyz_8x8x8[14][2]-1), + offset8x8x8(c3d_xyz_8x8x8[15][0]+1, c3d_xyz_8x8x8[15][1]-1, c3d_xyz_8x8x8[15][2]-1), + offset8x8x8(c3d_xyz_8x8x8[16][0]+1, c3d_xyz_8x8x8[16][1]-1, c3d_xyz_8x8x8[16][2]-1), + offset8x8x8(c3d_xyz_8x8x8[17][0]+1, c3d_xyz_8x8x8[17][1]-1, c3d_xyz_8x8x8[17][2]-1), + offset8x8x8(c3d_xyz_8x8x8[18][0]+1, c3d_xyz_8x8x8[18][1]-1, c3d_xyz_8x8x8[18][2]-1), + offset8x8x8(c3d_xyz_8x8x8[19][0]+1, c3d_xyz_8x8x8[19][1]-1, c3d_xyz_8x8x8[19][2]-1), + offset8x8x8(c3d_xyz_8x8x8[20][0]+1, c3d_xyz_8x8x8[20][1]-1, c3d_xyz_8x8x8[20][2]-1), + offset8x8x8(c3d_xyz_8x8x8[21][0]+1, c3d_xyz_8x8x8[21][1]-1, c3d_xyz_8x8x8[21][2]-1), + offset8x8x8(c3d_xyz_8x8x8[22][0]+1, c3d_xyz_8x8x8[22][1]-1, c3d_xyz_8x8x8[22][2]-1), + offset8x8x8(c3d_xyz_8x8x8[23][0]+1, c3d_xyz_8x8x8[23][1]-1, c3d_xyz_8x8x8[23][2]-1), + offset8x8x8(c3d_xyz_8x8x8[24][0]+1, c3d_xyz_8x8x8[24][1]-1, c3d_xyz_8x8x8[24][2]-1), + offset8x8x8(c3d_xyz_8x8x8[25][0]+1, c3d_xyz_8x8x8[25][1]-1, c3d_xyz_8x8x8[25][2]-1), + offset8x8x8(c3d_xyz_8x8x8[26][0]+1, c3d_xyz_8x8x8[26][1]-1, c3d_xyz_8x8x8[26][2]-1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_RLR_Offset_8x8x8(SIZE i) { + static constexpr int offset[27] = { offset8x8x8(c3d_xyz_8x8x8[0][0]+1, c3d_xyz_8x8x8[0][1]-1, c3d_xyz_8x8x8[0][2]+1), // XY + offset8x8x8(c3d_xyz_8x8x8[1][0]+1, c3d_xyz_8x8x8[1][1]-1, c3d_xyz_8x8x8[1][2]+1), + offset8x8x8(c3d_xyz_8x8x8[2][0]+1, c3d_xyz_8x8x8[2][1]-1, c3d_xyz_8x8x8[2][2]+1), + offset8x8x8(c3d_xyz_8x8x8[3][0]+1, c3d_xyz_8x8x8[3][1]-1, c3d_xyz_8x8x8[3][2]+1), + offset8x8x8(c3d_xyz_8x8x8[4][0]+1, c3d_xyz_8x8x8[4][1]-1, c3d_xyz_8x8x8[4][2]+1), + offset8x8x8(c3d_xyz_8x8x8[5][0]+1, c3d_xyz_8x8x8[5][1]-1, c3d_xyz_8x8x8[5][2]+1), + offset8x8x8(c3d_xyz_8x8x8[6][0]+1, c3d_xyz_8x8x8[6][1]-1, c3d_xyz_8x8x8[6][2]+1), + offset8x8x8(c3d_xyz_8x8x8[7][0]+1, c3d_xyz_8x8x8[7][1]-1, c3d_xyz_8x8x8[7][2]+1), + offset8x8x8(c3d_xyz_8x8x8[8][0]+1, c3d_xyz_8x8x8[8][1]-1, c3d_xyz_8x8x8[8][2]+1), + offset8x8x8(c3d_xyz_8x8x8[9][0]+1, c3d_xyz_8x8x8[9][1]-1, c3d_xyz_8x8x8[9][2]+1), + offset8x8x8(c3d_xyz_8x8x8[10][0]+1, c3d_xyz_8x8x8[10][1]-1, c3d_xyz_8x8x8[10][2]+1), + offset8x8x8(c3d_xyz_8x8x8[11][0]+1, c3d_xyz_8x8x8[11][1]-1, c3d_xyz_8x8x8[11][2]+1), + offset8x8x8(c3d_xyz_8x8x8[12][0]+1, c3d_xyz_8x8x8[12][1]-1, c3d_xyz_8x8x8[12][2]+1), + offset8x8x8(c3d_xyz_8x8x8[13][0]+1, c3d_xyz_8x8x8[13][1]-1, c3d_xyz_8x8x8[13][2]+1), + offset8x8x8(c3d_xyz_8x8x8[14][0]+1, c3d_xyz_8x8x8[14][1]-1, c3d_xyz_8x8x8[14][2]+1), + offset8x8x8(c3d_xyz_8x8x8[15][0]+1, c3d_xyz_8x8x8[15][1]-1, c3d_xyz_8x8x8[15][2]+1), + offset8x8x8(c3d_xyz_8x8x8[16][0]+1, c3d_xyz_8x8x8[16][1]-1, c3d_xyz_8x8x8[16][2]+1), + offset8x8x8(c3d_xyz_8x8x8[17][0]+1, c3d_xyz_8x8x8[17][1]-1, c3d_xyz_8x8x8[17][2]+1), + offset8x8x8(c3d_xyz_8x8x8[18][0]+1, c3d_xyz_8x8x8[18][1]-1, c3d_xyz_8x8x8[18][2]+1), + offset8x8x8(c3d_xyz_8x8x8[19][0]+1, c3d_xyz_8x8x8[19][1]-1, c3d_xyz_8x8x8[19][2]+1), + offset8x8x8(c3d_xyz_8x8x8[20][0]+1, c3d_xyz_8x8x8[20][1]-1, c3d_xyz_8x8x8[20][2]+1), + offset8x8x8(c3d_xyz_8x8x8[21][0]+1, c3d_xyz_8x8x8[21][1]-1, c3d_xyz_8x8x8[21][2]+1), + offset8x8x8(c3d_xyz_8x8x8[22][0]+1, c3d_xyz_8x8x8[22][1]-1, c3d_xyz_8x8x8[22][2]+1), + offset8x8x8(c3d_xyz_8x8x8[23][0]+1, c3d_xyz_8x8x8[23][1]-1, c3d_xyz_8x8x8[23][2]+1), + offset8x8x8(c3d_xyz_8x8x8[24][0]+1, c3d_xyz_8x8x8[24][1]-1, c3d_xyz_8x8x8[24][2]+1), + offset8x8x8(c3d_xyz_8x8x8[25][0]+1, c3d_xyz_8x8x8[25][1]-1, c3d_xyz_8x8x8[25][2]+1), + offset8x8x8(c3d_xyz_8x8x8[26][0]+1, c3d_xyz_8x8x8[26][1]-1, c3d_xyz_8x8x8[26][2]+1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_RRL_Offset_8x8x8(SIZE i) { + static constexpr int offset[27] = { offset8x8x8(c3d_xyz_8x8x8[0][0]+1, c3d_xyz_8x8x8[0][1]+1, c3d_xyz_8x8x8[0][2]-1), // XY + offset8x8x8(c3d_xyz_8x8x8[1][0]+1, c3d_xyz_8x8x8[1][1]+1, c3d_xyz_8x8x8[1][2]-1), + offset8x8x8(c3d_xyz_8x8x8[2][0]+1, c3d_xyz_8x8x8[2][1]+1, c3d_xyz_8x8x8[2][2]-1), + offset8x8x8(c3d_xyz_8x8x8[3][0]+1, c3d_xyz_8x8x8[3][1]+1, c3d_xyz_8x8x8[3][2]-1), + offset8x8x8(c3d_xyz_8x8x8[4][0]+1, c3d_xyz_8x8x8[4][1]+1, c3d_xyz_8x8x8[4][2]-1), + offset8x8x8(c3d_xyz_8x8x8[5][0]+1, c3d_xyz_8x8x8[5][1]+1, c3d_xyz_8x8x8[5][2]-1), + offset8x8x8(c3d_xyz_8x8x8[6][0]+1, c3d_xyz_8x8x8[6][1]+1, c3d_xyz_8x8x8[6][2]-1), + offset8x8x8(c3d_xyz_8x8x8[7][0]+1, c3d_xyz_8x8x8[7][1]+1, c3d_xyz_8x8x8[7][2]-1), + offset8x8x8(c3d_xyz_8x8x8[8][0]+1, c3d_xyz_8x8x8[8][1]+1, c3d_xyz_8x8x8[8][2]-1), + offset8x8x8(c3d_xyz_8x8x8[9][0]+1, c3d_xyz_8x8x8[9][1]+1, c3d_xyz_8x8x8[9][2]-1), + offset8x8x8(c3d_xyz_8x8x8[10][0]+1, c3d_xyz_8x8x8[10][1]+1, c3d_xyz_8x8x8[10][2]-1), + offset8x8x8(c3d_xyz_8x8x8[11][0]+1, c3d_xyz_8x8x8[11][1]+1, c3d_xyz_8x8x8[11][2]-1), + offset8x8x8(c3d_xyz_8x8x8[12][0]+1, c3d_xyz_8x8x8[12][1]+1, c3d_xyz_8x8x8[12][2]-1), + offset8x8x8(c3d_xyz_8x8x8[13][0]+1, c3d_xyz_8x8x8[13][1]+1, c3d_xyz_8x8x8[13][2]-1), + offset8x8x8(c3d_xyz_8x8x8[14][0]+1, c3d_xyz_8x8x8[14][1]+1, c3d_xyz_8x8x8[14][2]-1), + offset8x8x8(c3d_xyz_8x8x8[15][0]+1, c3d_xyz_8x8x8[15][1]+1, c3d_xyz_8x8x8[15][2]-1), + offset8x8x8(c3d_xyz_8x8x8[16][0]+1, c3d_xyz_8x8x8[16][1]+1, c3d_xyz_8x8x8[16][2]-1), + offset8x8x8(c3d_xyz_8x8x8[17][0]+1, c3d_xyz_8x8x8[17][1]+1, c3d_xyz_8x8x8[17][2]-1), + offset8x8x8(c3d_xyz_8x8x8[18][0]+1, c3d_xyz_8x8x8[18][1]+1, c3d_xyz_8x8x8[18][2]-1), + offset8x8x8(c3d_xyz_8x8x8[19][0]+1, c3d_xyz_8x8x8[19][1]+1, c3d_xyz_8x8x8[19][2]-1), + offset8x8x8(c3d_xyz_8x8x8[20][0]+1, c3d_xyz_8x8x8[20][1]+1, c3d_xyz_8x8x8[20][2]-1), + offset8x8x8(c3d_xyz_8x8x8[21][0]+1, c3d_xyz_8x8x8[21][1]+1, c3d_xyz_8x8x8[21][2]-1), + offset8x8x8(c3d_xyz_8x8x8[22][0]+1, c3d_xyz_8x8x8[22][1]+1, c3d_xyz_8x8x8[22][2]-1), + offset8x8x8(c3d_xyz_8x8x8[23][0]+1, c3d_xyz_8x8x8[23][1]+1, c3d_xyz_8x8x8[23][2]-1), + offset8x8x8(c3d_xyz_8x8x8[24][0]+1, c3d_xyz_8x8x8[24][1]+1, c3d_xyz_8x8x8[24][2]-1), + offset8x8x8(c3d_xyz_8x8x8[25][0]+1, c3d_xyz_8x8x8[25][1]+1, c3d_xyz_8x8x8[25][2]-1), + offset8x8x8(c3d_xyz_8x8x8[26][0]+1, c3d_xyz_8x8x8[26][1]+1, c3d_xyz_8x8x8[26][2]-1)}; + + return offset[i]; +} + +MGARDX_EXEC int Coeff3D_RRR_Offset_8x8x8(SIZE i) { + static constexpr int offset[27] = { offset8x8x8(c3d_xyz_8x8x8[0][0]+1, c3d_xyz_8x8x8[0][1]+1, c3d_xyz_8x8x8[0][2]+1), // XY + offset8x8x8(c3d_xyz_8x8x8[1][0]+1, c3d_xyz_8x8x8[1][1]+1, c3d_xyz_8x8x8[1][2]+1), + offset8x8x8(c3d_xyz_8x8x8[2][0]+1, c3d_xyz_8x8x8[2][1]+1, c3d_xyz_8x8x8[2][2]+1), + offset8x8x8(c3d_xyz_8x8x8[3][0]+1, c3d_xyz_8x8x8[3][1]+1, c3d_xyz_8x8x8[3][2]+1), + offset8x8x8(c3d_xyz_8x8x8[4][0]+1, c3d_xyz_8x8x8[4][1]+1, c3d_xyz_8x8x8[4][2]+1), + offset8x8x8(c3d_xyz_8x8x8[5][0]+1, c3d_xyz_8x8x8[5][1]+1, c3d_xyz_8x8x8[5][2]+1), + offset8x8x8(c3d_xyz_8x8x8[6][0]+1, c3d_xyz_8x8x8[6][1]+1, c3d_xyz_8x8x8[6][2]+1), + offset8x8x8(c3d_xyz_8x8x8[7][0]+1, c3d_xyz_8x8x8[7][1]+1, c3d_xyz_8x8x8[7][2]+1), + offset8x8x8(c3d_xyz_8x8x8[8][0]+1, c3d_xyz_8x8x8[8][1]+1, c3d_xyz_8x8x8[8][2]+1), + offset8x8x8(c3d_xyz_8x8x8[9][0]+1, c3d_xyz_8x8x8[9][1]+1, c3d_xyz_8x8x8[9][2]+1), + offset8x8x8(c3d_xyz_8x8x8[10][0]+1, c3d_xyz_8x8x8[10][1]+1, c3d_xyz_8x8x8[10][2]+1), + offset8x8x8(c3d_xyz_8x8x8[11][0]+1, c3d_xyz_8x8x8[11][1]+1, c3d_xyz_8x8x8[11][2]+1), + offset8x8x8(c3d_xyz_8x8x8[12][0]+1, c3d_xyz_8x8x8[12][1]+1, c3d_xyz_8x8x8[12][2]+1), + offset8x8x8(c3d_xyz_8x8x8[13][0]+1, c3d_xyz_8x8x8[13][1]+1, c3d_xyz_8x8x8[13][2]+1), + offset8x8x8(c3d_xyz_8x8x8[14][0]+1, c3d_xyz_8x8x8[14][1]+1, c3d_xyz_8x8x8[14][2]+1), + offset8x8x8(c3d_xyz_8x8x8[15][0]+1, c3d_xyz_8x8x8[15][1]+1, c3d_xyz_8x8x8[15][2]+1), + offset8x8x8(c3d_xyz_8x8x8[16][0]+1, c3d_xyz_8x8x8[16][1]+1, c3d_xyz_8x8x8[16][2]+1), + offset8x8x8(c3d_xyz_8x8x8[17][0]+1, c3d_xyz_8x8x8[17][1]+1, c3d_xyz_8x8x8[17][2]+1), + offset8x8x8(c3d_xyz_8x8x8[18][0]+1, c3d_xyz_8x8x8[18][1]+1, c3d_xyz_8x8x8[18][2]+1), + offset8x8x8(c3d_xyz_8x8x8[19][0]+1, c3d_xyz_8x8x8[19][1]+1, c3d_xyz_8x8x8[19][2]+1), + offset8x8x8(c3d_xyz_8x8x8[20][0]+1, c3d_xyz_8x8x8[20][1]+1, c3d_xyz_8x8x8[20][2]+1), + offset8x8x8(c3d_xyz_8x8x8[21][0]+1, c3d_xyz_8x8x8[21][1]+1, c3d_xyz_8x8x8[21][2]+1), + offset8x8x8(c3d_xyz_8x8x8[22][0]+1, c3d_xyz_8x8x8[22][1]+1, c3d_xyz_8x8x8[22][2]+1), + offset8x8x8(c3d_xyz_8x8x8[23][0]+1, c3d_xyz_8x8x8[23][1]+1, c3d_xyz_8x8x8[23][2]+1), + offset8x8x8(c3d_xyz_8x8x8[24][0]+1, c3d_xyz_8x8x8[24][1]+1, c3d_xyz_8x8x8[24][2]+1), + offset8x8x8(c3d_xyz_8x8x8[25][0]+1, c3d_xyz_8x8x8[25][1]+1, c3d_xyz_8x8x8[25][2]+1), + offset8x8x8(c3d_xyz_8x8x8[26][0]+1, c3d_xyz_8x8x8[26][1]+1, c3d_xyz_8x8x8[26][2]+1)}; + + return offset[i]; +} + +MGARDX_EXEC int MassTrans_X_DistCase_8x8x8(SIZE i) { + #define DIST(Z, Y) 0, 1, 1, 2, 3 + + static constexpr int offset[320] = { + DIST(0, 0), DIST(0, 1), DIST(0, 2), DIST(0, 3), DIST(0, 4), DIST(0, 5), DIST(0, 6), DIST(0, 7), + DIST(1, 0), DIST(1, 1), DIST(1, 2), DIST(1, 3), DIST(1, 4), DIST(1, 5), DIST(1, 6), DIST(1, 7), + DIST(2, 0), DIST(2, 1), DIST(2, 2), DIST(2, 3), DIST(2, 4), DIST(2, 5), DIST(2, 6), DIST(2, 7), + DIST(3, 0), DIST(3, 1), DIST(3, 2), DIST(3, 3), DIST(3, 4), DIST(3, 5), DIST(3, 6), DIST(3, 7), + DIST(4, 0), DIST(4, 1), DIST(4, 2), DIST(4, 3), DIST(4, 4), DIST(4, 5), DIST(4, 6), DIST(4, 7), + DIST(5, 0), DIST(5, 1), DIST(5, 2), DIST(5, 3), DIST(5, 4), DIST(5, 5), DIST(5, 6), DIST(5, 7), + DIST(6, 0), DIST(6, 1), DIST(6, 2), DIST(6, 3), DIST(6, 4), DIST(6, 5), DIST(6, 6), DIST(6, 7), + DIST(7, 0), DIST(7, 1), DIST(7, 2), DIST(7, 3), DIST(7, 4), DIST(7, 5), DIST(7, 6), DIST(7, 7) + }; + #undef DIST + return offset[i]; +} + + +template +MGARDX_EXEC constexpr T wa(T h1, T h2, T h3, T h4) { + T r1 = 0.0; + if (h1 + h2 != 0) { + r1 = h1 / (h1 + h2); + } + return (h1 / 6) * r1; +} + +template +MGARDX_EXEC constexpr T wb(T h1, T h2, T h3, T h4) { + T r1 = 0.0; + if (h1 + h2 != 0) { + r1 = h1 / (h1 + h2); + } + return ((h1 + h2) / 3) * r1 + (h2 / 6); +} + +template +MGARDX_EXEC constexpr T wc(T h1, T h2, T h3, T h4) { + T r1 = 0.0; + if (h1 + h2 != 0) { + r1 = h1 / (h1 + h2); + } + T r4 = 0.0; + if (h3 + h4 != 0) { + r4 = h4 / (h3 + h4); + } + return ((h2 + h3) / 3) + (h2 / 6) * r1 + (h3 / 6) * r4; +} + +template +MGARDX_EXEC constexpr T wd(T h1, T h2, T h3, T h4) { + T r4 = 0.0; + if (h3 + h4 != 0) { + r4 = h4 / (h3 + h4); + } + return ((h3 + h4) / 3) * r4 + (h3 / 6); +} + +template +MGARDX_EXEC constexpr T we(T h1, T h2, T h3, T h4) { + T r4 = 0.0; + if (h3 + h4 != 0) { + r4 = h4 / (h3 + h4); + } + return (h4 / 6) * r4; +} + +template +MGARDX_EXEC T const *MassTrans_Weights_8x8x8(SIZE i) { + + static constexpr T offset[5][5] = { + {wa(0.0, 0.0, 1.0, 1.0), wb(0.0, 0.0, 1.0, 1.0), wc(0.0, 0.0, 1.0, 1.0), wd(0.0, 0.0, 1.0, 1.0), we(0.0, 0.0, 1.0, 1.0)}, + {wa(1.0, 1.0, 1.0, 1.0), wb(1.0, 1.0, 1.0, 1.0), wc(1.0, 1.0, 1.0, 1.0), wd(1.0, 1.0, 1.0, 1.0), we(1.0, 1.0, 1.0, 1.0)}, + {wa(1.0, 1.0, 1.0, 1.0), wb(1.0, 1.0, 1.0, 1.0), wc(1.0, 1.0, 1.0, 1.0), wd(1.0, 1.0, 1.0, 1.0), we(1.0, 1.0, 1.0, 1.0)}, + {wa(1.0, 1.0, 0.5, 0.5), wb(1.0, 1.0, 0.5, 0.5), wc(1.0, 1.0, 0.5, 0.5), wd(1.0, 1.0, 0.5, 0.5), we(1.0, 1.0, 0.5, 0.5)}, + {wa(0.5, 0.5, 0.0, 0.0), wb(0.5, 0.5, 0.0, 0.0), wc(0.5, 0.5, 0.0, 0.0), wd(0.5, 0.5, 0.0, 0.0), we(0.5, 0.5, 0.0, 0.0)} + }; + return offset[i]; +} + +MGARDX_EXEC int const *MassTrans_X_Offset_8x8x8(SIZE i) { + static constexpr int zero_offset = SMV_SIZE_8x8x8 + 8*8*5 + 8*5*5 + 5*5*5; + #define OFFSET1(Z, Y) \ + { \ + zero_offset, \ + zero_offset, \ + zero_offset, \ + offset8x8x8(Z, Y, coarse_x_8x8x8[0]+1, 8, 8), \ + zero_offset, \ + offset8x8x8(Z, Y, 0, 5, 8), \ + 0 \ + }, \ + { \ + zero_offset, \ + offset8x8x8(Z, Y, coarse_x_8x8x8[1]-1, 8, 8), \ + zero_offset, \ + offset8x8x8(Z, Y, coarse_x_8x8x8[1]+1, 8, 8), \ + zero_offset, \ + offset8x8x8(Z, Y, 1, 5, 8), \ + 1 \ + }, \ + { \ + zero_offset, \ + offset8x8x8(Z, Y, coarse_x_8x8x8[2]-1, 8, 8), \ + zero_offset, \ + offset8x8x8(Z, Y, coarse_x_8x8x8[2]+1, 8, 8), \ + zero_offset, \ + offset8x8x8(Z, Y, 2, 5, 8), \ + 2 \ + }, \ + { \ + zero_offset, \ + offset8x8x8(Z, Y, coarse_x_8x8x8[3]-1, 8, 8), \ + zero_offset, \ + zero_offset, \ + zero_offset, \ + offset8x8x8(Z, Y, 3, 5, 8), \ + 3 \ + }, \ + { \ + zero_offset, \ + zero_offset, \ + zero_offset, \ + zero_offset, \ + zero_offset, \ + offset8x8x8(Z, Y, 4, 5, 8), \ + 4 \ + } + + #define OFFSET2(Z, Y) \ + { \ + zero_offset, \ + zero_offset, \ + offset8x8x8(Z, Y, coarse_x_8x8x8[0], 8, 8), \ + offset8x8x8(Z, Y, coarse_x_8x8x8[0]+1, 8, 8), \ + offset8x8x8(Z, Y, coarse_x_8x8x8[0]+2, 8, 8), \ + offset8x8x8(Z, Y, 0, 5, 8), \ + 0 \ + }, \ + { \ + offset8x8x8(Z, Y, coarse_x_8x8x8[1]-2, 8, 8), \ + offset8x8x8(Z, Y, coarse_x_8x8x8[1]-1, 8, 8), \ + offset8x8x8(Z, Y, coarse_x_8x8x8[1], 8, 8), \ + offset8x8x8(Z, Y, coarse_x_8x8x8[1]+1, 8, 8), \ + offset8x8x8(Z, Y, coarse_x_8x8x8[1]+2, 8, 8), \ + offset8x8x8(Z, Y, 1, 5, 8), \ + 1 \ + }, \ + { \ + offset8x8x8(Z, Y, coarse_x_8x8x8[2]-2, 8, 8), \ + offset8x8x8(Z, Y, coarse_x_8x8x8[2]-1, 8, 8), \ + offset8x8x8(Z, Y, coarse_x_8x8x8[2], 8, 8), \ + offset8x8x8(Z, Y, coarse_x_8x8x8[2]+1, 8, 8), \ + offset8x8x8(Z, Y, coarse_x_8x8x8[2]+2, 8, 8), \ + offset8x8x8(Z, Y, 2, 5, 8), \ + 2 \ + }, \ + { \ + offset8x8x8(Z, Y, coarse_x_8x8x8[3]-2, 8, 8), \ + offset8x8x8(Z, Y, coarse_x_8x8x8[3]-1, 8, 8), \ + offset8x8x8(Z, Y, coarse_x_8x8x8[3], 8, 8), \ + zero_offset, \ + offset8x8x8(Z, Y, coarse_x_8x8x8[3]+1, 8, 8), \ + offset8x8x8(Z, Y, 3, 5, 8), \ + 3 \ + }, \ + { \ + offset8x8x8(Z, Y, coarse_x_8x8x8[4]-1, 8, 8), \ + zero_offset, \ + offset8x8x8(Z, Y, coarse_x_8x8x8[4], 8, 8), \ + zero_offset, \ + zero_offset, \ + offset8x8x8(Z, Y, 4, 5, 8), \ + 4 \ + } + + static constexpr int offset[320][7] = { + OFFSET1(0, 0), OFFSET2(0, 1), OFFSET1(0, 2), OFFSET2(0, 3), OFFSET1(0, 4), OFFSET2(0, 5), OFFSET1(0, 6), OFFSET1(0, 7), + OFFSET2(1, 0), OFFSET2(1, 1), OFFSET2(1, 2), OFFSET2(1, 3), OFFSET2(1, 4), OFFSET2(1, 5), OFFSET2(1, 6), OFFSET2(1, 7), + OFFSET1(2, 0), OFFSET2(2, 1), OFFSET1(2, 2), OFFSET2(2, 3), OFFSET1(2, 4), OFFSET2(2, 5), OFFSET1(2, 6), OFFSET1(2, 7), + OFFSET2(3, 0), OFFSET2(3, 1), OFFSET2(3, 2), OFFSET2(3, 3), OFFSET2(3, 4), OFFSET2(3, 5), OFFSET2(3, 6), OFFSET2(3, 7), + OFFSET1(4, 0), OFFSET2(4, 1), OFFSET1(4, 2), OFFSET2(4, 3), OFFSET1(4, 4), OFFSET2(4, 5), OFFSET1(4, 6), OFFSET1(4, 7), + OFFSET2(5, 0), OFFSET2(5, 1), OFFSET2(5, 2), OFFSET2(5, 3), OFFSET2(5, 4), OFFSET2(5, 5), OFFSET2(5, 6), OFFSET2(5, 7), + OFFSET1(6, 0), OFFSET2(6, 1), OFFSET1(6, 2), OFFSET2(6, 3), OFFSET1(6, 4), OFFSET2(6, 5), OFFSET1(6, 6), OFFSET1(6, 7), + OFFSET1(7, 0), OFFSET2(7, 1), OFFSET1(7, 2), OFFSET2(7, 3), OFFSET1(7, 4), OFFSET2(7, 5), OFFSET1(7, 6), OFFSET1(7, 7) + }; + #undef OFFSET1 + #undef OFFSET2 + return offset[i]; +} + + +MGARDX_EXEC int const *MassTrans_Y_Offset_8x8x8(SIZE i) { + static constexpr int zero_offset = 8*8*5 + 8*5*5 + 5*5*5; + #define OFFSET(Z, X) \ + { \ + zero_offset, \ + zero_offset, \ + offset8x8x8(Z, coarse_y_8x8x8[0], X, 5, 8), \ + offset8x8x8(Z, coarse_y_8x8x8[0]+1, X, 5, 8), \ + offset8x8x8(Z, coarse_y_8x8x8[0]+2, X, 5, 8), \ + offset8x8x8(Z, 0, X, 5, 5), \ + 0 \ + }, \ + { \ + offset8x8x8(Z, coarse_y_8x8x8[1]-2, X, 5, 8), \ + offset8x8x8(Z, coarse_y_8x8x8[1]-1, X, 5, 8), \ + offset8x8x8(Z, coarse_y_8x8x8[1], X, 5, 8), \ + offset8x8x8(Z, coarse_y_8x8x8[1]+1, X, 5, 8), \ + offset8x8x8(Z, coarse_y_8x8x8[1]+2, X, 5, 8), \ + offset8x8x8(Z, 1, X, 5, 5), \ + 1 \ + }, \ + { \ + offset8x8x8(Z, coarse_y_8x8x8[2]-2, X, 5, 8), \ + offset8x8x8(Z, coarse_y_8x8x8[2]-1, X, 5, 8), \ + offset8x8x8(Z, coarse_y_8x8x8[2], X, 5, 8), \ + offset8x8x8(Z, coarse_y_8x8x8[2]+1, X, 5, 8), \ + offset8x8x8(Z, coarse_y_8x8x8[2]+2, X, 5, 8), \ + offset8x8x8(Z, 2, X, 5, 5), \ + 2 \ + }, \ + { \ + offset8x8x8(Z, coarse_y_8x8x8[3]-2, X, 5, 8), \ + offset8x8x8(Z, coarse_y_8x8x8[3]-1, X, 5, 8), \ + offset8x8x8(Z, coarse_y_8x8x8[3], X, 5, 8), \ + zero_offset, \ + offset8x8x8(Z, coarse_y_8x8x8[3]+1, X, 5, 8), \ + offset8x8x8(Z, 3, X, 5, 5), \ + 3 \ + }, \ + { \ + offset8x8x8(Z, coarse_y_8x8x8[4]-1, X, 5, 8), \ + zero_offset, \ + offset8x8x8(Z, coarse_y_8x8x8[4], X, 5, 8), \ + zero_offset, \ + zero_offset, \ + offset8x8x8(Z, 4, X, 5, 5), \ + 4 \ + } + + static constexpr int offset[200][7] = { + OFFSET(0, 0), OFFSET(0, 1), OFFSET(0, 2), OFFSET(0, 3), OFFSET(0, 4), + OFFSET(1, 0), OFFSET(1, 1), OFFSET(1, 2), OFFSET(1, 3), OFFSET(1, 4), + OFFSET(2, 0), OFFSET(2, 1), OFFSET(2, 2), OFFSET(2, 3), OFFSET(2, 4), + OFFSET(3, 0), OFFSET(3, 1), OFFSET(3, 2), OFFSET(3, 3), OFFSET(3, 4), + OFFSET(4, 0), OFFSET(4, 1), OFFSET(4, 2), OFFSET(4, 3), OFFSET(4, 4), + OFFSET(5, 0), OFFSET(5, 1), OFFSET(5, 2), OFFSET(5, 3), OFFSET(5, 4), + OFFSET(6, 0), OFFSET(6, 1), OFFSET(6, 2), OFFSET(6, 3), OFFSET(6, 4), + OFFSET(7, 0), OFFSET(7, 1), OFFSET(7, 2), OFFSET(7, 3), OFFSET(7, 4) + }; + #undef OFFSET + return offset[i]; +} + +MGARDX_EXEC int const *MassTrans_Z_Offset_8x8x8(SIZE i) { + static constexpr int zero_offset = 8*5*5 + 5*5*5; + #define OFFSET(Y, X) \ + { \ + zero_offset, \ + zero_offset, \ + offset8x8x8(coarse_z_8x8x8[0], Y, X, 5, 5), \ + offset8x8x8(coarse_z_8x8x8[0]+1, Y, X, 5, 5), \ + offset8x8x8(coarse_z_8x8x8[0]+2, Y, X, 5, 5), \ + offset8x8x8(0, Y, X, 5, 5), \ + 0 \ + }, \ + { \ + offset8x8x8(coarse_z_8x8x8[1]-2, Y, X, 5, 5), \ + offset8x8x8(coarse_z_8x8x8[1]-1, Y, X, 5, 5), \ + offset8x8x8(coarse_z_8x8x8[1], Y, X, 5, 5), \ + offset8x8x8(coarse_z_8x8x8[1]+1, Y, X, 5, 5), \ + offset8x8x8(coarse_z_8x8x8[1]+2, Y, X, 5, 5), \ + offset8x8x8(1, Y, X, 5, 5), \ + 1 \ + }, \ + { \ + offset8x8x8(coarse_z_8x8x8[2]-2, Y, X, 5, 5), \ + offset8x8x8(coarse_z_8x8x8[2]-1, Y, X, 5, 5), \ + offset8x8x8(coarse_z_8x8x8[2], Y, X, 5, 5), \ + offset8x8x8(coarse_z_8x8x8[2]+1, Y, X, 5, 5), \ + offset8x8x8(coarse_z_8x8x8[2]+2, Y, X, 5, 5), \ + offset8x8x8(2 , Y, X, 5, 5), \ + 2 \ + }, \ + { \ + offset8x8x8(coarse_z_8x8x8[3]-2, Y, X, 5, 5), \ + offset8x8x8(coarse_z_8x8x8[3]-1, Y, X, 5, 5), \ + offset8x8x8(coarse_z_8x8x8[3], Y, X, 5, 5), \ + zero_offset, \ + offset8x8x8(coarse_z_8x8x8[3]+1, Y, X, 5, 5), \ + offset8x8x8(3, Y, X, 5, 5), \ + 3 \ + }, \ + { \ + offset8x8x8(coarse_z_8x8x8[4]-1, Y, X, 5, 5), \ + zero_offset, \ + offset8x8x8(coarse_z_8x8x8[4], Y, X, 5, 5), \ + zero_offset, \ + zero_offset, \ + offset8x8x8(4, Y, X, 5, 5), \ + 4 \ + } + + static constexpr int offset[125][7] = { + OFFSET(0, 0), OFFSET(0, 1), OFFSET(0, 2), OFFSET(0, 3), OFFSET(0, 4), + OFFSET(1, 0), OFFSET(1, 1), OFFSET(1, 2), OFFSET(1, 3), OFFSET(1, 4), + OFFSET(2, 0), OFFSET(2, 1), OFFSET(2, 2), OFFSET(2, 3), OFFSET(2, 4), + OFFSET(3, 0), OFFSET(3, 1), OFFSET(3, 2), OFFSET(3, 3), OFFSET(3, 4), + OFFSET(4, 0), OFFSET(4, 1), OFFSET(4, 2), OFFSET(4, 3), OFFSET(4, 4) + }; + #undef OFFSET + return offset[i]; +} + +template +constexpr T am_8x8x8(int index) { + T h_dist[4] = {2.0, 2.0, 2.0, 1.0}; + T am[6] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + T bm[6] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + bm[0] = 2 * h_dist[0] / 6; + am[0] = 0.0; + SIZE dof = 5; + for (int i = 1; i < dof - 1; i++) { + T a_j = h_dist[i - 1] / 6; + T w = a_j / bm[i - 1]; + bm[i] = 2 * (h_dist[i - 1] + h_dist[i]) / 6 - w * a_j; + am[i] = a_j; + } + T a_j = h_dist[dof - 2] / 6; + T w = a_j / bm[dof - 2]; + bm[dof - 1] = 2 * h_dist[dof - 2] / 6 - w * a_j; + am[dof - 1] = a_j; + + am[dof] = 0.0; + for (int i = dof; i >= 1; i--) { + bm[i] = bm[i-1]; + } + bm[0] = 1.0; + return am[index] * -1; +} + +template +constexpr T bm_8x8x8(int index) { + T h_dist[4] = {2.0, 2.0, 2.0, 1.0}; + T am[6] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + T bm[6] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + bm[0] = 2 * h_dist[0] / 6; + am[0] = 0.0; + SIZE dof = 5; + for (int i = 1; i < dof - 1; i++) { + T a_j = h_dist[i - 1] / 6; + T w = a_j / bm[i - 1]; + bm[i] = 2 * (h_dist[i - 1] + h_dist[i]) / 6 - w * a_j; + am[i] = a_j; + } + T a_j = h_dist[dof - 2] / 6; + T w = a_j / bm[dof - 2]; + bm[dof - 1] = 2 * h_dist[dof - 2] / 6 - w * a_j; + am[dof - 1] = a_j; + + am[dof] = 0.0; + for (int i = dof; i >= 1; i--) { + bm[i] = bm[i-1]; + } + bm[0] = 1.0; + return 1.0 / bm[index]; +} + +template +constexpr T amxbm_8x8x8(int index) { + return am_8x8x8(index) * bm_8x8x8(index); +} + + +MGARDX_EXEC int const *TriDiag_X_Offset_8x8x8(SIZE i) { + static constexpr int offset[25][5] = { + {offset8x8x8(0, 0, 0, 5, 5), offset8x8x8(0, 0, 1, 5, 5), offset8x8x8(0, 0, 2, 5, 5), offset8x8x8(0, 0, 3, 5, 5), offset8x8x8(0, 0, 4, 5, 5)}, + {offset8x8x8(0, 1, 0, 5, 5), offset8x8x8(0, 1, 1, 5, 5), offset8x8x8(0, 1, 2, 5, 5), offset8x8x8(0, 1, 3, 5, 5), offset8x8x8(0, 1, 4, 5, 5)}, + {offset8x8x8(0, 2, 0, 5, 5), offset8x8x8(0, 2, 1, 5, 5), offset8x8x8(0, 2, 2, 5, 5), offset8x8x8(0, 2, 3, 5, 5), offset8x8x8(0, 2, 4, 5, 5)}, + {offset8x8x8(0, 3, 0, 5, 5), offset8x8x8(0, 3, 1, 5, 5), offset8x8x8(0, 3, 2, 5, 5), offset8x8x8(0, 3, 3, 5, 5), offset8x8x8(0, 3, 4, 5, 5)}, + {offset8x8x8(0, 4, 0, 5, 5), offset8x8x8(0, 4, 1, 5, 5), offset8x8x8(0, 4, 2, 5, 5), offset8x8x8(0, 4, 3, 5, 5), offset8x8x8(0, 4, 4, 5, 5)}, + + {offset8x8x8(1, 0, 0, 5, 5), offset8x8x8(1, 0, 1, 5, 5), offset8x8x8(1, 0, 2, 5, 5), offset8x8x8(1, 0, 3, 5, 5), offset8x8x8(1, 0, 4, 5, 5)}, + {offset8x8x8(1, 1, 0, 5, 5), offset8x8x8(1, 1, 1, 5, 5), offset8x8x8(1, 1, 2, 5, 5), offset8x8x8(1, 1, 3, 5, 5), offset8x8x8(1, 1, 4, 5, 5)}, + {offset8x8x8(1, 2, 0, 5, 5), offset8x8x8(1, 2, 1, 5, 5), offset8x8x8(1, 2, 2, 5, 5), offset8x8x8(1, 2, 3, 5, 5), offset8x8x8(1, 2, 4, 5, 5)}, + {offset8x8x8(1, 3, 0, 5, 5), offset8x8x8(1, 3, 1, 5, 5), offset8x8x8(1, 3, 2, 5, 5), offset8x8x8(1, 3, 3, 5, 5), offset8x8x8(1, 3, 4, 5, 5)}, + {offset8x8x8(1, 4, 0, 5, 5), offset8x8x8(1, 4, 1, 5, 5), offset8x8x8(1, 4, 2, 5, 5), offset8x8x8(1, 4, 3, 5, 5), offset8x8x8(1, 4, 4, 5, 5)}, + + {offset8x8x8(2, 0, 0, 5, 5), offset8x8x8(2, 0, 1, 5, 5), offset8x8x8(2, 0, 2, 5, 5), offset8x8x8(2, 0, 3, 5, 5), offset8x8x8(2, 0, 4, 5, 5)}, + {offset8x8x8(2, 1, 0, 5, 5), offset8x8x8(2, 1, 1, 5, 5), offset8x8x8(2, 1, 2, 5, 5), offset8x8x8(2, 1, 3, 5, 5), offset8x8x8(2, 1, 4, 5, 5)}, + {offset8x8x8(2, 2, 0, 5, 5), offset8x8x8(2, 2, 1, 5, 5), offset8x8x8(2, 2, 2, 5, 5), offset8x8x8(2, 2, 3, 5, 5), offset8x8x8(2, 2, 4, 5, 5)}, + {offset8x8x8(2, 3, 0, 5, 5), offset8x8x8(2, 3, 1, 5, 5), offset8x8x8(2, 3, 2, 5, 5), offset8x8x8(2, 3, 3, 5, 5), offset8x8x8(2, 3, 4, 5, 5)}, + {offset8x8x8(2, 4, 0, 5, 5), offset8x8x8(2, 4, 1, 5, 5), offset8x8x8(2, 4, 2, 5, 5), offset8x8x8(2, 4, 3, 5, 5), offset8x8x8(2, 4, 4, 5, 5)}, + + {offset8x8x8(3, 0, 0, 5, 5), offset8x8x8(3, 0, 1, 5, 5), offset8x8x8(3, 0, 2, 5, 5), offset8x8x8(3, 0, 3, 5, 5), offset8x8x8(3, 0, 4, 5, 5)}, + {offset8x8x8(3, 1, 0, 5, 5), offset8x8x8(3, 1, 1, 5, 5), offset8x8x8(3, 1, 2, 5, 5), offset8x8x8(3, 1, 3, 5, 5), offset8x8x8(3, 1, 4, 5, 5)}, + {offset8x8x8(3, 2, 0, 5, 5), offset8x8x8(3, 2, 1, 5, 5), offset8x8x8(3, 2, 2, 5, 5), offset8x8x8(3, 2, 3, 5, 5), offset8x8x8(3, 2, 4, 5, 5)}, + {offset8x8x8(3, 3, 0, 5, 5), offset8x8x8(3, 3, 1, 5, 5), offset8x8x8(3, 3, 2, 5, 5), offset8x8x8(3, 3, 3, 5, 5), offset8x8x8(3, 3, 4, 5, 5)}, + {offset8x8x8(3, 4, 0, 5, 5), offset8x8x8(3, 4, 1, 5, 5), offset8x8x8(3, 4, 2, 5, 5), offset8x8x8(3, 4, 3, 5, 5), offset8x8x8(3, 4, 4, 5, 5)}, + + {offset8x8x8(4, 0, 0, 5, 5), offset8x8x8(4, 0, 1, 5, 5), offset8x8x8(4, 0, 2, 5, 5), offset8x8x8(4, 0, 3, 5, 5), offset8x8x8(4, 0, 4, 5, 5)}, + {offset8x8x8(4, 1, 0, 5, 5), offset8x8x8(4, 1, 1, 5, 5), offset8x8x8(4, 1, 2, 5, 5), offset8x8x8(4, 1, 3, 5, 5), offset8x8x8(4, 1, 4, 5, 5)}, + {offset8x8x8(4, 2, 0, 5, 5), offset8x8x8(4, 2, 1, 5, 5), offset8x8x8(4, 2, 2, 5, 5), offset8x8x8(4, 2, 3, 5, 5), offset8x8x8(4, 2, 4, 5, 5)}, + {offset8x8x8(4, 3, 0, 5, 5), offset8x8x8(4, 3, 1, 5, 5), offset8x8x8(4, 3, 2, 5, 5), offset8x8x8(4, 3, 3, 5, 5), offset8x8x8(4, 3, 4, 5, 5)}, + {offset8x8x8(4, 4, 0, 5, 5), offset8x8x8(4, 4, 1, 5, 5), offset8x8x8(4, 4, 2, 5, 5), offset8x8x8(4, 4, 3, 5, 5), offset8x8x8(4, 4, 4, 5, 5)} + }; + return offset[i]; +} + +MGARDX_EXEC int const *TriDiag_Y_Offset_8x8x8(SIZE i) { + static constexpr int offset[25][5] = { + {offset8x8x8(0, 0, 0, 5, 5), offset8x8x8(0, 1, 0, 5, 5), offset8x8x8(0, 2, 0, 5, 5), offset8x8x8(0, 3, 0, 5, 5), offset8x8x8(0, 4, 0, 5, 5)}, + {offset8x8x8(0, 0, 1, 5, 5), offset8x8x8(0, 1, 1, 5, 5), offset8x8x8(0, 2, 1, 5, 5), offset8x8x8(0, 3, 1, 5, 5), offset8x8x8(0, 4, 1, 5, 5)}, + {offset8x8x8(0, 0, 2, 5, 5), offset8x8x8(0, 1, 2, 5, 5), offset8x8x8(0, 2, 2, 5, 5), offset8x8x8(0, 3, 2, 5, 5), offset8x8x8(0, 4, 2, 5, 5)}, + {offset8x8x8(0, 0, 3, 5, 5), offset8x8x8(0, 1, 3, 5, 5), offset8x8x8(0, 2, 3, 5, 5), offset8x8x8(0, 3, 3, 5, 5), offset8x8x8(0, 4, 3, 5, 5)}, + {offset8x8x8(0, 0, 4, 5, 5), offset8x8x8(0, 1, 4, 5, 5), offset8x8x8(0, 2, 4, 5, 5), offset8x8x8(0, 3, 4, 5, 5), offset8x8x8(0, 4, 4, 5, 5)}, + + {offset8x8x8(1, 0, 0, 5, 5), offset8x8x8(1, 1, 0, 5, 5), offset8x8x8(1, 2, 0, 5, 5), offset8x8x8(1, 3, 0, 5, 5), offset8x8x8(1, 4, 0, 5, 5)}, + {offset8x8x8(1, 0, 1, 5, 5), offset8x8x8(1, 1, 1, 5, 5), offset8x8x8(1, 2, 1, 5, 5), offset8x8x8(1, 3, 1, 5, 5), offset8x8x8(1, 4, 1, 5, 5)}, + {offset8x8x8(1, 0, 2, 5, 5), offset8x8x8(1, 1, 2, 5, 5), offset8x8x8(1, 2, 2, 5, 5), offset8x8x8(1, 3, 2, 5, 5), offset8x8x8(1, 4, 2, 5, 5)}, + {offset8x8x8(1, 0, 3, 5, 5), offset8x8x8(1, 1, 3, 5, 5), offset8x8x8(1, 2, 3, 5, 5), offset8x8x8(1, 3, 3, 5, 5), offset8x8x8(1, 4, 3, 5, 5)}, + {offset8x8x8(1, 0, 4, 5, 5), offset8x8x8(1, 1, 4, 5, 5), offset8x8x8(1, 2, 4, 5, 5), offset8x8x8(1, 3, 4, 5, 5), offset8x8x8(1, 4, 4, 5, 5)}, + + {offset8x8x8(2, 0, 0, 5, 5), offset8x8x8(2, 1, 0, 5, 5), offset8x8x8(2, 2, 0, 5, 5), offset8x8x8(2, 3, 0, 5, 5), offset8x8x8(2, 4, 0, 5, 5)}, + {offset8x8x8(2, 0, 1, 5, 5), offset8x8x8(2, 1, 1, 5, 5), offset8x8x8(2, 2, 1, 5, 5), offset8x8x8(2, 3, 1, 5, 5), offset8x8x8(2, 4, 1, 5, 5)}, + {offset8x8x8(2, 0, 2, 5, 5), offset8x8x8(2, 1, 2, 5, 5), offset8x8x8(2, 2, 2, 5, 5), offset8x8x8(2, 3, 2, 5, 5), offset8x8x8(2, 4, 2, 5, 5)}, + {offset8x8x8(2, 0, 3, 5, 5), offset8x8x8(2, 1, 3, 5, 5), offset8x8x8(2, 2, 3, 5, 5), offset8x8x8(2, 3, 3, 5, 5), offset8x8x8(2, 4, 3, 5, 5)}, + {offset8x8x8(2, 0, 4, 5, 5), offset8x8x8(2, 1, 4, 5, 5), offset8x8x8(2, 2, 4, 5, 5), offset8x8x8(2, 3, 4, 5, 5), offset8x8x8(2, 4, 4, 5, 5)}, + + {offset8x8x8(3, 0, 0, 5, 5), offset8x8x8(3, 1, 0, 5, 5), offset8x8x8(3, 2, 0, 5, 5), offset8x8x8(3, 3, 0, 5, 5), offset8x8x8(3, 4, 0, 5, 5)}, + {offset8x8x8(3, 0, 1, 5, 5), offset8x8x8(3, 1, 1, 5, 5), offset8x8x8(3, 2, 1, 5, 5), offset8x8x8(3, 3, 1, 5, 5), offset8x8x8(3, 4, 1, 5, 5)}, + {offset8x8x8(3, 0, 2, 5, 5), offset8x8x8(3, 1, 2, 5, 5), offset8x8x8(3, 2, 2, 5, 5), offset8x8x8(3, 3, 2, 5, 5), offset8x8x8(3, 4, 2, 5, 5)}, + {offset8x8x8(3, 0, 3, 5, 5), offset8x8x8(3, 1, 3, 5, 5), offset8x8x8(3, 2, 3, 5, 5), offset8x8x8(3, 3, 3, 5, 5), offset8x8x8(3, 4, 3, 5, 5)}, + {offset8x8x8(3, 0, 4, 5, 5), offset8x8x8(3, 1, 4, 5, 5), offset8x8x8(3, 2, 4, 5, 5), offset8x8x8(3, 3, 4, 5, 5), offset8x8x8(3, 4, 4, 5, 5)}, + + {offset8x8x8(4, 0, 0, 5, 5), offset8x8x8(4, 1, 0, 5, 5), offset8x8x8(4, 2, 0, 5, 5), offset8x8x8(4, 3, 0, 5, 5), offset8x8x8(4, 4, 0, 5, 5)}, + {offset8x8x8(4, 0, 1, 5, 5), offset8x8x8(4, 1, 1, 5, 5), offset8x8x8(4, 2, 1, 5, 5), offset8x8x8(4, 3, 1, 5, 5), offset8x8x8(4, 4, 1, 5, 5)}, + {offset8x8x8(4, 0, 2, 5, 5), offset8x8x8(4, 1, 2, 5, 5), offset8x8x8(4, 2, 2, 5, 5), offset8x8x8(4, 3, 2, 5, 5), offset8x8x8(4, 4, 2, 5, 5)}, + {offset8x8x8(4, 0, 3, 5, 5), offset8x8x8(4, 1, 3, 5, 5), offset8x8x8(4, 2, 3, 5, 5), offset8x8x8(4, 3, 3, 5, 5), offset8x8x8(4, 4, 3, 5, 5)}, + {offset8x8x8(4, 0, 4, 5, 5), offset8x8x8(4, 1, 4, 5, 5), offset8x8x8(4, 2, 4, 5, 5), offset8x8x8(4, 3, 4, 5, 5), offset8x8x8(4, 4, 4, 5, 5)} + }; + return offset[i]; +} + +MGARDX_EXEC int const *TriDiag_Z_Offset_8x8x8(SIZE i) { + static constexpr int offset[25][5] = { + {offset8x8x8(0, 0, 0, 5, 5), offset8x8x8(1, 0, 0, 5, 5), offset8x8x8(2, 0, 0, 5, 5), offset8x8x8(3, 0, 0, 5, 5), offset8x8x8(4, 0, 0, 5, 5)}, + {offset8x8x8(0, 0, 1, 5, 5), offset8x8x8(1, 0, 1, 5, 5), offset8x8x8(2, 0, 1, 5, 5), offset8x8x8(3, 0, 1, 5, 5), offset8x8x8(4, 0, 1, 5, 5)}, + {offset8x8x8(0, 0, 2, 5, 5), offset8x8x8(1, 0, 2, 5, 5), offset8x8x8(2, 0, 2, 5, 5), offset8x8x8(3, 0, 2, 5, 5), offset8x8x8(4, 0, 2, 5, 5)}, + {offset8x8x8(0, 0, 3, 5, 5), offset8x8x8(1, 0, 3, 5, 5), offset8x8x8(2, 0, 3, 5, 5), offset8x8x8(3, 0, 3, 5, 5), offset8x8x8(4, 0, 3, 5, 5)}, + {offset8x8x8(0, 0, 4, 5, 5), offset8x8x8(1, 0, 4, 5, 5), offset8x8x8(2, 0, 4, 5, 5), offset8x8x8(3, 0, 4, 5, 5), offset8x8x8(4, 0, 4, 5, 5)}, + + {offset8x8x8(0, 1, 0, 5, 5), offset8x8x8(1, 1, 0, 5, 5), offset8x8x8(2, 1, 0, 5, 5), offset8x8x8(3, 1, 0, 5, 5), offset8x8x8(4, 1, 0, 5, 5)}, + {offset8x8x8(0, 1, 1, 5, 5), offset8x8x8(1, 1, 1, 5, 5), offset8x8x8(2, 1, 1, 5, 5), offset8x8x8(3, 1, 1, 5, 5), offset8x8x8(4, 1, 1, 5, 5)}, + {offset8x8x8(0, 1, 2, 5, 5), offset8x8x8(1, 1, 2, 5, 5), offset8x8x8(2, 1, 2, 5, 5), offset8x8x8(3, 1, 2, 5, 5), offset8x8x8(4, 1, 2, 5, 5)}, + {offset8x8x8(0, 1, 3, 5, 5), offset8x8x8(1, 1, 3, 5, 5), offset8x8x8(2, 1, 3, 5, 5), offset8x8x8(3, 1, 3, 5, 5), offset8x8x8(4, 1, 3, 5, 5)}, + {offset8x8x8(0, 1, 4, 5, 5), offset8x8x8(1, 1, 4, 5, 5), offset8x8x8(2, 1, 4, 5, 5), offset8x8x8(3, 1, 4, 5, 5), offset8x8x8(4, 1, 4, 5, 5)}, + + {offset8x8x8(0, 2, 0, 5, 5), offset8x8x8(1, 2, 0, 5, 5), offset8x8x8(2, 2, 0, 5, 5), offset8x8x8(3, 2, 0, 5, 5), offset8x8x8(4, 2, 0, 5, 5)}, + {offset8x8x8(0, 2, 1, 5, 5), offset8x8x8(1, 2, 1, 5, 5), offset8x8x8(2, 2, 1, 5, 5), offset8x8x8(3, 2, 1, 5, 5), offset8x8x8(4, 2, 1, 5, 5)}, + {offset8x8x8(0, 2, 2, 5, 5), offset8x8x8(1, 2, 2, 5, 5), offset8x8x8(2, 2, 2, 5, 5), offset8x8x8(3, 2, 2, 5, 5), offset8x8x8(4, 2, 2, 5, 5)}, + {offset8x8x8(0, 2, 3, 5, 5), offset8x8x8(1, 2, 3, 5, 5), offset8x8x8(2, 2, 3, 5, 5), offset8x8x8(3, 2, 3, 5, 5), offset8x8x8(4, 2, 3, 5, 5)}, + {offset8x8x8(0, 2, 4, 5, 5), offset8x8x8(1, 2, 4, 5, 5), offset8x8x8(2, 2, 4, 5, 5), offset8x8x8(3, 2, 4, 5, 5), offset8x8x8(4, 2, 4, 5, 5)}, + + {offset8x8x8(0, 3, 0, 5, 5), offset8x8x8(1, 3, 0, 5, 5), offset8x8x8(2, 3, 0, 5, 5), offset8x8x8(3, 3, 0, 5, 5), offset8x8x8(4, 3, 0, 5, 5)}, + {offset8x8x8(0, 3, 1, 5, 5), offset8x8x8(1, 3, 1, 5, 5), offset8x8x8(2, 3, 1, 5, 5), offset8x8x8(3, 3, 1, 5, 5), offset8x8x8(4, 3, 1, 5, 5)}, + {offset8x8x8(0, 3, 2, 5, 5), offset8x8x8(1, 3, 2, 5, 5), offset8x8x8(2, 3, 2, 5, 5), offset8x8x8(3, 3, 2, 5, 5), offset8x8x8(4, 3, 2, 5, 5)}, + {offset8x8x8(0, 3, 3, 5, 5), offset8x8x8(1, 3, 3, 5, 5), offset8x8x8(2, 3, 3, 5, 5), offset8x8x8(3, 3, 3, 5, 5), offset8x8x8(4, 3, 3, 5, 5)}, + {offset8x8x8(0, 3, 4, 5, 5), offset8x8x8(1, 3, 4, 5, 5), offset8x8x8(2, 3, 4, 5, 5), offset8x8x8(3, 3, 4, 5, 5), offset8x8x8(4, 3, 4, 5, 5)}, + + {offset8x8x8(0, 4, 0, 5, 5), offset8x8x8(1, 4, 0, 5, 5), offset8x8x8(2, 4, 0, 5, 5), offset8x8x8(3, 4, 0, 5, 5), offset8x8x8(4, 4, 0, 5, 5)}, + {offset8x8x8(0, 4, 1, 5, 5), offset8x8x8(1, 4, 1, 5, 5), offset8x8x8(2, 4, 1, 5, 5), offset8x8x8(3, 4, 1, 5, 5), offset8x8x8(4, 4, 1, 5, 5)}, + {offset8x8x8(0, 4, 2, 5, 5), offset8x8x8(1, 4, 2, 5, 5), offset8x8x8(2, 4, 2, 5, 5), offset8x8x8(3, 4, 2, 5, 5), offset8x8x8(4, 4, 2, 5, 5)}, + {offset8x8x8(0, 4, 3, 5, 5), offset8x8x8(1, 4, 3, 5, 5), offset8x8x8(2, 4, 3, 5, 5), offset8x8x8(3, 4, 3, 5, 5), offset8x8x8(4, 4, 3, 5, 5)}, + {offset8x8x8(0, 4, 4, 5, 5), offset8x8x8(1, 4, 4, 5, 5), offset8x8x8(2, 4, 4, 5, 5), offset8x8x8(3, 4, 4, 5, 5), offset8x8x8(4, 4, 4, 5, 5)} + }; + return offset[i]; +} + + +MGARDX_EXEC int Coarse_Offset_8x8x8(SIZE i) { + static constexpr int offset[125] = { + offset8x8x8(0, 0, 0, 8, 8), offset8x8x8(0, 0, 2, 8, 8), offset8x8x8(0, 0, 4, 8, 8), offset8x8x8(0, 0, 6, 8, 8), offset8x8x8(0, 0, 7, 8, 8), + offset8x8x8(0, 2, 0, 8, 8), offset8x8x8(0, 2, 2, 8, 8), offset8x8x8(0, 2, 4, 8, 8), offset8x8x8(0, 2, 6, 8, 8), offset8x8x8(0, 2, 7, 8, 8), + offset8x8x8(0, 4, 0, 8, 8), offset8x8x8(0, 4, 2, 8, 8), offset8x8x8(0, 4, 4, 8, 8), offset8x8x8(0, 4, 6, 8, 8), offset8x8x8(0, 4, 7, 8, 8), + offset8x8x8(0, 6, 0, 8, 8), offset8x8x8(0, 6, 2, 8, 8), offset8x8x8(0, 6, 4, 8, 8), offset8x8x8(0, 6, 6, 8, 8), offset8x8x8(0, 6, 7, 8, 8), + offset8x8x8(0, 7, 0, 8, 8), offset8x8x8(0, 7, 2, 8, 8), offset8x8x8(0, 7, 4, 8, 8), offset8x8x8(0, 7, 6, 8, 8), offset8x8x8(0, 7, 7, 8, 8), + + offset8x8x8(2, 0, 0, 8, 8), offset8x8x8(2, 0, 2, 8, 8), offset8x8x8(2, 0, 4, 8, 8), offset8x8x8(2, 0, 6, 8, 8), offset8x8x8(2, 0, 7, 8, 8), + offset8x8x8(2, 2, 0, 8, 8), offset8x8x8(2, 2, 2, 8, 8), offset8x8x8(2, 2, 4, 8, 8), offset8x8x8(2, 2, 6, 8, 8), offset8x8x8(2, 2, 7, 8, 8), + offset8x8x8(2, 4, 0, 8, 8), offset8x8x8(2, 4, 2, 8, 8), offset8x8x8(2, 4, 4, 8, 8), offset8x8x8(2, 4, 6, 8, 8), offset8x8x8(2, 4, 7, 8, 8), + offset8x8x8(2, 6, 0, 8, 8), offset8x8x8(2, 6, 2, 8, 8), offset8x8x8(2, 6, 4, 8, 8), offset8x8x8(2, 6, 6, 8, 8), offset8x8x8(2, 6, 7, 8, 8), + offset8x8x8(2, 7, 0, 8, 8), offset8x8x8(2, 7, 2, 8, 8), offset8x8x8(2, 7, 4, 8, 8), offset8x8x8(2, 7, 6, 8, 8), offset8x8x8(2, 7, 7, 8, 8), + + offset8x8x8(4, 0, 0, 8, 8), offset8x8x8(4, 0, 2, 8, 8), offset8x8x8(4, 0, 4, 8, 8), offset8x8x8(4, 0, 6, 8, 8), offset8x8x8(4, 0, 7, 8, 8), + offset8x8x8(4, 2, 0, 8, 8), offset8x8x8(4, 2, 2, 8, 8), offset8x8x8(4, 2, 4, 8, 8), offset8x8x8(4, 2, 6, 8, 8), offset8x8x8(4, 2, 7, 8, 8), + offset8x8x8(4, 4, 0, 8, 8), offset8x8x8(4, 4, 2, 8, 8), offset8x8x8(4, 4, 4, 8, 8), offset8x8x8(4, 4, 6, 8, 8), offset8x8x8(4, 4, 7, 8, 8), + offset8x8x8(4, 6, 0, 8, 8), offset8x8x8(4, 6, 2, 8, 8), offset8x8x8(4, 6, 4, 8, 8), offset8x8x8(4, 6, 6, 8, 8), offset8x8x8(4, 6, 7, 8, 8), + offset8x8x8(4, 7, 0, 8, 8), offset8x8x8(4, 7, 2, 8, 8), offset8x8x8(4, 7, 4, 8, 8), offset8x8x8(4, 7, 6, 8, 8), offset8x8x8(4, 7, 7, 8, 8), + + offset8x8x8(6, 0, 0, 8, 8), offset8x8x8(6, 0, 2, 8, 8), offset8x8x8(6, 0, 4, 8, 8), offset8x8x8(6, 0, 6, 8, 8), offset8x8x8(6, 0, 7, 8, 8), + offset8x8x8(6, 2, 0, 8, 8), offset8x8x8(6, 2, 2, 8, 8), offset8x8x8(6, 2, 4, 8, 8), offset8x8x8(6, 2, 6, 8, 8), offset8x8x8(6, 2, 7, 8, 8), + offset8x8x8(6, 4, 0, 8, 8), offset8x8x8(6, 4, 2, 8, 8), offset8x8x8(6, 4, 4, 8, 8), offset8x8x8(6, 4, 6, 8, 8), offset8x8x8(6, 4, 7, 8, 8), + offset8x8x8(6, 6, 0, 8, 8), offset8x8x8(6, 6, 2, 8, 8), offset8x8x8(6, 6, 4, 8, 8), offset8x8x8(6, 6, 6, 8, 8), offset8x8x8(6, 6, 7, 8, 8), + offset8x8x8(6, 7, 0, 8, 8), offset8x8x8(6, 7, 2, 8, 8), offset8x8x8(6, 7, 4, 8, 8), offset8x8x8(6, 7, 6, 8, 8), offset8x8x8(6, 7, 7, 8, 8), + + offset8x8x8(7, 0, 0, 8, 8), offset8x8x8(7, 0, 2, 8, 8), offset8x8x8(7, 0, 4, 8, 8), offset8x8x8(7, 0, 6, 8, 8), offset8x8x8(7, 0, 7, 8, 8), + offset8x8x8(7, 2, 0, 8, 8), offset8x8x8(7, 2, 2, 8, 8), offset8x8x8(7, 2, 4, 8, 8), offset8x8x8(7, 2, 6, 8, 8), offset8x8x8(7, 2, 7, 8, 8), + offset8x8x8(7, 4, 0, 8, 8), offset8x8x8(7, 4, 2, 8, 8), offset8x8x8(7, 4, 4, 8, 8), offset8x8x8(7, 4, 6, 8, 8), offset8x8x8(7, 4, 7, 8, 8), + offset8x8x8(7, 6, 0, 8, 8), offset8x8x8(7, 6, 2, 8, 8), offset8x8x8(7, 6, 4, 8, 8), offset8x8x8(7, 6, 6, 8, 8), offset8x8x8(7, 6, 7, 8, 8), + offset8x8x8(7, 7, 0, 8, 8), offset8x8x8(7, 7, 2, 8, 8), offset8x8x8(7, 7, 4, 8, 8), offset8x8x8(7, 7, 6, 8, 8), offset8x8x8(7, 7, 7, 8, 8) + }; + return offset[i]; +} + +MGARDX_EXEC int const * Coarse_Reorder_8x8x8(SIZE i) { + static constexpr int offset[125][3] = { + {0, 0, 0}, {0, 0, 1}, {0, 0, 2}, {0, 0, 3}, {0, 0, 4}, + {0, 1, 0}, {0, 1, 1}, {0, 1, 2}, {0, 1, 3}, {0, 1, 4}, + {0, 2, 0}, {0, 2, 1}, {0, 2, 2}, {0, 2, 3}, {0, 2, 4}, + {0, 3, 0}, {0, 3, 1}, {0, 3, 2}, {0, 3, 3}, {0, 3, 4}, + {0, 4, 0}, {0, 4, 1}, {0, 4, 2}, {0, 4, 3}, {0, 4, 4}, + + {1, 0, 0}, {1, 0, 1}, {1, 0, 2}, {1, 0, 3}, {1, 0, 4}, + {1, 1, 0}, {1, 1, 1}, {1, 1, 2}, {1, 1, 3}, {1, 1, 4}, + {1, 2, 0}, {1, 2, 1}, {1, 2, 2}, {1, 2, 3}, {1, 2, 4}, + {1, 3, 0}, {1, 3, 1}, {1, 3, 2}, {1, 3, 3}, {1, 3, 4}, + {1, 4, 0}, {1, 4, 1}, {1, 4, 2}, {1, 4, 3}, {1, 4, 4}, + + {2, 0, 0}, {2, 0, 1}, {2, 0, 2}, {2, 0, 3}, {2, 0, 4}, + {2, 1, 0}, {2, 1, 1}, {2, 1, 2}, {2, 1, 3}, {2, 1, 4}, + {2, 2, 0}, {2, 2, 1}, {2, 2, 2}, {2, 2, 3}, {2, 2, 4}, + {2, 3, 0}, {2, 3, 1}, {2, 3, 2}, {2, 3, 3}, {2, 3, 4}, + {2, 4, 0}, {2, 4, 1}, {2, 4, 2}, {2, 4, 3}, {2, 4, 4}, + + {3, 0, 0}, {3, 0, 1}, {3, 0, 2}, {3, 0, 3}, {3, 0, 4}, + {3, 1, 0}, {3, 1, 1}, {3, 1, 2}, {3, 1, 3}, {3, 1, 4}, + {3, 2, 0}, {3, 2, 1}, {3, 2, 2}, {3, 2, 3}, {3, 2, 4}, + {3, 3, 0}, {3, 3, 1}, {3, 3, 2}, {3, 3, 3}, {3, 3, 4}, + {3, 4, 0}, {3, 4, 1}, {3, 4, 2}, {3, 4, 3}, {3, 4, 4}, + + {4, 0, 0}, {4, 0, 1}, {4, 0, 2}, {4, 0, 3}, {4, 0, 4}, + {4, 1, 0}, {4, 1, 1}, {4, 1, 2}, {4, 1, 3}, {4, 1, 4}, + {4, 2, 0}, {4, 2, 1}, {4, 2, 2}, {4, 2, 3}, {4, 2, 4}, + {4, 3, 0}, {4, 3, 1}, {4, 3, 2}, {4, 3, 3}, {4, 3, 4}, + {4, 4, 0}, {4, 4, 1}, {4, 4, 2}, {4, 4, 3}, {4, 4, 4} + }; + return offset[i]; +} + + +MGARDX_EXEC int Coeff_Offset_8x8x8(SIZE i) { + static constexpr int offset[387] = { + offset8x8x8(0, 0, 1, 8, 8), offset8x8x8(0, 0, 3, 8, 8), offset8x8x8(0, 0, 5, 8, 8), + offset8x8x8(0, 1, 0, 8, 8), offset8x8x8(0, 1, 1, 8, 8), offset8x8x8(0, 1, 2, 8, 8), offset8x8x8(0, 1, 3, 8, 8), offset8x8x8(0, 1, 4, 8, 8), offset8x8x8(0, 1, 5, 8, 8), offset8x8x8(0, 1, 6, 8, 8), offset8x8x8(0, 1, 7, 8, 8), + offset8x8x8(0, 2, 1, 8, 8), offset8x8x8(0, 2, 3, 8, 8), offset8x8x8(0, 2, 5, 8, 8), + offset8x8x8(0, 3, 0, 8, 8), offset8x8x8(0, 3, 1, 8, 8), offset8x8x8(0, 3, 2, 8, 8), offset8x8x8(0, 3, 3, 8, 8), offset8x8x8(0, 3, 4, 8, 8), offset8x8x8(0, 3, 5, 8, 8), offset8x8x8(0, 3, 6, 8, 8), offset8x8x8(0, 3, 7, 8, 8), + offset8x8x8(0, 4, 1, 8, 8), offset8x8x8(0, 4, 3, 8, 8), offset8x8x8(0, 4, 5, 8, 8), + offset8x8x8(0, 5, 0, 8, 8), offset8x8x8(0, 5, 1, 8, 8), offset8x8x8(0, 5, 2, 8, 8), offset8x8x8(0, 5, 3, 8, 8), offset8x8x8(0, 5, 4, 8, 8), offset8x8x8(0, 5, 5, 8, 8), offset8x8x8(0, 5, 6, 8, 8), offset8x8x8(0, 5, 7, 8, 8), + offset8x8x8(0, 6, 1, 8, 8), offset8x8x8(0, 6, 3, 8, 8), offset8x8x8(0, 6, 5, 8, 8), + offset8x8x8(0, 7, 1, 8, 8), offset8x8x8(0, 7, 3, 8, 8), offset8x8x8(0, 7, 5, 8, 8), + + offset8x8x8(1, 0, 0, 8, 8), offset8x8x8(1, 0, 1, 8, 8), offset8x8x8(1, 0, 2, 8, 8), offset8x8x8(1, 0, 3, 8, 8), offset8x8x8(1, 0, 4, 8, 8), offset8x8x8(1, 0, 5, 8, 8), offset8x8x8(1, 0, 6, 8, 8), offset8x8x8(1, 0, 7, 8, 8), + offset8x8x8(1, 1, 0, 8, 8), offset8x8x8(1, 1, 1, 8, 8), offset8x8x8(1, 1, 2, 8, 8), offset8x8x8(1, 1, 3, 8, 8), offset8x8x8(1, 1, 4, 8, 8), offset8x8x8(1, 1, 5, 8, 8), offset8x8x8(1, 1, 6, 8, 8), offset8x8x8(1, 1, 7, 8, 8), + offset8x8x8(1, 2, 0, 8, 8), offset8x8x8(1, 2, 1, 8, 8), offset8x8x8(1, 2, 2, 8, 8), offset8x8x8(1, 2, 3, 8, 8), offset8x8x8(1, 2, 4, 8, 8), offset8x8x8(1, 2, 5, 8, 8), offset8x8x8(1, 2, 6, 8, 8), offset8x8x8(1, 2, 7, 8, 8), + offset8x8x8(1, 3, 0, 8, 8), offset8x8x8(1, 3, 1, 8, 8), offset8x8x8(1, 3, 2, 8, 8), offset8x8x8(1, 3, 3, 8, 8), offset8x8x8(1, 3, 4, 8, 8), offset8x8x8(1, 3, 5, 8, 8), offset8x8x8(1, 3, 6, 8, 8), offset8x8x8(1, 3, 7, 8, 8), + offset8x8x8(1, 4, 0, 8, 8), offset8x8x8(1, 4, 1, 8, 8), offset8x8x8(1, 4, 2, 8, 8), offset8x8x8(1, 4, 3, 8, 8), offset8x8x8(1, 4, 4, 8, 8), offset8x8x8(1, 4, 5, 8, 8), offset8x8x8(1, 4, 6, 8, 8), offset8x8x8(1, 4, 7, 8, 8), + offset8x8x8(1, 5, 0, 8, 8), offset8x8x8(1, 5, 1, 8, 8), offset8x8x8(1, 5, 2, 8, 8), offset8x8x8(1, 5, 3, 8, 8), offset8x8x8(1, 5, 4, 8, 8), offset8x8x8(1, 5, 5, 8, 8), offset8x8x8(1, 5, 6, 8, 8), offset8x8x8(1, 5, 7, 8, 8), + offset8x8x8(1, 6, 0, 8, 8), offset8x8x8(1, 6, 1, 8, 8), offset8x8x8(1, 6, 2, 8, 8), offset8x8x8(1, 6, 3, 8, 8), offset8x8x8(1, 6, 4, 8, 8), offset8x8x8(1, 6, 5, 8, 8), offset8x8x8(1, 6, 6, 8, 8), offset8x8x8(1, 6, 7, 8, 8), + offset8x8x8(1, 7, 0, 8, 8), offset8x8x8(1, 7, 1, 8, 8), offset8x8x8(1, 7, 2, 8, 8), offset8x8x8(1, 7, 3, 8, 8), offset8x8x8(1, 7, 4, 8, 8), offset8x8x8(1, 7, 5, 8, 8), offset8x8x8(1, 7, 6, 8, 8), offset8x8x8(1, 7, 7, 8, 8), + + offset8x8x8(2, 0, 1, 8, 8), offset8x8x8(2, 0, 3, 8, 8), offset8x8x8(2, 0, 5, 8, 8), + offset8x8x8(2, 1, 0, 8, 8), offset8x8x8(2, 1, 1, 8, 8), offset8x8x8(2, 1, 2, 8, 8), offset8x8x8(2, 1, 3, 8, 8), offset8x8x8(2, 1, 4, 8, 8), offset8x8x8(2, 1, 5, 8, 8), offset8x8x8(2, 1, 6, 8, 8), offset8x8x8(2, 1, 7, 8, 8), + offset8x8x8(2, 2, 1, 8, 8), offset8x8x8(2, 2, 3, 8, 8), offset8x8x8(2, 2, 5, 8, 8), + offset8x8x8(2, 3, 0, 8, 8), offset8x8x8(2, 3, 1, 8, 8), offset8x8x8(2, 3, 2, 8, 8), offset8x8x8(2, 3, 3, 8, 8), offset8x8x8(2, 3, 4, 8, 8), offset8x8x8(2, 3, 5, 8, 8), offset8x8x8(2, 3, 6, 8, 8), offset8x8x8(2, 3, 7, 8, 8), + offset8x8x8(2, 4, 1, 8, 8), offset8x8x8(2, 4, 3, 8, 8), offset8x8x8(2, 4, 5, 8, 8), + offset8x8x8(2, 5, 0, 8, 8), offset8x8x8(2, 5, 1, 8, 8), offset8x8x8(2, 5, 2, 8, 8), offset8x8x8(2, 5, 3, 8, 8), offset8x8x8(2, 5, 4, 8, 8), offset8x8x8(2, 5, 5, 8, 8), offset8x8x8(2, 5, 6, 8, 8), offset8x8x8(2, 5, 7, 8, 8), + offset8x8x8(2, 6, 1, 8, 8), offset8x8x8(2, 6, 3, 8, 8), offset8x8x8(2, 6, 5, 8, 8), + offset8x8x8(2, 7, 1, 8, 8), offset8x8x8(2, 7, 3, 8, 8), offset8x8x8(2, 7, 5, 8, 8), + + offset8x8x8(3, 0, 0, 8, 8), offset8x8x8(3, 0, 1, 8, 8), offset8x8x8(3, 0, 2, 8, 8), offset8x8x8(3, 0, 3, 8, 8), offset8x8x8(3, 0, 4, 8, 8), offset8x8x8(3, 0, 5, 8, 8), offset8x8x8(3, 0, 6, 8, 8), offset8x8x8(3, 0, 7, 8, 8), + offset8x8x8(3, 1, 0, 8, 8), offset8x8x8(3, 1, 1, 8, 8), offset8x8x8(3, 1, 2, 8, 8), offset8x8x8(3, 1, 3, 8, 8), offset8x8x8(3, 1, 4, 8, 8), offset8x8x8(3, 1, 5, 8, 8), offset8x8x8(3, 1, 6, 8, 8), offset8x8x8(3, 1, 7, 8, 8), + offset8x8x8(3, 2, 0, 8, 8), offset8x8x8(3, 2, 1, 8, 8), offset8x8x8(3, 2, 2, 8, 8), offset8x8x8(3, 2, 3, 8, 8), offset8x8x8(3, 2, 4, 8, 8), offset8x8x8(3, 2, 5, 8, 8), offset8x8x8(3, 2, 6, 8, 8), offset8x8x8(3, 2, 7, 8, 8), + offset8x8x8(3, 3, 0, 8, 8), offset8x8x8(3, 3, 1, 8, 8), offset8x8x8(3, 3, 2, 8, 8), offset8x8x8(3, 3, 3, 8, 8), offset8x8x8(3, 3, 4, 8, 8), offset8x8x8(3, 3, 5, 8, 8), offset8x8x8(3, 3, 6, 8, 8), offset8x8x8(3, 3, 7, 8, 8), + offset8x8x8(3, 4, 0, 8, 8), offset8x8x8(3, 4, 1, 8, 8), offset8x8x8(3, 4, 2, 8, 8), offset8x8x8(3, 4, 3, 8, 8), offset8x8x8(3, 4, 4, 8, 8), offset8x8x8(3, 4, 5, 8, 8), offset8x8x8(3, 4, 6, 8, 8), offset8x8x8(3, 4, 7, 8, 8), + offset8x8x8(3, 5, 0, 8, 8), offset8x8x8(3, 5, 1, 8, 8), offset8x8x8(3, 5, 2, 8, 8), offset8x8x8(3, 5, 3, 8, 8), offset8x8x8(3, 5, 4, 8, 8), offset8x8x8(3, 5, 5, 8, 8), offset8x8x8(3, 5, 6, 8, 8), offset8x8x8(3, 5, 7, 8, 8), + offset8x8x8(3, 6, 0, 8, 8), offset8x8x8(3, 6, 1, 8, 8), offset8x8x8(3, 6, 2, 8, 8), offset8x8x8(3, 6, 3, 8, 8), offset8x8x8(3, 6, 4, 8, 8), offset8x8x8(3, 6, 5, 8, 8), offset8x8x8(3, 6, 6, 8, 8), offset8x8x8(3, 6, 7, 8, 8), + offset8x8x8(3, 7, 0, 8, 8), offset8x8x8(3, 7, 1, 8, 8), offset8x8x8(3, 7, 2, 8, 8), offset8x8x8(3, 7, 3, 8, 8), offset8x8x8(3, 7, 4, 8, 8), offset8x8x8(3, 7, 5, 8, 8), offset8x8x8(3, 7, 6, 8, 8), offset8x8x8(3, 7, 7, 8, 8), + + offset8x8x8(4, 0, 1, 8, 8), offset8x8x8(4, 0, 3, 8, 8), offset8x8x8(4, 0, 5, 8, 8), + offset8x8x8(4, 1, 0, 8, 8), offset8x8x8(4, 1, 1, 8, 8), offset8x8x8(4, 1, 2, 8, 8), offset8x8x8(4, 1, 3, 8, 8), offset8x8x8(4, 1, 4, 8, 8), offset8x8x8(4, 1, 5, 8, 8), offset8x8x8(4, 1, 6, 8, 8), offset8x8x8(4, 1, 7, 8, 8), + offset8x8x8(4, 2, 1, 8, 8), offset8x8x8(4, 2, 3, 8, 8), offset8x8x8(4, 2, 5, 8, 8), + offset8x8x8(4, 3, 0, 8, 8), offset8x8x8(4, 3, 1, 8, 8), offset8x8x8(4, 3, 2, 8, 8), offset8x8x8(4, 3, 3, 8, 8), offset8x8x8(4, 3, 4, 8, 8), offset8x8x8(4, 3, 5, 8, 8), offset8x8x8(4, 3, 6, 8, 8), offset8x8x8(4, 3, 7, 8, 8), + offset8x8x8(4, 4, 1, 8, 8), offset8x8x8(4, 4, 3, 8, 8), offset8x8x8(4, 4, 5, 8, 8), + offset8x8x8(4, 5, 0, 8, 8), offset8x8x8(4, 5, 1, 8, 8), offset8x8x8(4, 5, 2, 8, 8), offset8x8x8(4, 5, 3, 8, 8), offset8x8x8(4, 5, 4, 8, 8), offset8x8x8(4, 5, 5, 8, 8), offset8x8x8(4, 5, 6, 8, 8), offset8x8x8(4, 5, 7, 8, 8), + offset8x8x8(4, 6, 1, 8, 8), offset8x8x8(4, 6, 3, 8, 8), offset8x8x8(4, 6, 5, 8, 8), + offset8x8x8(4, 7, 1, 8, 8), offset8x8x8(4, 7, 3, 8, 8), offset8x8x8(4, 7, 5, 8, 8), + + offset8x8x8(5, 0, 0, 8, 8), offset8x8x8(5, 0, 1, 8, 8), offset8x8x8(5, 0, 2, 8, 8), offset8x8x8(5, 0, 3, 8, 8), offset8x8x8(5, 0, 4, 8, 8), offset8x8x8(5, 0, 5, 8, 8), offset8x8x8(5, 0, 6, 8, 8), offset8x8x8(5, 0, 7, 8, 8), + offset8x8x8(5, 1, 0, 8, 8), offset8x8x8(5, 1, 1, 8, 8), offset8x8x8(5, 1, 2, 8, 8), offset8x8x8(5, 1, 3, 8, 8), offset8x8x8(5, 1, 4, 8, 8), offset8x8x8(5, 1, 5, 8, 8), offset8x8x8(5, 1, 6, 8, 8), offset8x8x8(5, 1, 7, 8, 8), + offset8x8x8(5, 2, 0, 8, 8), offset8x8x8(5, 2, 1, 8, 8), offset8x8x8(5, 2, 2, 8, 8), offset8x8x8(5, 2, 3, 8, 8), offset8x8x8(5, 2, 4, 8, 8), offset8x8x8(5, 2, 5, 8, 8), offset8x8x8(5, 2, 6, 8, 8), offset8x8x8(5, 2, 7, 8, 8), + offset8x8x8(5, 3, 0, 8, 8), offset8x8x8(5, 3, 1, 8, 8), offset8x8x8(5, 3, 2, 8, 8), offset8x8x8(5, 3, 3, 8, 8), offset8x8x8(5, 3, 4, 8, 8), offset8x8x8(5, 3, 5, 8, 8), offset8x8x8(5, 3, 6, 8, 8), offset8x8x8(5, 3, 7, 8, 8), + offset8x8x8(5, 4, 0, 8, 8), offset8x8x8(5, 4, 1, 8, 8), offset8x8x8(5, 4, 2, 8, 8), offset8x8x8(5, 4, 3, 8, 8), offset8x8x8(5, 4, 4, 8, 8), offset8x8x8(5, 4, 5, 8, 8), offset8x8x8(5, 4, 6, 8, 8), offset8x8x8(5, 4, 7, 8, 8), + offset8x8x8(5, 5, 0, 8, 8), offset8x8x8(5, 5, 1, 8, 8), offset8x8x8(5, 5, 2, 8, 8), offset8x8x8(5, 5, 3, 8, 8), offset8x8x8(5, 5, 4, 8, 8), offset8x8x8(5, 5, 5, 8, 8), offset8x8x8(5, 5, 6, 8, 8), offset8x8x8(5, 5, 7, 8, 8), + offset8x8x8(5, 6, 0, 8, 8), offset8x8x8(5, 6, 1, 8, 8), offset8x8x8(5, 6, 2, 8, 8), offset8x8x8(5, 6, 3, 8, 8), offset8x8x8(5, 6, 4, 8, 8), offset8x8x8(5, 6, 5, 8, 8), offset8x8x8(5, 6, 6, 8, 8), offset8x8x8(5, 6, 7, 8, 8), + offset8x8x8(5, 7, 0, 8, 8), offset8x8x8(5, 7, 1, 8, 8), offset8x8x8(5, 7, 2, 8, 8), offset8x8x8(5, 7, 3, 8, 8), offset8x8x8(5, 7, 4, 8, 8), offset8x8x8(5, 7, 5, 8, 8), offset8x8x8(5, 7, 6, 8, 8), offset8x8x8(5, 7, 7, 8, 8), + + offset8x8x8(6, 0, 1, 8, 8), offset8x8x8(6, 0, 3, 8, 8), offset8x8x8(6, 0, 5, 8, 8), + offset8x8x8(6, 1, 0, 8, 8), offset8x8x8(6, 1, 1, 8, 8), offset8x8x8(6, 1, 2, 8, 8), offset8x8x8(6, 1, 3, 8, 8), offset8x8x8(6, 1, 4, 8, 8), offset8x8x8(6, 1, 5, 8, 8), offset8x8x8(6, 1, 6, 8, 8), offset8x8x8(6, 1, 7, 8, 8), + offset8x8x8(6, 2, 1, 8, 8), offset8x8x8(6, 2, 3, 8, 8), offset8x8x8(6, 2, 5, 8, 8), + offset8x8x8(6, 3, 0, 8, 8), offset8x8x8(6, 3, 1, 8, 8), offset8x8x8(6, 3, 2, 8, 8), offset8x8x8(6, 3, 3, 8, 8), offset8x8x8(6, 3, 4, 8, 8), offset8x8x8(6, 3, 5, 8, 8), offset8x8x8(6, 3, 6, 8, 8), offset8x8x8(6, 3, 7, 8, 8), + offset8x8x8(6, 4, 1, 8, 8), offset8x8x8(6, 4, 3, 8, 8), offset8x8x8(6, 4, 5, 8, 8), + offset8x8x8(6, 5, 0, 8, 8), offset8x8x8(6, 5, 1, 8, 8), offset8x8x8(6, 5, 2, 8, 8), offset8x8x8(6, 5, 3, 8, 8), offset8x8x8(6, 5, 4, 8, 8), offset8x8x8(6, 5, 5, 8, 8), offset8x8x8(6, 5, 6, 8, 8), offset8x8x8(6, 5, 7, 8, 8), + offset8x8x8(6, 6, 1, 8, 8), offset8x8x8(6, 6, 3, 8, 8), offset8x8x8(6, 6, 5, 8, 8), + offset8x8x8(6, 7, 1, 8, 8), offset8x8x8(6, 7, 3, 8, 8), offset8x8x8(6, 7, 5, 8, 8), + + offset8x8x8(7, 0, 1, 8, 8), offset8x8x8(7, 0, 3, 8, 8), offset8x8x8(7, 0, 5, 8, 8), + offset8x8x8(7, 1, 0, 8, 8), offset8x8x8(7, 1, 1, 8, 8), offset8x8x8(7, 1, 2, 8, 8), offset8x8x8(7, 1, 3, 8, 8), offset8x8x8(7, 1, 4, 8, 8), offset8x8x8(7, 1, 5, 8, 8), offset8x8x8(7, 1, 6, 8, 8), offset8x8x8(7, 1, 7, 8, 8), + offset8x8x8(7, 2, 1, 8, 8), offset8x8x8(7, 2, 3, 8, 8), offset8x8x8(7, 2, 5, 8, 8), + offset8x8x8(7, 3, 0, 8, 8), offset8x8x8(7, 3, 1, 8, 8), offset8x8x8(7, 3, 2, 8, 8), offset8x8x8(7, 3, 3, 8, 8), offset8x8x8(7, 3, 4, 8, 8), offset8x8x8(7, 3, 5, 8, 8), offset8x8x8(7, 3, 6, 8, 8), offset8x8x8(7, 3, 7, 8, 8), + offset8x8x8(7, 4, 1, 8, 8), offset8x8x8(7, 4, 3, 8, 8), offset8x8x8(7, 4, 5, 8, 8), + offset8x8x8(7, 5, 0, 8, 8), offset8x8x8(7, 5, 1, 8, 8), offset8x8x8(7, 5, 2, 8, 8), offset8x8x8(7, 5, 3, 8, 8), offset8x8x8(7, 5, 4, 8, 8), offset8x8x8(7, 5, 5, 8, 8), offset8x8x8(7, 5, 6, 8, 8), offset8x8x8(7, 5, 7, 8, 8), + offset8x8x8(7, 6, 1, 8, 8), offset8x8x8(7, 6, 3, 8, 8), offset8x8x8(7, 6, 5, 8, 8), + offset8x8x8(7, 7, 1, 8, 8), offset8x8x8(7, 7, 3, 8, 8), offset8x8x8(7, 7, 5, 8, 8) + }; + return offset[i]; +} + +// clang-format on +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTableLowDim.hpp b/include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTableLowDim.hpp new file mode 100644 index 000000000..2dff8b33a --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/InCacheBlock/IndexTableLowDim.hpp @@ -0,0 +1,373 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_INDEX_TABLE_LOW_DIM_HPP +#define MGARD_X_160_INDEX_TABLE_LOW_DIM_HPP + +#include "IndexTable8x8x8.hpp" + +namespace mgard_x_160 { + +// --------------------------------------------------------------------------- +// Index tables for the 1D (8) and 2D (8x8) in-cache block transforms. +// +// The block geometry is the same in every dimension as the 3D block: an +// 8-wide block coarsens to the 5 nodes {0, 2, 4, 6, 7}, and the trailing cell +// [6, 7] is treated as two half cells around a phantom node at 6.5. So the +// coarse-grid spacing is {2, 2, 2, 1} and the mass-transform weight cases are +// identical to the 3D block's -- am_8x8x8 / bm_8x8x8 / +// MassTrans_Weights_8x8x8 are reused here verbatim rather than duplicated. +// +// IndexTable8x8x8.hpp spells out every table entry by hand. The low-dimension +// tables are two orders of magnitude smaller, so they are built with constexpr +// loops instead: the geometry rules are stated once below and the compiler +// expands them into the same kind of `static constexpr` lookup tables. +// --------------------------------------------------------------------------- + +// Minimal constexpr table containers. Not std::array: these are read from +// device code, where std::array's constexpr accessors are not portably usable. +template struct LowDimTable { int v[N][K]; }; + +static constexpr int LowDim_Block = 8; // fine nodes per block per dim +static constexpr int LowDim_Coarse = 5; // coarse nodes per block per dim +static constexpr int LowDim_Coeff1D = 3; // coefficient nodes per dim + +static constexpr int lowdim_coarse_idx[LowDim_Coarse] = {0, 2, 4, 6, 7}; +static constexpr int lowdim_coeff_idx[LowDim_Coeff1D] = {1, 3, 5}; + +MGARDX_CONT_EXEC constexpr bool lowdim_is_coarse(int i) { + return i == 0 || i == 2 || i == 4 || i == 6 || i == 7; +} + +// Fine-grid position feeding stencil slot `slot` (0..4, i.e. -2..+2 half-cell +// steps) of coarse node `j`, or -1 when that slot has no node -- either +// outside the block or the phantom node at 6.5. +// +// `masked` additionally drops fine nodes that are themselves coarse nodes. +// The correction is the mass matrix applied to the *coefficient* field, which +// is zero at coarse nodes; along a line whose other coordinates are all +// coarse, the fine nodes at coarse positions therefore contribute nothing. +// This mirrors the OFFSET1 (masked) / OFFSET2 (full) split of +// MassTrans_X_Offset_8x8x8. +MGARDX_CONT_EXEC constexpr int lowdim_mt_pos(int j, int slot, bool masked) { + int p = -1; + if (j == 3) { + // Coarse node 6: slot 3 is the phantom node at 6.5, slot 4 is node 7. + const int tail[5] = {4, 5, 6, -1, 7}; + p = tail[slot]; + } else if (j == 4) { + // Coarse node 7: slot 1 is the phantom node at 6.5, nothing to the right. + const int tail[5] = {6, -1, 7, -1, -1}; + p = tail[slot]; + } else { + p = lowdim_coarse_idx[j] + slot - 2; + if (p < 0 || p >= LowDim_Block) { + p = -1; + } + } + if (masked && p >= 0 && lowdim_is_coarse(p)) { + p = -1; + } + return p; +} + +// --------------------------------------------------------------------------- +// 2D: 8x8 block +// +// Shared memory, in one allocation, offsets below relative to sm_v: +// sm_v [8][9] the block itself; the x-row length is padded from 8 to 9 so +// the power-of-two coefficient strides scatter across the 32 +// banks, same rationale as SMV_LDY_8x8x8 +// sm_x [8][5] after the x mass transform +// sm_y [5][5] after the y mass transform, then the tridiagonal solves +// one trailing slot holding a constant zero, read by stencil slots that +// have no node. +// --------------------------------------------------------------------------- +static constexpr int SMV_LDY_8x8 = 9; +static constexpr int SMV_SIZE_8x8 = 7 * SMV_LDY_8x8 + 7 + 1; // 71 +static constexpr int SMX_SIZE_8x8 = LowDim_Block * LowDim_Coarse; +static constexpr int SMY_SIZE_8x8 = LowDim_Coarse * LowDim_Coarse; +static constexpr int SM_SIZE_8x8 = + SMV_SIZE_8x8 + SMX_SIZE_8x8 + SMY_SIZE_8x8 + 1; +// The one zero slot, addressed relative to sm_v and to sm_x respectively. +static constexpr int ZERO_V_8x8 = SMV_SIZE_8x8 + SMX_SIZE_8x8 + SMY_SIZE_8x8; +static constexpr int ZERO_X_8x8 = SMX_SIZE_8x8 + SMY_SIZE_8x8; + +static constexpr int NumCoarse_8x8 = LowDim_Coarse * LowDim_Coarse; // 25 +static constexpr int NumCoeff_8x8 = + LowDim_Block * LowDim_Block - NumCoarse_8x8; // 39 +static constexpr int NumCoeff1D_8x8 = 2 * LowDim_Coarse * LowDim_Coeff1D; // 30 +static constexpr int NumCoeff2D_8x8 = LowDim_Coeff1D * LowDim_Coeff1D; // 9 +static constexpr int NumMassTransX_8x8 = LowDim_Block * LowDim_Coarse; // 40 +static constexpr int NumMassTransY_8x8 = LowDim_Coarse * LowDim_Coarse; // 25 + +MGARDX_CONT_EXEC constexpr int offset8x8(int y, int x) { + return y * SMV_LDY_8x8 + x; +} + +// Interpolation, 1D coefficients: {middle, left, right} sm_v offsets. +// First the 15 x-direction coefficients (on coarse rows), then the 15 +// y-direction ones (on coarse columns). +MGARDX_CONT_EXEC constexpr LowDimTable make_coeff1d_8x8() { + LowDimTable t{}; + int n = 0; + for (int iy = 0; iy < LowDim_Coarse; iy++) { + for (int j = 0; j < LowDim_Coeff1D; j++) { + int y = lowdim_coarse_idx[iy]; + int x = lowdim_coeff_idx[j]; + t.v[n][0] = offset8x8(y, x); + t.v[n][1] = offset8x8(y, x - 1); + t.v[n][2] = offset8x8(y, x + 1); + n++; + } + } + for (int j = 0; j < LowDim_Coeff1D; j++) { + for (int ix = 0; ix < LowDim_Coarse; ix++) { + int y = lowdim_coeff_idx[j]; + int x = lowdim_coarse_idx[ix]; + t.v[n][0] = offset8x8(y, x); + t.v[n][1] = offset8x8(y - 1, x); + t.v[n][2] = offset8x8(y + 1, x); + n++; + } + } + return t; +} + +MGARDX_EXEC int const *Coeff1D_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = make_coeff1d_8x8(); + return t.v[i]; +} + +// Interpolation, 2D coefficients: {middle, and the four surrounding corners}. +MGARDX_CONT_EXEC constexpr LowDimTable make_coeff2d_8x8() { + LowDimTable t{}; + int n = 0; + for (int jy = 0; jy < LowDim_Coeff1D; jy++) { + for (int jx = 0; jx < LowDim_Coeff1D; jx++) { + int y = lowdim_coeff_idx[jy]; + int x = lowdim_coeff_idx[jx]; + t.v[n][0] = offset8x8(y, x); + t.v[n][1] = offset8x8(y - 1, x - 1); + t.v[n][2] = offset8x8(y - 1, x + 1); + t.v[n][3] = offset8x8(y + 1, x - 1); + t.v[n][4] = offset8x8(y + 1, x + 1); + n++; + } + } + return t; +} + +MGARDX_EXEC int const *Coeff2D_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = make_coeff2d_8x8(); + return t.v[i]; +} + +// X mass transform: {5 sm_v inputs, sm_x output, weight case}, one row per +// (fine y, coarse x). Rows on coarse y are masked, as explained above. +MGARDX_CONT_EXEC constexpr LowDimTable +make_masstrans_x_8x8() { + LowDimTable t{}; + for (int y = 0; y < LowDim_Block; y++) { + bool masked = lowdim_is_coarse(y); + for (int j = 0; j < LowDim_Coarse; j++) { + int n = y * LowDim_Coarse + j; + for (int s = 0; s < 5; s++) { + int p = lowdim_mt_pos(j, s, masked); + t.v[n][s] = (p < 0) ? ZERO_V_8x8 : offset8x8(y, p); + } + t.v[n][5] = y * LowDim_Coarse + j; + t.v[n][6] = j; + } + } + return t; +} + +MGARDX_EXEC int const *MassTrans_X_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = make_masstrans_x_8x8(); + return t.v[i]; +} + +// Y mass transform: {5 sm_x inputs, sm_y output, weight case}, one row per +// (coarse y, coarse x). Never masked: the x pass already zeroed the coarse +// nodes' contribution, and the remaining passes are plain 1D mass transforms. +MGARDX_CONT_EXEC constexpr LowDimTable +make_masstrans_y_8x8() { + LowDimTable t{}; + for (int jy = 0; jy < LowDim_Coarse; jy++) { + for (int jx = 0; jx < LowDim_Coarse; jx++) { + int n = jy * LowDim_Coarse + jx; + for (int s = 0; s < 5; s++) { + int p = lowdim_mt_pos(jy, s, false); + t.v[n][s] = (p < 0) ? ZERO_X_8x8 : (p * LowDim_Coarse + jx); + } + t.v[n][5] = jy * LowDim_Coarse + jx; + t.v[n][6] = jy; + } + } + return t; +} + +MGARDX_EXEC int const *MassTrans_Y_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = make_masstrans_y_8x8(); + return t.v[i]; +} + +// Tridiagonal solves over sm_y: one line per row (x solve) or column (y solve). +MGARDX_CONT_EXEC constexpr LowDimTable +make_tridiag_x_8x8() { + LowDimTable t{}; + for (int jy = 0; jy < LowDim_Coarse; jy++) { + for (int k = 0; k < LowDim_Coarse; k++) { + t.v[jy][k] = jy * LowDim_Coarse + k; + } + } + return t; +} + +MGARDX_EXEC int const *TriDiag_X_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = + make_tridiag_x_8x8(); + return t.v[i]; +} + +MGARDX_CONT_EXEC constexpr LowDimTable +make_tridiag_y_8x8() { + LowDimTable t{}; + for (int jx = 0; jx < LowDim_Coarse; jx++) { + for (int k = 0; k < LowDim_Coarse; k++) { + t.v[jx][k] = k * LowDim_Coarse + jx; + } + } + return t; +} + +MGARDX_EXEC int const *TriDiag_Y_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = + make_tridiag_y_8x8(); + return t.v[i]; +} + +// sm_v offsets of the 25 coarse nodes, in the same row-major order as sm_y, +// so the correction at sm_y[i] belongs to Coarse_Offset_8x8(i). +MGARDX_CONT_EXEC constexpr LowDimTable make_coarse_8x8() { + LowDimTable t{}; + for (int jy = 0; jy < LowDim_Coarse; jy++) { + for (int jx = 0; jx < LowDim_Coarse; jx++) { + t.v[jy * LowDim_Coarse + jx][0] = + offset8x8(lowdim_coarse_idx[jy], lowdim_coarse_idx[jx]); + } + } + return t; +} + +MGARDX_EXEC int Coarse_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = make_coarse_8x8(); + return t.v[i][0]; +} + +// sm_v offsets of the 39 coefficient nodes, enumerated row-major over the +// non-coarse positions -- the 2D analogue of Coeff_Offset_8x8x8's ordering. +// This ordering is part of the compressed layout: it is what the ROI +// per-block quantizer indexing and the recompose kernel both assume. +MGARDX_CONT_EXEC constexpr LowDimTable make_coeff_8x8() { + LowDimTable t{}; + int n = 0; + for (int y = 0; y < LowDim_Block; y++) { + for (int x = 0; x < LowDim_Block; x++) { + if (lowdim_is_coarse(y) && lowdim_is_coarse(x)) { + continue; + } + t.v[n][0] = offset8x8(y, x); + n++; + } + } + return t; +} + +MGARDX_EXEC int Coeff_Offset_8x8(SIZE i) { + static constexpr LowDimTable t = make_coeff_8x8(); + return t.v[i][0]; +} + +// --------------------------------------------------------------------------- +// 1D: 8-element block +// +// A single 8-element block is far too little work for a thread block, so the +// 1D kernels give each thread block LowDim_Tiles_1D independent tiles, laid +// out side by side in shared memory: +// sm_v [tiles][9] 8 values plus a per-tile zero slot at index 8 +// sm_x [tiles][5] after the mass transform, then the tridiagonal solve +// A per-tile zero slot (rather than one shared slot) keeps every table entry +// a pure within-tile offset, so the kernels can add tile * stride uniformly. +// --------------------------------------------------------------------------- +static constexpr SIZE LowDim_Tiles_1D = 32; +static constexpr int SMV_STRIDE_8 = LowDim_Block + 1; // 9 +static constexpr int SMV_ZERO_8 = LowDim_Block; // per-tile zero slot +static constexpr int SMX_STRIDE_8 = LowDim_Coarse; // 5 +static constexpr SIZE SM_SIZE_8 = + LowDim_Tiles_1D * (SMV_STRIDE_8 + SMX_STRIDE_8); + +static constexpr int NumCoarse_8 = LowDim_Coarse; // 5 +static constexpr int NumCoeff_8 = LowDim_Block - LowDim_Coarse; // 3 +static constexpr int NumMassTransX_8 = LowDim_Coarse; // 5 + +// Interpolation: {middle, left, right} within-tile sm_v offsets. +MGARDX_CONT_EXEC constexpr LowDimTable make_coeff1d_8() { + LowDimTable t{}; + for (int j = 0; j < NumCoeff_8; j++) { + int x = lowdim_coeff_idx[j]; + t.v[j][0] = x; + t.v[j][1] = x - 1; + t.v[j][2] = x + 1; + } + return t; +} + +MGARDX_EXEC int const *Coeff1D_Offset_8(SIZE i) { + static constexpr LowDimTable t = make_coeff1d_8(); + return t.v[i]; +} + +// Mass transform: {5 sm_v inputs, sm_x output, weight case}. Always masked -- +// in 1D every line is a line of coarse nodes. +MGARDX_CONT_EXEC constexpr LowDimTable +make_masstrans_x_8() { + LowDimTable t{}; + for (int j = 0; j < LowDim_Coarse; j++) { + for (int s = 0; s < 5; s++) { + int p = lowdim_mt_pos(j, s, true); + t.v[j][s] = (p < 0) ? SMV_ZERO_8 : p; + } + t.v[j][5] = j; + t.v[j][6] = j; + } + return t; +} + +MGARDX_EXEC int const *MassTrans_X_Offset_8(SIZE i) { + static constexpr LowDimTable t = make_masstrans_x_8(); + return t.v[i]; +} + +MGARDX_EXEC int Coarse_Offset_8(SIZE i) { + static constexpr LowDimTable t = {{{lowdim_coarse_idx[0]}, + {lowdim_coarse_idx[1]}, + {lowdim_coarse_idx[2]}, + {lowdim_coarse_idx[3]}, + {lowdim_coarse_idx[4]}}}; + return t.v[i][0]; +} + +MGARDX_EXEC int Coeff_Offset_8(SIZE i) { + static constexpr LowDimTable t = { + {{lowdim_coeff_idx[0]}, {lowdim_coeff_idx[1]}, {lowdim_coeff_idx[2]}}}; + return t.v[i][0]; +} + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/InCacheBlock/Recompose8.hpp b/include/mgard-x-160/DataRefactoring/InCacheBlock/Recompose8.hpp new file mode 100644 index 000000000..06e68288e --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/InCacheBlock/Recompose8.hpp @@ -0,0 +1,310 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_RECOMPOSE_8_KERNEL_TEMPLATE +#define MGARD_X_160_RECOMPOSE_8_KERNEL_TEMPLATE + +#include "../../RuntimeX/RuntimeX.h" + +#include "IndexTableLowDim.hpp" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace in_cache_block { + +// Exact inverse of Decompose8Functor: same stages in reverse, with the +// correction subtracted instead of added and the interpolation added back. +template +class Recompose8Functor : public Functor { +public: + MGARDX_CONT Recompose8Functor() {} + MGARDX_CONT Recompose8Functor(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) { + Functor(); + } + + MGARDX_EXEC void initialize_sm_8() { + sm_v = (T *)FunctorBase::GetSharedMemory(); + sm_x = sm_v + TILES * SMV_STRIDE_8; + } + + MGARDX_EXEC void init_indices() { + item = FunctorBase::GetThreadIdX(); + tile = FunctorBase::GetThreadIdY(); + bid = FunctorBase::GetBlockIdX() * TILES + tile; + num_tiles = (v.shape(D - 1) + X - 1) / X; + active = bid < (int)num_tiles; + base_v = tile * SMV_STRIDE_8; + base_x = tile * SMX_STRIDE_8; + x_gl = bid * X + item; + if (item == 0) + sm_v[base_v + SMV_ZERO_8] = (T)0; + } + + // Load + MGARDX_EXEC void Operation1() { + initialize_sm_8(); + init_indices(); + if (!active) + return; + if (item < NumCoarse_8) { + sm_v[base_v + Coarse_Offset_8(item)] = + *coarse(bid * LowDim_Coarse + item); + } else { + int op_tid = item - NumCoarse_8; + sm_v[base_v + Coeff_Offset_8(op_tid)] = *coeff(bid * NumCoeff_8 + op_tid); + } + } + + // MassTransX + MGARDX_EXEC void Operation2() { + if (!orthogonal_projection) + return; + if (active && item < NumMassTransX_8) { + int const *index = MassTrans_X_Offset_8(item); + T a = sm_v[base_v + index[0]]; + T b = sm_v[base_v + index[1]]; + T c = sm_v[base_v + index[2]]; + T d = sm_v[base_v + index[3]]; + T e = sm_v[base_v + index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_x[base_x + index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + } + + MGARDX_EXEC void solve_tridiag() { + T a = sm_x[base_x + 0]; + T b = sm_x[base_x + 1]; + T c = sm_x[base_x + 2]; + T d = sm_x[base_x + 3]; + T e = sm_x[base_x + 4]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_x[base_x + 0] = a; + sm_x[base_x + 1] = b; + sm_x[base_x + 2] = c; + sm_x[base_x + 3] = d; + sm_x[base_x + 4] = e; + } + + // TridiagX + MGARDX_EXEC void Operation3() { + if (!orthogonal_projection) + return; + if (active && item == 0) { + solve_tridiag(); + } + } + + // Subtract correction + MGARDX_EXEC void Operation4() { + if (orthogonal_projection && active && item < NumCoarse_8) { + sm_v[base_v + Coarse_Offset_8(item)] -= sm_x[base_x + item]; + } + } + + // Undo interpolation + MGARDX_EXEC void Operation5() { + if (active && item < NumCoeff_8) { + int const *index = Coeff1D_Offset_8(item); + T middle = sm_v[base_v + index[0]]; + T left = sm_v[base_v + index[1]]; + T right = sm_v[base_v + index[2]]; + sm_v[base_v + index[0]] = middle + (left + right) * (T)0.5; + } + } + + // Store + MGARDX_EXEC void Operation6() { + if (active) { + *v(x_gl) = sm_v[base_v + item]; + } + } + + MGARDX_CONT size_t shared_memory_size() { + return (size_t)TILES * (SMV_STRIDE_8 + SMX_STRIDE_8) * sizeof(T); + } + +protected: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; + T *sm_v, *sm_x; + int item, tile, bid, x_gl; + int base_v, base_x; + SIZE num_tiles; + bool active; +}; + +// Fused dequantize+recompose variant. See RecomposeDequantize8x8x8Functor: +// the 3 per-tile coefficients are read as Q symbols and dequantized while +// being staged into shared memory, and the store is bounds checked so an +// unpadded destination can be filled directly. +template +class RecomposeDequantize8Functor + : public Recompose8Functor { + using Base = Recompose8Functor; + +public: + MGARDX_CONT RecomposeDequantize8Functor() {} + MGARDX_CONT RecomposeDequantize8Functor( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : Base(v, coarse, SubArray<1, T, DeviceType>(), orthogonal_projection), + quantized_coeff(quantized_coeff), quantizer(quantizer), + block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size) {} + + MGARDX_EXEC void Operation1() { + this->initialize_sm_8(); + this->init_indices(); + if (!this->active) + return; + if (this->item < NumCoarse_8) { + this->sm_v[this->base_v + Coarse_Offset_8(this->item)] = + *this->coarse(this->bid * LowDim_Coarse + this->item); + } else { + int op_tid = this->item - NumCoarse_8; + Q quantized_data = *quantized_coeff(this->bid * NumCoeff_8 + op_tid); + if (prep_huffman) { + quantized_data -= dict_size / 2; + } + T q = use_block_quantizers ? *block_quantizers(this->bid) : quantizer; + // Must stay bit-identical to QuantizeLocalLevelFunctor (volume == 1, + // non-reciprocal quantizer). + this->sm_v[this->base_v + Coeff_Offset_8(op_tid)] = q * (T)quantized_data; + } + } + + MGARDX_EXEC void Operation6() { + if (this->active && this->x_gl < (int)this->v.shape(D - 1)) { + *this->v(this->x_gl) = this->sm_v[this->base_v + this->item]; + } + } + +protected: + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; +}; + +template +class Recompose8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_1d"; + MGARDX_CONT + Recompose8Kernel(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Recompose8Functor; + FunctorType functor(v, coarse, coeff, orthogonal_projection); + + SIZE num_tiles = (v.shape(D - 1) + 7) / 8; + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = LowDim_Tiles_1D, tbx = 8; + SIZE gridz = 1, gridy = 1; + SIZE gridx = ceil((double)num_tiles / LowDim_Tiles_1D); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; +}; + +template +class RecomposeDequantize8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_1d_fq"; + MGARDX_CONT + RecomposeDequantize8Kernel(SubArray v, + SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, + T quantizer, + SubArray<1, T, DeviceType> block_quantizers, + bool use_block_quantizers, bool prep_huffman, + SIZE dict_size, bool orthogonal_projection = true) + : v(v), coarse(coarse), quantized_coeff(quantized_coeff), + quantizer(quantizer), block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size), orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT + Task> + GenTask(int queue_idx) { + using FunctorType = + RecomposeDequantize8Functor; + FunctorType functor(v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection); + + SIZE num_tiles = (v.shape(D - 1) + 7) / 8; + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = LowDim_Tiles_1D, tbx = 8; + SIZE gridz = 1, gridy = 1; + SIZE gridx = ceil((double)num_tiles / LowDim_Tiles_1D); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; + bool orthogonal_projection; +}; + +} // namespace in_cache_block + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/InCacheBlock/Recompose8x8.hpp b/include/mgard-x-160/DataRefactoring/InCacheBlock/Recompose8x8.hpp new file mode 100644 index 000000000..2c90e2d0f --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/InCacheBlock/Recompose8x8.hpp @@ -0,0 +1,345 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_RECOMPOSE_8x8_KERNEL_TEMPLATE +#define MGARD_X_160_RECOMPOSE_8x8_KERNEL_TEMPLATE + +#include "../../RuntimeX/RuntimeX.h" + +#include "IndexTableLowDim.hpp" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace in_cache_block { + +// Exact inverse of Decompose8x8Functor: same stages in reverse, with the +// correction subtracted instead of added and the interpolation added back. +template +class Recompose8x8Functor : public Functor { +public: + MGARDX_CONT Recompose8x8Functor() {} + MGARDX_CONT Recompose8x8Functor(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) { + Functor(); + } + + MGARDX_EXEC void initialize_sm_8x8() { + sm_v = (T *)FunctorBase::GetSharedMemory(); + sm_x = sm_v + SMV_SIZE_8x8; // sm_v is padded for bank-conflict avoidance + sm_y = sm_x + SMX_SIZE_8x8; + } + + MGARDX_EXEC void init_indices() { + x = FunctorBase::GetThreadIdX(); + y = FunctorBase::GetThreadIdY(); + x_tb = FunctorBase::GetBlockIdX(); + y_tb = FunctorBase::GetBlockIdY(); + x_gl = X * x_tb + x; + y_gl = Y * y_tb + y; + tid = y * X + x; + bid = y_tb * FunctorBase::GetGridDimX() + x_tb; + if (tid == 0) + sm_v[ZERO_V_8x8] = (T)0; + offset = offset8x8(y, x); + } + + // Load + MGARDX_EXEC void Operation1() { + initialize_sm_8x8(); + init_indices(); + if (tid < NumCoarse_8x8) { + sm_v[Coarse_Offset_8x8(tid)] = + *coarse(y_tb * LowDim_Coarse + tid / LowDim_Coarse, + x_tb * LowDim_Coarse + tid % LowDim_Coarse); + } else { + int op_tid = tid - NumCoarse_8x8; + sm_v[Coeff_Offset_8x8(op_tid)] = *coeff(bid * NumCoeff_8x8 + op_tid); + } + } + + // MassTransX + MGARDX_EXEC void Operation2() { + if (!orthogonal_projection) + return; + if (tid < NumMassTransX_8x8) { + int const *index = MassTrans_X_Offset_8x8(tid); + T a = sm_v[index[0]]; + T b = sm_v[index[1]]; + T c = sm_v[index[2]]; + T d = sm_v[index[3]]; + T e = sm_v[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_x[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + } + + // MassTransY + MGARDX_EXEC void Operation3() { + if (!orthogonal_projection) + return; + if (tid < NumMassTransY_8x8) { + int const *index = MassTrans_Y_Offset_8x8(tid); + T a = sm_x[index[0]]; + T b = sm_x[index[1]]; + T c = sm_x[index[2]]; + T d = sm_x[index[3]]; + T e = sm_x[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_y[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + } + + MGARDX_EXEC void solve_tridiag(int const *index) { + T a = sm_y[index[0]]; + T b = sm_y[index[1]]; + T c = sm_y[index[2]]; + T d = sm_y[index[3]]; + T e = sm_y[index[4]]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_y[index[0]] = a; + sm_y[index[1]] = b; + sm_y[index[2]] = c; + sm_y[index[3]] = d; + sm_y[index[4]] = e; + } + + // TridiagX + MGARDX_EXEC void Operation4() { + if (!orthogonal_projection) + return; + if (tid < LowDim_Coarse) { + solve_tridiag(TriDiag_X_Offset_8x8(tid)); + } + } + + // TridiagY + MGARDX_EXEC void Operation5() { + if (!orthogonal_projection) + return; + if (tid < LowDim_Coarse) { + solve_tridiag(TriDiag_Y_Offset_8x8(tid)); + } + } + + // Subtract correction + MGARDX_EXEC void Operation6() { + if (orthogonal_projection && tid < NumCoarse_8x8) { + sm_v[Coarse_Offset_8x8(tid)] -= sm_y[tid]; + } + } + + // Undo interpolation + MGARDX_EXEC void Operation7() { + if (tid < NumCoeff1D_8x8) { + int const *index = Coeff1D_Offset_8x8(tid); + T middle = sm_v[index[0]]; + T left = sm_v[index[1]]; + T right = sm_v[index[2]]; + sm_v[index[0]] = middle + (left + right) * (T)0.5; + } else if (tid >= 32 && tid < 32 + NumCoeff2D_8x8) { + int const *index = Coeff2D_Offset_8x8(tid - 32); + T c11 = sm_v[index[0]]; + T c00 = sm_v[index[1]]; + T c02 = sm_v[index[2]]; + T c20 = sm_v[index[3]]; + T c22 = sm_v[index[4]]; + sm_v[index[0]] = c11 + (c00 + c02 + c20 + c22) / 4; + } + } + + // Store + MGARDX_EXEC void Operation8() { *v(y_gl, x_gl) = sm_v[offset]; } + + MGARDX_CONT size_t shared_memory_size() { + return (size_t)SM_SIZE_8x8 * sizeof(T); + } + +protected: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; + T *sm_v, *sm_x, *sm_y; + int y, x, y_tb, x_tb, y_gl, x_gl; + int tid, bid; + int offset; +}; + +// Fused dequantize+recompose variant (inverse of DecomposeQuantize8x8): +// identical transform pipeline, but +// (1) the 39 per-tile coefficients are read as Q symbols and dequantized +// while being staged into shared memory, and +// (2) the output is written with bounds checks so an unpadded destination can +// be filled directly. +template +class RecomposeDequantize8x8Functor + : public Recompose8x8Functor { + using Base = Recompose8x8Functor; + +public: + MGARDX_CONT RecomposeDequantize8x8Functor() {} + MGARDX_CONT RecomposeDequantize8x8Functor( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : Base(v, coarse, SubArray<1, T, DeviceType>(), orthogonal_projection), + quantized_coeff(quantized_coeff), quantizer(quantizer), + block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size) {} + + MGARDX_EXEC void Operation1() { + this->initialize_sm_8x8(); + this->init_indices(); + if (this->tid < NumCoarse_8x8) { + this->sm_v[Coarse_Offset_8x8(this->tid)] = + *this->coarse(this->y_tb * LowDim_Coarse + this->tid / LowDim_Coarse, + this->x_tb * LowDim_Coarse + this->tid % LowDim_Coarse); + } else { + int op_tid = this->tid - NumCoarse_8x8; + Q quantized_data = *quantized_coeff(this->bid * NumCoeff_8x8 + op_tid); + if (prep_huffman) { + quantized_data -= dict_size / 2; + } + T q = use_block_quantizers ? *block_quantizers(this->bid) : quantizer; + // Must stay bit-identical to QuantizeLocalLevelFunctor (volume == 1). + this->sm_v[Coeff_Offset_8x8(op_tid)] = q * (T)quantized_data; + } + } + + MGARDX_EXEC void Operation8() { + // The destination is not padded to a multiple of 8, so edge tiles must + // drop their out-of-range values. + if (this->y_gl < (int)this->v.shape(D - 2) && + this->x_gl < (int)this->v.shape(D - 1)) { + *this->v(this->y_gl, this->x_gl) = this->sm_v[this->offset]; + } + } + +protected: + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; +}; + +template +class Recompose8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_2d"; + MGARDX_CONT + Recompose8x8Kernel(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Recompose8x8Functor; + FunctorType functor(v, coarse, coeff, orthogonal_projection); + + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = 8, tbx = 8; + SIZE gridz = 1; + SIZE gridy = ceil((double)total_thread_y / tby); + SIZE gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; +}; + +template +class RecomposeDequantize8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_2d_fq"; + MGARDX_CONT + RecomposeDequantize8x8Kernel( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : v(v), coarse(coarse), quantized_coeff(quantized_coeff), + quantizer(quantizer), block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size), orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + RecomposeDequantize8x8Functor; + FunctorType functor(v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection); + + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + size_t sm_size = functor.shared_memory_size(); + SIZE tbz = 1, tby = 8, tbx = 8; + SIZE gridz = 1; + SIZE gridy = ceil((double)total_thread_y / tby); + SIZE gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; + bool orthogonal_projection; +}; + +} // namespace in_cache_block + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/InCacheBlock/Recompose8x8x8.hpp b/include/mgard-x-160/DataRefactoring/InCacheBlock/Recompose8x8x8.hpp new file mode 100644 index 000000000..1e7545a84 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/InCacheBlock/Recompose8x8x8.hpp @@ -0,0 +1,663 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_RECOMPOSE_8x8x8_KERNEL_TEMPLATE +#define MGARD_X_160_RECOMPOSE_8x8x8_KERNEL_TEMPLATE + +#include "../../RuntimeX/RuntimeX.h" + +#include "../MultiDimension/Correction/LPKFunctor.h" + +#include "../MultiDimension/Correction/IPKFunctor.h" + +#include "IndexTable3x3x3.hpp" +#include "IndexTable5x5x5.hpp" +#include "IndexTable8x8x8.hpp" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace in_cache_block { + +/* + +v x y z c total +8*8*8(512) 5*8*8(320) 5*5*8(200) 5*5*5(125) 0 1157 +5*5*5(125) 3*5*5(75) 3*3*5(45) 3*3*3(27) 8*8*8-5*5*5(387) 659 +3*3*3(27) 2*3*3(18) 2*2*3(12) 2*2*2(8) 8*8*8-3*3*3(485) 550 + + v(512) x(320) y(200) z(125) +c8(512) v(125) x( 75) y( 45) z(27) +c8(512) c5( 98) x( 18) y( 12) z( 8) +c8(512) c5( 98) c3(19) c2( 8) +*/ + +template +class Recompose8x8x8Functor : public Functor { +public: + MGARDX_CONT Recompose8x8x8Functor() {} + MGARDX_CONT Recompose8x8x8Functor(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) { + Functor(); + } + + MGARDX_EXEC void initialize_sm_8x8x8() { + sm_v = (T *)FunctorBase::GetSharedMemory(); + sm_x = sm_v + SMV_SIZE_8x8x8; // sm_v is padded for bank-conflict avoidance + sm_y = sm_x + 5 * 8 * 8; + sm_z = sm_y + 5 * 5 * 8; + } + + MGARDX_EXEC void initialize_sm_5x5x5() { + sm_c8 = (T *)FunctorBase::GetSharedMemory(); + sm_v = sm_c8 + 8 * 8 * 8; + sm_x = sm_v + 5 * 5 * 5; + sm_y = sm_x + 5 * 5 * 3; + sm_z = sm_y + 5 * 3 * 3; + } + + MGARDX_EXEC void initialize_sm_3x3x3() { + sm_c8 = (T *)FunctorBase::GetSharedMemory(); + sm_c5 = sm_c8 + 8 * 8 * 8; + sm_v = sm_c5 + 5 * 5 * 5; + sm_x = sm_v + 3 * 3 * 3; + sm_y = sm_x + 3 * 3 * 2; + sm_z = sm_y + 3 * 2 * 2; + } + + MGARDX_EXEC void initialize_sm_2x2x2() { + sm_c8 = (T *)FunctorBase::GetSharedMemory(); + sm_c5 = sm_c8 + 8 * 8 * 8; + sm_c3 = sm_c5 + 5 * 5 * 5; + sm_c2 = sm_c3 + 3 * 3 * 3; + } + + // Load data + MGARDX_EXEC void Operation1() { + initialize_sm_8x8x8(); + x = FunctorBase::GetThreadIdX(); + y = FunctorBase::GetThreadIdY(); + z = FunctorBase::GetThreadIdZ(); + x_tb = FunctorBase::GetBlockIdX(); + y_tb = FunctorBase::GetBlockIdY(); + z_tb = FunctorBase::GetBlockIdZ(); + x_gl = X * x_tb + x; + y_gl = Y * y_tb + y; + z_gl = Z * z_tb + z; + + tid = z * X * Y + y * X + x; + bid = z_tb * FunctorBase::GetGridDimX() * + FunctorBase::GetGridDimY() + + y_tb * FunctorBase::GetGridDimX() + x_tb; + if (z == 0 && y == 0 && x == 0) + sm_v[zero_const_offset] = (T)0; + + if (tid < 125) { + int const *index = Coarse_Reorder_8x8x8(tid); + sm_v[Coarse_Offset_8x8x8(tid)] = *coarse( + z_tb * 5 + index[0], y_tb * 5 + index[1], x_tb * 5 + index[2]); + } else { + int op_tid = tid - 125; + sm_v[Coeff_Offset_8x8x8(op_tid)] = *coeff(bid * 387 + op_tid); + // if (bid == 0 && op_tid == 0) { + // for (int i = 0; i < 387; i++) { + // printf("%f ", sm_v[Coeff_Offset_8x8x8(i)]); + // } + // printf("\n"); + // } + } + } + + // MassTransX + MGARDX_EXEC void Operation2() { + if (!orthogonal_projection) + return; + + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 0) { + // printf("int\n"); + // for (int i = 0; i < 8; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 8; j++) { + // for (int k = 0; k < 8; k++) { + // printf("%.6f ", sm_v[get_idx(8, 8, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + + if (tid < 320) { + int const *index = MassTrans_X_Offset_8x8x8(tid); + T a = sm_v[index[0]]; + T b = sm_v[index[1]]; + T c = sm_v[index[2]]; + T d = sm_v[index[3]]; + T e = sm_v[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_x[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 5) { + // printf("tra - x\n"); + // for (int i = 0; i < 8; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 8; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%.6f ", sm_x[get_idx(5, 8, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // MassTransY + MGARDX_EXEC void Operation3() { + if (!orthogonal_projection) + return; + if (tid < 200) { + int const *index = MassTrans_Y_Offset_8x8x8(tid); + T a = sm_x[index[0]]; + T b = sm_x[index[1]]; + T c = sm_x[index[2]]; + T d = sm_x[index[3]]; + T e = sm_x[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_y[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 0) { + // printf("tra - y\n"); + // for (int i = 0; i < 8; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 5; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%.6f ", sm_y[get_idx(5, 5, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // MassTransZ + MGARDX_EXEC void Operation4() { + if (!orthogonal_projection) + return; + if (tid < 125) { + int const *index = MassTrans_Z_Offset_8x8x8(tid); + T a = sm_y[index[0]]; + T b = sm_y[index[1]]; + T c = sm_y[index[2]]; + T d = sm_y[index[3]]; + T e = sm_y[index[4]]; + T const *dist = MassTrans_Weights_8x8x8(index[6]); + sm_z[index[5]] = + a * dist[0] + b * dist[1] + c * dist[2] + d * dist[3] + e * dist[4]; + } + + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 0) { + // printf("tra - z\n"); + // for (int i = 0; i < 5; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 5; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%.6f ", sm_z[get_idx(5, 5, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // TriadiagX + MGARDX_EXEC void Operation5() { + if (!orthogonal_projection) + return; + if (tid < 25) { + int const *index = TriDiag_X_Offset_8x8x8(tid); + T a = sm_z[index[0]]; + T b = sm_z[index[1]]; + T c = sm_z[index[2]]; + T d = sm_z[index[3]]; + T e = sm_z[index[4]]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_z[index[0]] = a; + sm_z[index[1]] = b; + sm_z[index[2]] = c; + sm_z[index[3]] = d; + sm_z[index[4]] = e; + } + + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 0) { + // printf("tri - x\n"); + // for (int i = 0; i < 5; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 5; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%.6f ", sm_z[get_idx(5, 5, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // TriadiagY + MGARDX_EXEC void Operation6() { + if (!orthogonal_projection) + return; + if (tid < 25) { + int const *index = TriDiag_Y_Offset_8x8x8(tid); + T a = sm_z[index[0]]; + T b = sm_z[index[1]]; + T c = sm_z[index[2]]; + T d = sm_z[index[3]]; + T e = sm_z[index[4]]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_z[index[0]] = a; + sm_z[index[1]] = b; + sm_z[index[2]] = c; + sm_z[index[3]] = d; + sm_z[index[4]] = e; + } + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (tid == 0) { + // printf("tri - y\n"); + // for (int i = 0; i < 5; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 5; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%.6f ", sm_z[get_idx(5, 5, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // TriadiagZ + MGARDX_EXEC void Operation7() { + if (!orthogonal_projection) + return; + if (tid < 25) { + int const *index = TriDiag_Z_Offset_8x8x8(tid); + T a = sm_z[index[0]]; + T b = sm_z[index[1]]; + T c = sm_z[index[2]]; + T d = sm_z[index[3]]; + T e = sm_z[index[4]]; + + a += 0 * amxbm_8x8x8(0); + b += a * amxbm_8x8x8(1); + c += b * amxbm_8x8x8(2); + d += c * amxbm_8x8x8(3); + e += d * amxbm_8x8x8(4); + + e = (e + am_8x8x8(5) * 0) * bm_8x8x8(5); + d = (d + am_8x8x8(4) * e) * bm_8x8x8(4); + c = (c + am_8x8x8(3) * d) * bm_8x8x8(3); + b = (b + am_8x8x8(2) * c) * bm_8x8x8(2); + a = (a + am_8x8x8(1) * b) * bm_8x8x8(1); + + sm_z[index[0]] = a; + sm_z[index[1]] = b; + sm_z[index[2]] = c; + sm_z[index[3]] = d; + sm_z[index[4]] = e; + } + // #ifdef MGARDX_COMPILE_CUDA + // __syncthreads(); + // if (bid == 0 && tid == 0) { + // printf("tri - z\n"); + // for (int i = 0; i < 5; i++) { + // printf("sm[i = %d]\n", i); + // for (int j = 0; j < 5; j++) { + // for (int k = 0; k < 5; k++) { + // printf("%10.2f ", sm_z[get_idx(5, 5, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + // #endif + } + + // Subtract correction + MGARDX_EXEC void Operation8() { + if (orthogonal_projection && tid < 125) { + sm_v[Coarse_Offset_8x8x8(tid)] -= sm_z[tid]; + } + } + + MGARDX_EXEC void Operation9() { + // #ifdef MGARDX_COMPILE_CUDA + // start = clock(); + // #endif + + op_tid = tid; + if (tid < 225) { + left = sm_v[Coeff1D_L_Offset_8x8x8(op_tid)]; + right = sm_v[Coeff1D_R_Offset_8x8x8(op_tid)]; + middle = sm_v[Coeff1D_M_Offset_8x8x8(op_tid)]; + // printf("l %f, r %f, m %f\n", left, right, middle); + middle = middle + (left + right) * (T)0.5; + sm_v[Coeff1D_M_Offset_8x8x8(op_tid)] = middle; + } else if (tid >= 256 && tid < 256 + 135) { + op_tid -= 256; + T c00 = sm_v[Coeff2D_LL_Offset_8x8x8(op_tid)]; + T c02 = sm_v[Coeff2D_LR_Offset_8x8x8(op_tid)]; + T c20 = sm_v[Coeff2D_RL_Offset_8x8x8(op_tid)]; + T c22 = sm_v[Coeff2D_RR_Offset_8x8x8(op_tid)]; + T c11 = sm_v[Coeff2D_MM_Offset_8x8x8(op_tid)]; + c11 += (c00 + c02 + c20 + c22) / 4; + sm_v[Coeff2D_MM_Offset_8x8x8(op_tid)] = c11; + } else if (tid >= 416 && tid < 416 + 27) { + op_tid -= 416; + T c000 = sm_v[Coeff3D_LLL_Offset_8x8x8(op_tid)]; + T c002 = sm_v[Coeff3D_LLR_Offset_8x8x8(op_tid)]; + T c020 = sm_v[Coeff3D_LRL_Offset_8x8x8(op_tid)]; + T c022 = sm_v[Coeff3D_LRR_Offset_8x8x8(op_tid)]; + T c200 = sm_v[Coeff3D_RLL_Offset_8x8x8(op_tid)]; + T c202 = sm_v[Coeff3D_RLR_Offset_8x8x8(op_tid)]; + T c220 = sm_v[Coeff3D_RRL_Offset_8x8x8(op_tid)]; + T c222 = sm_v[Coeff3D_RRR_Offset_8x8x8(op_tid)]; + T c111 = sm_v[Coeff3D_MMM_Offset_8x8x8(op_tid)]; + c111 += (c000 + c002 + c020 + c022 + c200 + c202 + c220 + c222) / 8; + sm_v[Coeff3D_MMM_Offset_8x8x8(op_tid)] = c111; + } + } + + // store data + MGARDX_EXEC void Operation10() { + offset = offset8x8x8(z, y, x); // padded sm_v layout + *v(z_gl, y_gl, x_gl) = sm_v[offset]; + // printf("v[%d, %d, %d] = %f\n", z_gl, y_gl, x_gl, sm_v[offset]); + // } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = SMV_SIZE_8x8x8 + Z * Y * (X / 2 + 1) + + Z * (Y / 2 + 1) * (X / 2 + 1) + + (Z / 2 + 1) * (Y / 2 + 1) * (X / 2 + 1) + 1; + return size * sizeof(T); + } + +protected: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; + T *sm_v, *sm_x, *sm_y, *sm_z, *sm_c8, *sm_c5, *sm_c3, *sm_c2; + int ld1 = X; + int ld2 = Y; + int z, y, x, z_tb, y_tb, x_tb, z_gl, y_gl, x_gl; + int tid, bid, op_tid; + T left, right, middle; + int offset; + int zero_const_offset = SMV_SIZE_8x8x8 + Z * Y * (X / 2 + 1) + + Z * (Y / 2 + 1) * (X / 2 + 1) + + (Z / 2 + 1) * (Y / 2 + 1) * (X / 2 + 1); + // #ifdef MGARDX_COMPILE_CUDA + // clock_t start, end; + // #endif +}; + +// Fused dequantize+recompose variant (inverse of DecomposeQuantize8x8x8): +// identical transform pipeline, but +// (1) the 387 per-block coefficients are read as Q symbols and dequantized +// while being staged into shared memory (with the Huffman dictionary +// shift removed, mirroring QuantizeLocalLevelFunctor's DEQUANTIZE op), +// so the T-typed coefficient array never exists in global memory, and +// (2) the reconstructed values are stored with bounds checks so the final +// level can write directly into the unpadded output array. +// The 5x5x5 coarse input stays in T: it is the previous level's output. +template +class RecomposeDequantize8x8x8Functor + : public Recompose8x8x8Functor { + using Base = Recompose8x8x8Functor; + +public: + MGARDX_CONT RecomposeDequantize8x8x8Functor() {} + MGARDX_CONT RecomposeDequantize8x8x8Functor( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : Base(v, coarse, SubArray<1, T, DeviceType>(), orthogonal_projection), + quantized_coeff(quantized_coeff), quantizer(quantizer), + block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size) {} + + MGARDX_EXEC void Operation1() { + this->initialize_sm_8x8x8(); + this->x = FunctorBase::GetThreadIdX(); + this->y = FunctorBase::GetThreadIdY(); + this->z = FunctorBase::GetThreadIdZ(); + this->x_tb = FunctorBase::GetBlockIdX(); + this->y_tb = FunctorBase::GetBlockIdY(); + this->z_tb = FunctorBase::GetBlockIdZ(); + this->x_gl = X * this->x_tb + this->x; + this->y_gl = Y * this->y_tb + this->y; + this->z_gl = Z * this->z_tb + this->z; + + this->tid = this->z * X * Y + this->y * X + this->x; + this->bid = this->z_tb * FunctorBase::GetGridDimX() * + FunctorBase::GetGridDimY() + + this->y_tb * FunctorBase::GetGridDimX() + + this->x_tb; + if (this->z == 0 && this->y == 0 && this->x == 0) + this->sm_v[this->zero_const_offset] = (T)0; + + if (this->tid < 125) { + int const *index = Coarse_Reorder_8x8x8(this->tid); + this->sm_v[Coarse_Offset_8x8x8(this->tid)] = + *this->coarse(this->z_tb * 5 + index[0], this->y_tb * 5 + index[1], + this->x_tb * 5 + index[2]); + } else { + int op_tid = this->tid - 125; + Q quantized_data = *quantized_coeff(this->bid * 387 + op_tid); + if (prep_huffman) { + quantized_data -= dict_size / 2; + } + T q = use_block_quantizers ? *block_quantizers(this->bid) : quantizer; + // Must stay bit-identical to QuantizeLocalLevelFunctor (volume == 1, + // non-reciprocal quantizer). + this->sm_v[Coeff_Offset_8x8x8(op_tid)] = q * (T)quantized_data; + } + } + + MGARDX_EXEC void Operation10() { + // Unlike the unfused functor, keep a bounds check on the store: the + // final level writes directly into the unpadded output array, so edge + // blocks must drop out-of-range results. + if (this->z_gl < (int)this->v.shape(D - 3) && + this->y_gl < (int)this->v.shape(D - 2) && + this->x_gl < (int)this->v.shape(D - 1)) { + this->offset = offset8x8x8(this->z, this->y, this->x); + *this->v(this->z_gl, this->y_gl, this->x_gl) = this->sm_v[this->offset]; + } + } + +protected: + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; +}; + +template +class Recompose8x8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk"; + MGARDX_CONT + Recompose8x8x8Kernel(SubArray v, + SubArray coarse, + SubArray<1, T, DeviceType> coeff, + bool orthogonal_projection = true) + : v(v), coarse(coarse), coeff(coeff), + orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Recompose8x8x8Functor; + FunctorType functor(v, coarse, coeff, orthogonal_projection); + + SIZE total_thread_z = v.shape(D - 3); + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 8; + tby = 8; + tbx = 8; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, T, DeviceType> coeff; + bool orthogonal_projection; +}; + +template +class RecomposeDequantize8x8x8Kernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lwpk_fdq"; + MGARDX_CONT + RecomposeDequantize8x8x8Kernel( + SubArray v, SubArray coarse, + SubArray<1, Q, DeviceType> quantized_coeff, T quantizer, + SubArray<1, T, DeviceType> block_quantizers, bool use_block_quantizers, + bool prep_huffman, SIZE dict_size, bool orthogonal_projection = true) + : v(v), coarse(coarse), quantized_coeff(quantized_coeff), + quantizer(quantizer), block_quantizers(block_quantizers), + use_block_quantizers(use_block_quantizers), prep_huffman(prep_huffman), + dict_size(dict_size), orthogonal_projection(orthogonal_projection) {} + + MGARDX_CONT + Task> + GenTask(int queue_idx) { + using FunctorType = + RecomposeDequantize8x8x8Functor; + FunctorType functor(v, coarse, quantized_coeff, quantizer, block_quantizers, + use_block_quantizers, prep_huffman, dict_size, + orthogonal_projection); + + // Same launch geometry as Recompose8x8x8Kernel; v may be unpadded here + // but ceil(shape / 8) matches the padded grid exactly. + SIZE total_thread_z = v.shape(D - 3); + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 8; + tby = 8; + tbx = 8; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v; + SubArray coarse; + SubArray<1, Q, DeviceType> quantized_coeff; + T quantizer; + SubArray<1, T, DeviceType> block_quantizers; + bool use_block_quantizers; + bool prep_huffman; + SIZE dict_size; + bool orthogonal_projection; +}; + +} // namespace in_cache_block + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficients3D.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficients3D.hpp new file mode 100644 index 000000000..ae580a85b --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficients3D.hpp @@ -0,0 +1,101 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../../RuntimeX/RuntimeX.h" + +#include "../../../Hierarchy/Hierarchy.h" + +#include "../DataRefactoring.h" + +#include "GridProcessingKernel3D.hpp" + +#ifndef MGARD_X_160_DATA_REFACTORING_CALC_COEFFICIENTS_3D +#define MGARD_X_160_DATA_REFACTORING_CALC_COEFFICIENTS_3D + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +void CalcCoefficients3D(Hierarchy &hierarchy, + SubArray dinput, + SubArray &doutput, SIZE l, + int queue_idx) { + + std::string prefix = "decomp_"; + if (sizeof(T) == sizeof(double)) + prefix += "d_"; + if (sizeof(T) == sizeof(float)) + prefix += "f_"; + for (int d = 0; d < D; d++) + prefix += + std::to_string(hierarchy.level_shape(hierarchy.l_target(), d)) + "_"; + + dinput.project(D - 3, D - 2, D - 1); + doutput.project(D - 3, D - 2, D - 1); + SIZE f, c, r, ff, cc, rr; + + f = hierarchy.level_shape(l, D - 1); + c = hierarchy.level_shape(l, D - 2); + r = hierarchy.level_shape(l, D - 3); + ff = hierarchy.level_shape(l - 1, D - 1); + cc = hierarchy.level_shape(l - 1, D - 2); + rr = hierarchy.level_shape(l - 1, D - 3); + + SubArray dcoarse = doutput; + dcoarse.resize({rr, cc, ff}); + SubArray dcoeff_f = doutput; + dcoeff_f.offset({0, 0, ff}); + dcoeff_f.resize({rr, cc, f - ff}); + SubArray dcoeff_c = doutput; + dcoeff_c.offset({0, cc, 0}); + dcoeff_c.resize({rr, c - cc, ff}); + SubArray dcoeff_r = doutput; + dcoeff_r.offset({rr, 0, 0}); + dcoeff_r.resize({r - rr, cc, ff}); + SubArray dcoeff_cf = doutput; + dcoeff_cf.offset({0, cc, ff}); + dcoeff_cf.resize({rr, c - cc, f - ff}); + SubArray dcoeff_rf = doutput; + dcoeff_rf.offset({rr, 0, ff}); + dcoeff_rf.resize({r - rr, cc, f - ff}); + SubArray dcoeff_rc = doutput; + dcoeff_rc.offset({rr, cc, 0}); + dcoeff_rc.resize({r - rr, c - cc, ff}); + SubArray dcoeff_rcf = doutput; + dcoeff_rcf.offset({rr, cc, ff}); + dcoeff_rcf.resize({r - rr, c - cc, f - ff}); + + SubArray ratio_f(hierarchy.ratio(l, D - 1)); + SubArray ratio_c(hierarchy.ratio(l, D - 2)); + SubArray ratio_r(hierarchy.ratio(l, D - 3)); + + DeviceLauncher::Execute( + GpkReo3DKernel(r, c, f, rr, cc, ff, ratio_r, ratio_c, ratio_f, dinput, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, + dcoeff_rf, dcoeff_rc, dcoeff_rcf), + queue_idx); + + verify_matrix_cuda(r, c, f, doutput.data(), doutput.ld(D - 1), + doutput.ld(D - 2), doutput.ld(D - 1), + prefix + "gpk_reo_3d" + "_level_" + std::to_string(l), + multidim_refactoring_store, multidim_refactoring_verify); + + if (multidim_refactoring_debug_print) { + PrintSubarray("after pi_Ql_reo", doutput); + } +} + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsND.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsND.hpp new file mode 100644 index 000000000..3b70f975e --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsND.hpp @@ -0,0 +1,239 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../../Hierarchy/Hierarchy.h" +#include "../../../RuntimeX/RuntimeX.h" + +#include "../DataRefactoring.h" + +#include "CalcCoefficientsPointers.hpp" +#include "GridProcessingKernel.hpp" + +#ifndef MGARD_X_160_DATA_REFACTORING_CALC_COEFFICIENTS_ND +#define MGARD_X_160_DATA_REFACTORING_CALC_COEFFICIENTS_ND + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +void CalcCoefficientsND(Hierarchy &hierarchy, + SubArray dinput1, + SubArray dinput2, + SubArray &doutput, SIZE l, + int queue_idx) { + + std::string prefix = "decomp_"; + if (sizeof(T) == sizeof(double)) + prefix += "d_"; + if (sizeof(T) == sizeof(float)) + prefix += "f_"; + for (int d = 0; d < D; d++) + prefix += + std::to_string(hierarchy.level_shape(hierarchy.l_target(), d)) + "_"; + // printf("interpolate 1-3D\n"); + + SubArray dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, + dcoeff_rf, dcoeff_rc, dcoeff_rcf; + + DIM curr_dims[3]; + + SubArray<1, T, DeviceType> ratio_r, ratio_c, ratio_f; + + SubArray<1, SIZE, DeviceType> shape, shape_c; + + DIM unprocessed_n; + SubArray<1, DIM, DeviceType> unprocessed_dims; + + shape = SubArray<1, SIZE, DeviceType>(hierarchy.level_shape_array(l)); + shape_c = SubArray<1, SIZE, DeviceType>(hierarchy.level_shape_array(l - 1)); + + int unprocessed_idx = 0; + unprocessed_dims = hierarchy.unprocessed(unprocessed_idx, unprocessed_n); + + curr_dims[0] = D - 3; + curr_dims[1] = D - 2; + curr_dims[2] = D - 1; + + dinput1.project(curr_dims[0], curr_dims[1], curr_dims[2]); + doutput.project(curr_dims[0], curr_dims[1], curr_dims[2]); + + CalcCoefficientsPointers(hierarchy, curr_dims, l, doutput, dcoarse, dcoeff_f, + dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, + dcoeff_rcf); + ratio_r = SubArray(hierarchy.ratio(l, curr_dims[0])); + ratio_c = SubArray(hierarchy.ratio(l, curr_dims[1])); + ratio_f = SubArray(hierarchy.ratio(l, curr_dims[2])); + DeviceLauncher::Execute( + GpkReoKernel( + shape, shape_c, unprocessed_n, unprocessed_dims, curr_dims[0], + curr_dims[1], curr_dims[2], ratio_r, ratio_c, ratio_f, dinput1, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf), + queue_idx); + + for (DIM d = 3; d < D; d += 2) { + // copy back to input1 for interpolation again + CopyND(doutput, dinput1, queue_idx); + + // printf("interpolate %u-%uD\n", d+1, d+2); + + curr_dims[0] = D - (d + 1 + 1); + curr_dims[1] = D - (d + 1); + curr_dims[2] = D - 1; + + dinput1.project(curr_dims[0], curr_dims[1], curr_dims[2]); + doutput.project(curr_dims[0], curr_dims[1], curr_dims[2]); + + CalcCoefficientsPointers(hierarchy, curr_dims, l, doutput, dcoarse, + dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf); + ratio_r = SubArray(hierarchy.ratio(l, curr_dims[0])); + ratio_c = SubArray(hierarchy.ratio(l, curr_dims[1])); + ratio_f = SubArray(hierarchy.ratio(l, curr_dims[2])); + + if (D - d == 1) { + unprocessed_idx += 1; + unprocessed_dims = hierarchy.unprocessed(unprocessed_idx, unprocessed_n); + DeviceLauncher::Execute( + GpkReoKernel( + shape, shape_c, unprocessed_n, unprocessed_dims, curr_dims[0], + curr_dims[1], curr_dims[2], ratio_r, ratio_c, ratio_f, dinput1, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf), + queue_idx); + + } else { // D - d >= 2 + unprocessed_idx += 2; + unprocessed_dims = hierarchy.unprocessed(unprocessed_idx, unprocessed_n); + DeviceLauncher::Execute( + GpkReoKernel( + shape, shape_c, unprocessed_n, unprocessed_dims, curr_dims[0], + curr_dims[1], curr_dims[2], ratio_r, ratio_c, ratio_f, dinput1, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf), + queue_idx); + } + } + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D("after interpolation", doutput); + } // debug + + // printf("reorder 1-3D\n"); + + unprocessed_idx = 0; + unprocessed_dims = hierarchy.unprocessed(unprocessed_idx, unprocessed_n); + + curr_dims[0] = D - 3; + curr_dims[1] = D - 2; + curr_dims[2] = D - 1; + + dinput2.project(curr_dims[0], curr_dims[1], curr_dims[2]); + dinput1.project(curr_dims[0], curr_dims[1], + curr_dims[2]); // reuse input1 as temp output + + CalcCoefficientsPointers(hierarchy, curr_dims, l, dinput1, dcoarse, dcoeff_f, + dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, + dcoeff_rcf); + ratio_r = SubArray(hierarchy.ratio(l, curr_dims[0])); + ratio_c = SubArray(hierarchy.ratio(l, curr_dims[1])); + ratio_f = SubArray(hierarchy.ratio(l, curr_dims[2])); + + DeviceLauncher::Execute( + GpkReoKernel( + shape, shape_c, unprocessed_n, unprocessed_dims, curr_dims[0], + curr_dims[1], curr_dims[2], ratio_r, ratio_c, ratio_f, dinput2, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf), + queue_idx); + + DIM D_reduced = D % 2 == 0 ? D - 1 : D - 2; + for (DIM d = 3; d < D_reduced; d += 2) { + // copy back to input2 for reordering again + CopyND(dinput1, dinput2, queue_idx); + // printf("reorder %u-%uD\n", d+1, d+2); + + unprocessed_idx += 2; + unprocessed_dims = hierarchy.unprocessed(unprocessed_idx, unprocessed_n); + + curr_dims[0] = D - (d + 1 + 1); + curr_dims[1] = D - (d + 1); + curr_dims[2] = D - 1; + + dinput2.project(curr_dims[0], curr_dims[1], curr_dims[2]); + dinput1.project(curr_dims[0], curr_dims[1], + curr_dims[2]); // reuse input1 as temp output + + CalcCoefficientsPointers(hierarchy, curr_dims, l, dinput1, dcoarse, + dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf); + ratio_r = SubArray(hierarchy.ratio(l, curr_dims[0])); + ratio_c = SubArray(hierarchy.ratio(l, curr_dims[1])); + ratio_f = SubArray(hierarchy.ratio(l, curr_dims[2])); + + DeviceLauncher::Execute( + GpkReoKernel( + shape, shape_c, unprocessed_n, unprocessed_dims, curr_dims[0], + curr_dims[1], curr_dims[2], ratio_r, ratio_c, ratio_f, dinput2, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf), + queue_idx); + } + + curr_dims[0] = D - (D_reduced + 1 + 1); + curr_dims[1] = D - (D_reduced + 1); + curr_dims[2] = D - 1; + + dinput1.project(curr_dims[0], curr_dims[1], curr_dims[2]); + doutput.project(curr_dims[0], curr_dims[1], + curr_dims[2]); // reuse input1 as temp output + + CalcCoefficientsPointers(hierarchy, curr_dims, l, doutput, dcoarse, dcoeff_f, + dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, + dcoeff_rcf); + ratio_r = SubArray(hierarchy.ratio(l, curr_dims[0])); + ratio_c = SubArray(hierarchy.ratio(l, curr_dims[1])); + ratio_f = SubArray(hierarchy.ratio(l, curr_dims[2])); + + if (D - D_reduced == 1) { + unprocessed_idx += 1; + unprocessed_dims = hierarchy.unprocessed(unprocessed_idx, unprocessed_n); + DeviceLauncher::Execute( + GpkReoKernel( + shape, shape_c, unprocessed_n, unprocessed_dims, curr_dims[0], + curr_dims[1], curr_dims[2], ratio_r, ratio_c, ratio_f, dinput1, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf), + queue_idx); + + } else { // D-D_reduced == 2 + unprocessed_idx += 2; + unprocessed_dims = hierarchy.unprocessed(unprocessed_idx, unprocessed_n); + DeviceLauncher::Execute( + GpkReoKernel( + shape, shape_c, unprocessed_n, unprocessed_dims, curr_dims[0], + curr_dims[1], curr_dims[2], ratio_r, ratio_c, ratio_f, dinput1, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf), + queue_idx); + } + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D("after calc coeff", doutput); + } // debug +} + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsPointers.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsPointers.hpp new file mode 100644 index 000000000..f3108bec6 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsPointers.hpp @@ -0,0 +1,96 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../../Hierarchy/Hierarchy.h" +#include "../../../RuntimeX/RuntimeX.h" + +#ifndef MGARD_X_160_DATA_REFACTORING_CALC_COEFFICIENT_POINTERS +#define MGARD_X_160_DATA_REFACTORING_CALC_COEFFICIENT_POINTERS + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +void CalcCoefficientsPointers( + Hierarchy &hierarchy, DIM curr_dims[3], DIM l, + SubArray doutput, SubArray &dcoarse, + SubArray &dcoeff_f, SubArray &dcoeff_c, + SubArray &dcoeff_r, SubArray &dcoeff_cf, + SubArray &dcoeff_rf, + SubArray &dcoeff_rc, + SubArray &dcoeff_rcf) { + + SIZE n[3]; + SIZE nn[3]; + for (DIM d = 0; d < 3; d++) { + n[d] = hierarchy.level_shape(l, curr_dims[d]); + nn[d] = hierarchy.level_shape(l - 1, curr_dims[d]); + } + + dcoarse = doutput; + dcoarse.resize(curr_dims[0], nn[0]); + dcoarse.resize(curr_dims[1], nn[1]); + dcoarse.resize(curr_dims[2], nn[2]); + + dcoeff_r = doutput; + dcoeff_r.offset_dim(curr_dims[0], nn[0]); + dcoeff_r.resize(curr_dims[0], n[0] - nn[0]); + dcoeff_r.resize(curr_dims[1], nn[1]); + dcoeff_r.resize(curr_dims[2], nn[2]); + + dcoeff_c = doutput; + dcoeff_c.offset_dim(curr_dims[1], nn[1]); + dcoeff_c.resize(curr_dims[0], nn[0]); + dcoeff_c.resize(curr_dims[1], n[1] - nn[1]); + dcoeff_c.resize(curr_dims[2], nn[2]); + + dcoeff_f = doutput; + dcoeff_f.offset_dim(curr_dims[2], nn[2]); + dcoeff_f.resize(curr_dims[0], nn[0]); + dcoeff_f.resize(curr_dims[1], nn[1]); + dcoeff_f.resize(curr_dims[2], n[2] - nn[2]); + + dcoeff_rc = doutput; + dcoeff_rc.offset_dim(curr_dims[0], nn[0]); + dcoeff_rc.offset_dim(curr_dims[1], nn[1]); + dcoeff_rc.resize(curr_dims[0], n[0] - nn[0]); + dcoeff_rc.resize(curr_dims[1], n[1] - nn[1]); + dcoeff_rc.resize(curr_dims[2], nn[2]); + + dcoeff_rf = doutput; + dcoeff_rf.offset_dim(curr_dims[0], nn[0]); + dcoeff_rf.offset_dim(curr_dims[2], nn[2]); + dcoeff_rf.resize(curr_dims[0], n[0] - nn[0]); + dcoeff_rf.resize(curr_dims[1], nn[1]); + dcoeff_rf.resize(curr_dims[2], n[2] - nn[2]); + + dcoeff_cf = doutput; + dcoeff_cf.offset_dim(curr_dims[1], nn[1]); + dcoeff_cf.offset_dim(curr_dims[2], nn[2]); + dcoeff_cf.resize(curr_dims[0], nn[0]); + dcoeff_cf.resize(curr_dims[1], n[1] - nn[1]); + dcoeff_cf.resize(curr_dims[2], n[2] - nn[2]); + + dcoeff_rcf = doutput; + dcoeff_rcf.offset_dim(curr_dims[0], nn[0]); + dcoeff_rcf.offset_dim(curr_dims[1], nn[1]); + dcoeff_rcf.offset_dim(curr_dims[2], nn[2]); + dcoeff_rcf.resize(curr_dims[0], n[0] - nn[0]); + dcoeff_rcf.resize(curr_dims[1], n[1] - nn[1]); + dcoeff_rcf.resize(curr_dims[2], n[2] - nn[2]); +} + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestore3D.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestore3D.hpp new file mode 100644 index 000000000..e710976e3 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestore3D.hpp @@ -0,0 +1,99 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../../Hierarchy/Hierarchy.h" +#include "../../../RuntimeX/RuntimeX.h" + +#include "../DataRefactoring.h" + +#include "GridProcessingKernel3D.hpp" + +#ifndef MGARD_X_160_DATA_REFACTORING_COEFFICIENTS_RESTORE_3D +#define MGARD_X_160_DATA_REFACTORING_COEFFICIENTS_RESTORE_3D + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +void CoefficientsRestore3D(Hierarchy &hierarchy, + SubArray dinput, + SubArray &doutput, SIZE l, + int queue_idx) { + + std::string prefix = "decomp_"; + if (sizeof(T) == sizeof(double)) + prefix += "d_"; + if (sizeof(T) == sizeof(float)) + prefix += "f_"; + for (int d = 0; d < D; d++) + prefix += + std::to_string(hierarchy.level_shape(hierarchy.l_target(), d)) + "_"; + + dinput.project(D - 3, D - 2, D - 1); + doutput.project(D - 3, D - 2, D - 1); + + SIZE f = hierarchy.level_shape(l, D - 1); + SIZE c = hierarchy.level_shape(l, D - 2); + SIZE r = hierarchy.level_shape(l, D - 3); + SIZE ff = hierarchy.level_shape(l - 1, D - 1); + SIZE cc = hierarchy.level_shape(l - 1, D - 2); + SIZE rr = hierarchy.level_shape(l - 1, D - 3); + + SubArray dcoarse = dinput; + dcoarse.resize({rr, cc, ff}); + SubArray dcoeff_f = dinput; + dcoeff_f.offset({0, 0, ff}); + dcoeff_f.resize({rr, cc, f - ff}); + SubArray dcoeff_c = dinput; + dcoeff_c.offset({0, cc, 0}); + dcoeff_c.resize({rr, c - cc, ff}); + SubArray dcoeff_r = dinput; + dcoeff_r.offset({rr, 0, 0}); + dcoeff_r.resize({r - rr, cc, ff}); + SubArray dcoeff_cf = dinput; + dcoeff_cf.offset({0, cc, ff}); + dcoeff_cf.resize({rr, c - cc, f - ff}); + SubArray dcoeff_rf = dinput; + dcoeff_rf.offset({rr, 0, ff}); + dcoeff_rf.resize({r - rr, cc, f - ff}); + SubArray dcoeff_rc = dinput; + dcoeff_rc.offset({rr, cc, 0}); + dcoeff_rc.resize({r - rr, c - cc, ff}); + SubArray dcoeff_rcf = dinput; + dcoeff_rcf.offset({rr, cc, ff}); + dcoeff_rcf.resize({r - rr, c - cc, f - ff}); + + SubArray ratio_f(hierarchy.ratio(l, D - 1)); + SubArray ratio_c(hierarchy.ratio(l, D - 2)); + SubArray ratio_r(hierarchy.ratio(l, D - 3)); + + DeviceLauncher::Execute( + GpkRev3DKernel(r, c, f, rr, cc, ff, ratio_r, ratio_c, ratio_f, doutput, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, + dcoeff_rf, dcoeff_rc, dcoeff_rcf, 0, 0, 0, r, c, f), + queue_idx); + + verify_matrix_cuda(r, c, f, doutput.data(), doutput.ld(D - 1), + doutput.ld(D - 2), doutput.ld(D - 1), + prefix + "gpk_rev_3d" + "_level_" + std::to_string(l), + multidim_refactoring_store, multidim_refactoring_verify); + + if (multidim_refactoring_debug_print) { + PrintSubarray("after coeff-restore", doutput); + } +} + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestoreND.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestoreND.hpp new file mode 100644 index 000000000..3716d62ea --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestoreND.hpp @@ -0,0 +1,259 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../../Hierarchy/Hierarchy.h" +#include "../../../RuntimeX/RuntimeX.h" + +#include "../DataRefactoring.h" + +#include "CalcCoefficientsPointers.hpp" +#include "GridProcessingKernel.hpp" + +#ifndef MGARD_X_160_DATA_REFACTORING_COEFFICIENTS_RESTORE_ND +#define MGARD_X_160_DATA_REFACTORING_COEFFICIENTS_RESTORE_ND + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +void CoefficientsRestoreND(Hierarchy &hierarchy, + SubArray dinput1, + SubArray dinput2, + SubArray &doutput, SIZE l, + int queue_idx) { + + std::string prefix = "decomp_"; + if (sizeof(T) == sizeof(double)) + prefix += "d_"; + if (sizeof(T) == sizeof(float)) + prefix += "f_"; + for (int d = 0; d < D; d++) + prefix += + std::to_string(hierarchy.level_shape(hierarchy.l_target(), d)) + "_"; + + SubArray dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, + dcoeff_rf, dcoeff_rc, dcoeff_rcf; + + DIM curr_dims[3]; + + SubArray<1, T, DeviceType> ratio_r, ratio_c, ratio_f; + SubArray<1, SIZE, DeviceType> shape, shape_c; + DIM unprocessed_n; + SubArray<1, DIM, DeviceType> unprocessed_dims; + + shape = SubArray<1, SIZE, DeviceType>(hierarchy.level_shape_array(l)); + shape_c = SubArray<1, SIZE, DeviceType>(hierarchy.level_shape_array(l - 1)); + + int unprocessed_idx = 0; + unprocessed_dims = hierarchy.unprocessed(unprocessed_idx, unprocessed_n); + // printf("interpolate-restore 1-3D\n"); + + curr_dims[0] = D - 3; + curr_dims[1] = D - 2; + curr_dims[2] = D - 1; + + dinput1.project(curr_dims[0], curr_dims[1], curr_dims[2]); + doutput.project(curr_dims[0], curr_dims[1], curr_dims[2]); + + CalcCoefficientsPointers(hierarchy, curr_dims, l, dinput1, dcoarse, dcoeff_f, + dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, + dcoeff_rcf); + ratio_r = SubArray(hierarchy.ratio(l, curr_dims[0])); + ratio_c = SubArray(hierarchy.ratio(l, curr_dims[1])); + ratio_f = SubArray(hierarchy.ratio(l, curr_dims[2])); + DeviceLauncher::Execute( + GpkRevKernel( + shape, shape_c, unprocessed_n, unprocessed_dims, curr_dims[0], + curr_dims[1], curr_dims[2], ratio_r, ratio_c, ratio_f, doutput, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf, 0, 0, 0, + hierarchy.level_shape(l, curr_dims[0]), + hierarchy.level_shape(l, curr_dims[1]), + hierarchy.level_shape(l, curr_dims[2])), + queue_idx); + + for (DIM d = 3; d < D; d += 2) { + CopyND(doutput, dinput1, queue_idx); + + // printf("interpolate-restore %u-%uD\n", d+1, d+2); + curr_dims[0] = D - (d + 1 + 1); + curr_dims[1] = D - (d + 1); + curr_dims[2] = D - 1; + + dinput1.project(curr_dims[0], curr_dims[1], curr_dims[2]); + doutput.project(curr_dims[0], curr_dims[1], curr_dims[2]); + + CalcCoefficientsPointers(hierarchy, curr_dims, l, dinput1, dcoarse, + dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf); + ratio_r = SubArray(hierarchy.ratio(l, curr_dims[0])); + ratio_c = SubArray(hierarchy.ratio(l, curr_dims[1])); + ratio_f = SubArray(hierarchy.ratio(l, curr_dims[2])); + + if (D - d == 1) { + unprocessed_idx += 1; + unprocessed_dims = hierarchy.unprocessed(unprocessed_idx, unprocessed_n); + DeviceLauncher::Execute( + GpkRevKernel( + shape, shape_c, unprocessed_n, unprocessed_dims, curr_dims[0], + curr_dims[1], curr_dims[2], ratio_r, ratio_c, ratio_f, doutput, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf, 0, 0, 0, + hierarchy.level_shape(l, curr_dims[0]), + hierarchy.level_shape(l, curr_dims[1]), + hierarchy.level_shape(l, curr_dims[2])), + queue_idx); + + } else { // D - d >= 2 + unprocessed_idx += 2; + unprocessed_dims = hierarchy.unprocessed(unprocessed_idx, unprocessed_n); + DeviceLauncher::Execute( + GpkRevKernel( + shape, shape_c, unprocessed_n, unprocessed_dims, curr_dims[0], + curr_dims[1], curr_dims[2], ratio_r, ratio_c, ratio_f, doutput, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf, 0, 0, 0, + hierarchy.level_shape(l, curr_dims[0]), + hierarchy.level_shape(l, curr_dims[1]), + hierarchy.level_shape(l, curr_dims[2])), + queue_idx); + } + } + // Done interpolation-restore on doutput + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D("After interpolation reverse-reorder", doutput); + } // debug + + unprocessed_idx = 0; + unprocessed_dims = hierarchy.unprocessed(unprocessed_idx, unprocessed_n); + + // printf("reorder-restore 1-3D\n"); + + curr_dims[0] = D - 3; + curr_dims[1] = D - 2; + curr_dims[2] = D - 1; + + dinput2.project(curr_dims[0], curr_dims[1], curr_dims[2]); + dinput1.project(curr_dims[0], curr_dims[1], + curr_dims[2]); // reuse input1 as temp space + + CalcCoefficientsPointers(hierarchy, curr_dims, l, dinput2, dcoarse, dcoeff_f, + dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, + dcoeff_rcf); + + ratio_r = SubArray(hierarchy.ratio(l, curr_dims[0])); + ratio_c = SubArray(hierarchy.ratio(l, curr_dims[1])); + ratio_f = SubArray(hierarchy.ratio(l, curr_dims[2])); + DeviceLauncher::Execute( + GpkRevKernel( + shape, shape_c, unprocessed_n, unprocessed_dims, curr_dims[0], + curr_dims[1], curr_dims[2], ratio_r, ratio_c, ratio_f, dinput1, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf, 0, 0, 0, + hierarchy.level_shape(l, curr_dims[0]), + hierarchy.level_shape(l, curr_dims[1]), + hierarchy.level_shape(l, curr_dims[2])), + queue_idx); + + DIM D_reduced = D % 2 == 0 ? D - 1 : D - 2; + for (DIM d = 3; d < D_reduced; d += 2) { + // printf("reorder-reverse\n"); + // copy back to input2 for reordering again + CopyND(dinput1, dinput2, queue_idx); + + // printf("reorder-restore %u-%uD\n", d+1, d+2); + + curr_dims[0] = D - (d + 1 + 1); + curr_dims[1] = D - (d + 1); + curr_dims[2] = D - 1; + + dinput2.project(curr_dims[0], curr_dims[1], curr_dims[2]); + dinput1.project(curr_dims[0], curr_dims[1], + curr_dims[2]); // reuse input1 as temp output + + CalcCoefficientsPointers(hierarchy, curr_dims, l, dinput2, dcoarse, + dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf); + ratio_r = SubArray(hierarchy.ratio(l, curr_dims[0])); + ratio_c = SubArray(hierarchy.ratio(l, curr_dims[1])); + ratio_f = SubArray(hierarchy.ratio(l, curr_dims[2])); + + unprocessed_idx += 2; + unprocessed_dims = hierarchy.unprocessed(unprocessed_idx, unprocessed_n); + DeviceLauncher::Execute( + GpkRevKernel( + shape, shape_c, unprocessed_n, unprocessed_dims, curr_dims[0], + curr_dims[1], curr_dims[2], ratio_r, ratio_c, ratio_f, dinput1, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf, 0, 0, 0, + hierarchy.level_shape(l, curr_dims[0]), + hierarchy.level_shape(l, curr_dims[1]), + hierarchy.level_shape(l, curr_dims[2])), + queue_idx); + } + + // printf("coeff-restore %u-%dD\n", D_reduced+1, D_reduced+2); + curr_dims[0] = D - (D_reduced + 1 + 1); + curr_dims[1] = D - (D_reduced + 1); + curr_dims[2] = D - 1; + + dinput1.project(curr_dims[0], curr_dims[1], curr_dims[2]); + doutput.project(curr_dims[0], curr_dims[1], curr_dims[2]); + + CalcCoefficientsPointers(hierarchy, curr_dims, l, dinput1, dcoarse, dcoeff_f, + dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, dcoeff_rc, + dcoeff_rcf); + ratio_r = SubArray(hierarchy.ratio(l, curr_dims[0])); + ratio_c = SubArray(hierarchy.ratio(l, curr_dims[1])); + ratio_f = SubArray(hierarchy.ratio(l, curr_dims[2])); + + if (D - D_reduced == 1) { + // printf("coeff-restore %u-%dD\n", D_reduced+1, D_reduced+1); + unprocessed_idx += 1; + unprocessed_dims = hierarchy.unprocessed(unprocessed_idx, unprocessed_n); + DeviceLauncher::Execute( + GpkRevKernel( + shape, shape_c, unprocessed_n, unprocessed_dims, curr_dims[0], + curr_dims[1], curr_dims[2], ratio_r, ratio_c, ratio_f, doutput, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf, 0, 0, 0, + hierarchy.level_shape(l, curr_dims[0]), + hierarchy.level_shape(l, curr_dims[1]), + hierarchy.level_shape(l, curr_dims[2])), + queue_idx); + } else { // D - D_reduced >= 2 + // printf("coeff-restore %u-%dD\n", D_reduced+1, D_reduced+2); + unprocessed_idx += 2; + unprocessed_dims = hierarchy.unprocessed(unprocessed_idx, unprocessed_n); + DeviceLauncher::Execute( + GpkRevKernel( + shape, shape_c, unprocessed_n, unprocessed_dims, curr_dims[0], + curr_dims[1], curr_dims[2], ratio_r, ratio_c, ratio_f, doutput, + dcoarse, dcoeff_f, dcoeff_c, dcoeff_r, dcoeff_cf, dcoeff_rf, + dcoeff_rc, dcoeff_rcf, 0, 0, 0, + hierarchy.level_shape(l, curr_dims[0]), + hierarchy.level_shape(l, curr_dims[1]), + hierarchy.level_shape(l, curr_dims[2])), + queue_idx); + } + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D("After coeff restore", doutput); + } // debug +} + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/GPKFunctor.h b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/GPKFunctor.h new file mode 100644 index 000000000..f32f66107 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/GPKFunctor.h @@ -0,0 +1,29 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_GPK_FUNCTOR +#define MGARD_X_160_GPK_FUNCTOR + +namespace mgard_x_160 { + +template MGARDX_EXEC T lerp(T v0, T v1, T t) { +#ifdef MGARD_X_160_FMA + if (sizeof(T) == sizeof(double)) { + return fma(t, v1, fma(-t, v0, v0)); + } else if (sizeof(T) == sizeof(float)) { + return fmaf(t, v1, fmaf(-t, v0, v0)); + } +#else + T r = v0 + v0 * t * -1; + r = r + t * v1; + return r; +#endif +} + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel.hpp new file mode 100644 index 000000000..4727c8e59 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel.hpp @@ -0,0 +1,4661 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_GRID_PROCESSING_KERNEL_TEMPLATE +#define MGARD_X_160_GRID_PROCESSING_KERNEL_TEMPLATE + +#include "../../../RuntimeX/RuntimeX.h" + +#include "GPKFunctor.h" +// #include "GridProcessingKernel.h" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +class GpkReoFunctor : public Functor { +public: + MGARDX_CONT GpkReoFunctor() {} + MGARDX_CONT GpkReoFunctor( + SubArray<1, SIZE, DeviceType> shape, + SubArray<1, SIZE, DeviceType> shape_c, DIM unprocessed_n, + SubArray<1, DIM, DeviceType> unprocessed_dims, DIM curr_dim_r, + DIM curr_dim_c, DIM curr_dim_f, SubArray<1, T, DeviceType> ratio_r, + SubArray<1, T, DeviceType> ratio_c, SubArray<1, T, DeviceType> ratio_f, + SubArray v, SubArray w, + SubArray wf, + SubArray wc, + SubArray wr, + SubArray wcf, + SubArray wrf, + SubArray wrc, + SubArray wrcf) + : shape(shape), shape_c(shape_c), unprocessed_n(unprocessed_n), + unprocessed_dims(unprocessed_dims), curr_dim_r(curr_dim_r), + curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), ratio_r(ratio_r), + ratio_c(ratio_c), ratio_f(ratio_f), v(v), w(w), wf(wf), wc(wc), wr(wr), + wcf(wcf), wrf(wrf), wrc(wrc), wrcf(wrcf) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + + // bool debug = false; + // if (FunctorBase::GetBlockIdX() == 0 && + // FunctorBase::GetBlockIdY() ==0 && + // FunctorBase::GetBlockIdZ() == 0 && + // FunctorBase::GetThreadIdX() == 0 && + // FunctorBase::GetThreadIdY() == 0 && + // FunctorBase::GetThreadIdZ() == 0) debug = false; + + // volatile clock_t start = 0; + // volatile clock_t end = 0; + // volatile unsigned long long sum_time = 0; + + threadId = (FunctorBase::GetThreadIdZ() * + (FunctorBase::GetBlockDimX() * + FunctorBase::GetBlockDimY())) + + (FunctorBase::GetThreadIdY() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + + in_next = true; + + Byte *sm = FunctorBase::GetSharedMemory(); + SIZE offset = 0; + ldsm1 = (F / 2) * 2 + 1; + ldsm2 = (C / 2) * 2 + 1; + + v_sm = (T *)&sm[offset]; + advance_with_align(offset, (F + 1) * (C + 1) * (R + 1)); + ratio_f_sm = (T *)&sm[offset]; + advance_with_align(offset, F); + ratio_c_sm = (T *)&sm[offset]; + advance_with_align(offset, C); + ratio_r_sm = (T *)&sm[offset]; + advance_with_align(offset, R); + + // switching data type + align_byte_offset(offset); + + shape_sm = (SIZE *)&sm[offset]; + advance_with_align(offset, D_GLOBAL); + shape_c_sm = (SIZE *)&sm[offset]; + advance_with_align(offset, D_GLOBAL); + + // switching data type + align_byte_offset(offset); + + unprocessed_dims_sm = (DIM *)&sm[offset]; + advance_with_align(offset, D_GLOBAL); + + if (threadId < D_GLOBAL) { + shape_sm[threadId] = *shape(threadId); + shape_c_sm[threadId] = *shape_c(threadId); + // ldvs_sm[threadId] = ldvs[threadId]; + // ldws_sm[threadId] = ldws[threadId]; + } + + if (threadId < unprocessed_n) { + unprocessed_dims_sm[threadId] = *unprocessed_dims(threadId); + } + } + + MGARDX_EXEC void Operation2() { + // __syncthreads(); + + for (DIM d = 0; d < D_GLOBAL; d++) + idx[d] = 0; + + nr = curr_dim_r < D_GLOBAL ? shape_sm[curr_dim_r] : 1; + nc = curr_dim_c < D_GLOBAL ? shape_sm[curr_dim_c] : 1; + nf = curr_dim_f < D_GLOBAL ? shape_sm[curr_dim_f] : 1; + + nr_c = curr_dim_r < D_GLOBAL ? shape_c_sm[curr_dim_r] : 1; + nc_c = curr_dim_c < D_GLOBAL ? shape_c_sm[curr_dim_c] : 1; + nf_c = curr_dim_f < D_GLOBAL ? shape_c_sm[curr_dim_f] : 1; + + if (D_LOCAL < 3) { + nr = 1; + nr_c = 1; + } + if (D_LOCAL < 2) { + nc = 1; + nc_c = 1; + } + + r = FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ(); + c = FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY(); + SIZE bidx = FunctorBase::GetBlockIdX(); + SIZE firstD = div_roundup(shape_sm[D_GLOBAL - 1] - 1, + FunctorBase::GetBlockDimX()); + f = (bidx % firstD) * FunctorBase::GetBlockDimX(); + + bidx /= firstD; + + // if (debug) printf("n: %d %d %d rcf: %d %d %d\n", nr, nc, nf, r, c, f); + rest_r = nr - r; + rest_c = nc - c; + rest_f = nf - f; + + nr_p = nr; + nc_p = nc; + nf_p = nf; + + rest_r_p = rest_r; + rest_c_p = rest_c; + rest_f_p = rest_f; + + if (nr % 2 == 0) { + nr_p = nr + 1; + rest_r_p = nr_p - r; + } + if (nc % 2 == 0) { + nc_p = nc + 1; + rest_c_p = nc_p - c; + } + if (nf % 2 == 0) { + nf_p = nf + 1; + rest_f_p = nf_p - f; + } + + for (DIM d = 0; d < D_GLOBAL; d++) { + if (D_LOCAL == 3 && d != curr_dim_r && d != curr_dim_c && + d != curr_dim_f) { + idx[d] = bidx % shape_sm[d]; + bidx /= shape_sm[d]; + if (idx[d] >= shape_c_sm[d]) + in_next = false; + } + if (D_LOCAL == 2 && d != curr_dim_c && d != curr_dim_f) { + idx[d] = bidx % shape_sm[d]; + bidx /= shape_sm[d]; + if (idx[d] >= shape_c_sm[d]) + in_next = false; + } + } + + skip = 0; + // #pragma unroll 1 + for (DIM t = 0; t < D_GLOBAL; t++) { + for (DIM k = 0; k < unprocessed_n; k++) { + if (t == unprocessed_dims_sm[k] && + (shape_sm[t] % 2 == 1 && idx[t] % 2 == 1 || + shape_sm[t] % 2 == 0 && idx[t] % 2 == 1 && + idx[t] != shape_sm[t] - 1)) { + skip = 1; + } + } + } + + v.offset(idx); + w.offset(idx); + wf.offset(idx); + wc.offset(idx); + wr.offset(idx); + wcf.offset(idx); + wrf.offset(idx); + wrc.offset(idx); + wrcf.offset(idx); + + if (TYPE == 2) { + wf = w; + wcf = wc; + wrf = wr; + wrcf = wrc; + } + } + MGARDX_EXEC void Operation3() { + r_sm = FunctorBase::GetThreadIdZ(); + c_sm = FunctorBase::GetThreadIdY(); + f_sm = FunctorBase::GetThreadIdX(); + + r_sm_ex = (R / 2) * 2; + c_sm_ex = (C / 2) * 2; + f_sm_ex = (F / 2) * 2; + + r_gl = r + r_sm; + r_gl_ex = r + (R / 2) * 2; + c_gl = c + c_sm; + c_gl_ex = c + (C / 2) * 2; + f_gl = f + f_sm; + f_gl_ex = f + (F / 2) * 2; + + // __syncthreads(); + // if (r_sm == 0 && c_sm == 0 && f_sm == 0) { + // //printf("setting zeros\n"); + // for (int i = 0; i < (R/2) * 2 + 1; i++) { + // for (int j = 0; j < (C/2) * 2 + 1; j++) { + // for (int k = 0; k < (F/2) * 2 + 1; k++) { + // v_sm[get_idx(ldsm1, ldsm2, i, j, k)] = 0.0; + // } + // } + // } + // //printf("done zeros\n"); + // } + // __syncthreads(); + /* Load v */ + // loading extra rules + // case 1: input = odd (non-padding required) + // case 1.a: block size < rest (need to load extra); + // case 1.b: block size > rest (NO need to load extra); + // case 2: input = even (padding requried) + // case 2.a: block size < rest (need to load extra); + // case 2.b: block size >= rest (NO need to load extra, but need + // padding); + + // Load from dv + if (r_sm < rest_r && c_sm < rest_c && f_sm < rest_f) { + + // load cubic + // asm volatile("membar.cta;"); + // start = clock64(); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *v(r_gl, c_gl, f_gl); + // if (FunctorBase::GetBlockIdX()==0 && + // FunctorBase::GetBlockIdY()==0&&FunctorBase::GetBlockIdZ()==0) + // { + // printf("load (%d %d %d) %f <- %d+(%d %d %d) (ld: %d %d)\n", + // r_sm, c_sm, f_sm, + // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl)], + // other_offset_v+r_gl, c_gl, f_gl, lddv1, lddv2); + // } + if (r_sm == 0) { + if (rest_r > (R / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm)] = + *v(r_gl_ex, c_gl, f_gl); + } + } + if (c_sm == 0) { + if (rest_c > (C / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm)] = + *v(r_gl, c_gl_ex, f_gl); + } + } + if (f_sm == 0) { + if (rest_f > (F / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm_ex)] = + *v(r_gl, c_gl, f_gl_ex); + } + } + if (c_sm == 0 && f_sm == 0) { + if (rest_c > (C / 2) * 2 && rest_f > (F / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm_ex)] = + *v(r_gl, c_gl_ex, f_gl_ex); + } + } + if (r_sm == 0 && f_sm == 0) { + if (rest_r > (R / 2) * 2 && rest_f > (F / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm_ex)] = + *v(r_gl_ex, c_gl, f_gl_ex); + } + } + if (r_sm == 0 && c_sm == 0) { + if (rest_r > (R / 2) * 2 && rest_c > (C / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm)] = + *v(r_gl_ex, c_gl_ex, f_gl); + } + } + if (r_sm == 0 && c_sm == 0 && f_sm == 0) { + if (rest_r > (R / 2) * 2 && rest_c > (C / 2) * 2 && + rest_f > (F / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm_ex)] = + *v(r_gl_ex, c_gl_ex, f_gl_ex); + } + } + } + } + + MGARDX_EXEC void Operation4() { + // __syncthreads(); + + // apply padding is necessary + if (r_sm < rest_r && c_sm < rest_c && f_sm < rest_f) { + + // printf("load main[%d %d %d]:%f --> [%d %d %d] (%d %d %d)\n", r_gl, + // c_gl, f_gl, + // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl)], r_sm, c_sm, f_sm, nr, + // nc, nf); + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[load main] block id %d,%d,%d elapsed %lu\n", + // FunctorBase::GetBlockIdZ(), + // FunctorBase::GetBlockIdY(), + // FunctorBase::GetBlockIdX(), start); start = clock64(); + + // load extra surface + + if (r_sm == 0) { + if (rest_r > (R / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm)] = + // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl)]; + // printf("load-r[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl, f_gl, + // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl)], r_sm_ex, c_sm, + // f_sm); + } else if (nr % 2 == 0) { + // if (r == 16 && c == 0 && f == 0) { + // printf("padding (%d %d %d) %f <- (%f %f %f)\n", rest_r_p - 1, + // c_sm, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)], rest_r + // - 1, c_sm, f_sm); + // padded = true; + // aa = v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)]; + // bb = v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm)]; + // } + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)]; + } + } + + if (c_sm == 0) { + if (rest_c > (C / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm)] = + // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl)]; + // printf("load-c[%d %d %d]:%f --> [%d %d %d]\n", r_gl, c_gl_ex, f_gl, + // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl)], r_sm, c_sm_ex, + // f_sm); + } else if (nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, f_sm)]; + } + } + + if (f_sm == 0) { + if (rest_f > (F / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm_ex)] = + // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl_ex)]; + // printf("load-f[%d %d %d]:%f --> [%d %d %d]\n", r_gl, c_gl, f_gl_ex, + // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl_ex)], r_sm, c_sm, + // f_sm_ex); + } else if (nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f - 1)]; + } + } + + // load extra edges + if (c_sm == 0 && f_sm == 0) { + if (rest_c > (C / 2) * 2 && rest_f > (F / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm_ex)] = + // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)]; + // printf("load-cf[%d %d %d]:%f --> [%d %d %d]\n", r_gl, c_gl_ex, + // f_gl_ex, dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)], r_sm, + // c_sm_ex, f_sm_ex); + } else if (rest_c <= (C / 2) * 2 && rest_f <= (F / 2) * 2 && + nc % 2 == 0 && nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, rest_f - 1)]; + } else if (rest_c > (C / 2) * 2 && rest_f <= (F / 2) * 2 && + nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f - 1)]; + } else if (rest_c <= (C / 2) * 2 && rest_f > (F / 2) * 2 && + nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm_ex)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, f_sm_ex)]; + } + } + + if (r_sm == 0 && f_sm == 0) { + if (rest_r > (R / 2) * 2 && rest_f > (F / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm_ex)] = + // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)]; + // printf("load-rf[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl, + // f_gl_ex, dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)], + // r_sm_ex, c_sm, f_sm_ex); + } else if (rest_r <= (R / 2) * 2 && rest_f <= (F / 2) * 2 && + nr % 2 == 0 && nf % 2 == 0) { + // printf("padding (%d %d %d) <- (%d %d %d)\n", rest_r_p - 1, c_sm, + // rest_f_p - 1, rest_r - 1, c_sm, rest_f - 1); + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, rest_f - 1)]; + } else if (rest_r > (R / 2) * 2 && rest_f <= (F / 2) * 2 && + nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f - 1)]; + } else if (rest_r <= (R / 2) * 2 && rest_f > (F / 2) * 2 && + nr % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm_ex)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm_ex)]; + } + } + + if (r_sm == 0 && c_sm == 0) { + if (rest_r > (R / 2) * 2 && rest_c > (C / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm)] = + // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)]; + // printf("load-rc[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl_ex, + // f_gl, dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)], r_sm_ex, + // c_sm_ex, f_sm); + } else if (rest_r <= (R / 2) * 2 && rest_c <= (C / 2) * 2 && + nr % 2 == 0 && nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm)]; + // printf("padding (%d %d %d) <- (%d %d %d): %f\n", rest_r_p - 1, + // rest_c_p - 1, f_sm, rest_r - 1, rest_c - 1, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm)]); + } else if (rest_r > (R / 2) * 2 && rest_c <= (C / 2) * 2 && + nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c - 1, f_sm)]; + } else if (rest_r <= (R / 2) * 2 && rest_c > (C / 2) * 2 && + nr % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm_ex, f_sm)]; + } + } + // load extra vertex + + if (r_sm == 0 && c_sm == 0 && f_sm == 0) { + if (rest_r > (R / 2) * 2 && rest_c > (C / 2) * 2 && + rest_f > (F / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm_ex)] = + *v(r_gl_ex, c_gl_ex, f_gl_ex); + // printf("load-rcf[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl_ex, + // f_gl_ex, dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)], + // r_sm_ex, c_sm_ex, f_sm_ex); + } else if (rest_r <= (R / 2) * 2 && rest_c <= (C / 2) * 2 && + rest_f <= (F / 2) * 2 && nr % 2 == 0 && nc % 2 == 0 && + nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, + rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, rest_f - 1)]; + } else if (rest_r > (R / 2) * 2 && rest_c > (C / 2) * 2 && + rest_f <= (F / 2) * 2 && nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f - 1)]; + } else if (rest_r > (R / 2) * 2 && rest_c <= (C / 2) * 2 && + rest_f > (F / 2) * 2 && nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm_ex)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c - 1, f_sm_ex)]; + } else if (rest_r > (R / 2) * 2 && rest_c <= (C / 2) * 2 && + rest_f <= (F / 2) * 2 && nc % 2 == 0 && nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c - 1, rest_f - 1)]; + } else if (rest_r <= (R / 2) * 2 && rest_c > (C / 2) * 2 && + rest_f > (F / 2) * 2 && nr % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm_ex)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm_ex, f_sm_ex)]; + } else if (rest_r <= (R / 2) * 2 && rest_c > (C / 2) * 2 && + rest_f <= (F / 2) * 2 && nr % 2 == 0 && nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm_ex, rest_f - 1)]; + } else if (rest_r <= (R / 2) * 2 && rest_c <= (C / 2) * 2 && + rest_f > (F / 2) * 2 && nr % 2 == 0 && nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm_ex)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm_ex)]; + } + } + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[load extra] block id %d,%d,%d elapsed %lu\n", + // FunctorBase::GetBlockIdZ(), + // FunctorBase::GetBlockIdY(), + // FunctorBase::GetBlockIdX(), start); start = clock64(); + + // load dist + if (c_sm == 0 && f_sm == 0 && r_sm < rest_r_p - 2) { + // printf("%d/%d load %f\n", r_sm, rest_r - 2, dratio_r[r + r_sm]); + ratio_r_sm[r_sm] = *ratio_r(r + r_sm); + // padding ratio calculated in Hanlde + // if (nr % 2 == 0 && (R/2) * 2 + 1 >= rest_r_p && r_sm == 0) { + // ratio_r_sm[rest_r_p - 3] = 0.5; + // } + } + if (r_sm == 0 && f_sm == 0 && c_sm < rest_c_p - 2) { + ratio_c_sm[c_sm] = *ratio_c(c + c_sm); + // padding ratio calculated in Hanlde + // if (nc % 2 == 0 && (C/2) * 2 + 1 >= rest_c_p && c_sm == 0) { + // ratio_c_sm[rest_c_p - 3] = 0.5; + // } + } + if (c_sm == 0 && r_sm == 0 && f_sm < rest_f_p - 2) { + ratio_f_sm[f_sm] = *ratio_f(f + f_sm); + // padding ratio calculated in Hanlde + // if (nf % 2 == 0 && (F/2) * 2 + 1 >= rest_f_p && f_sm == 0) { + // ratio_f_sm[rest_f_p - 3] = 0.5; + // } + } + + // if (r == 0 && c == 0 && f == 0 && r_sm == 0 && c_sm == 0 && f_sm == 0) + // { + // printf("ratio:"); + // for (int i = 0; i < (R/2) * 2 + 1; i++) { + // printf("%2.2f ", ratio_r_sm[i]); + // } + // printf("\n"); + // } + + } // restrict boundary + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[load ratio] block id %d,%d,%d elapsed %lu\n", + // FunctorBase::GetBlockIdZ(), + // FunctorBase::GetBlockIdY(), + // FunctorBase::GetBlockIdX(), start); start = clock64(); + + // __syncthreads(); + // // debug print + // if (debug) { + // printf("in config: %d %d %d (%d %d %d)\n", (R/2), (C/2), (F/2), r,c,f); + // printf("rest_p: %d %d %d\n", rest_r_p, rest_c_p, rest_f_p); + // bool print = false; + // for (int i = 0; i < (R/2) * 2 + 1; i++) { + // for (int j = 0; j < (C/2) * 2 + 1; j++) { + // for (int k = 0; k < (F/2) * 2 + 1; k++) { + // // if (abs(v_sm[get_idx(ldsm1, ldsm2, i, j, k)]) > 10000) { + // // print = true; + // // printf("(block %d %d %d) %2.2f \n", r,c,f, + // v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); + // // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); + // // } + // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + } + + MGARDX_EXEC void Operation5() { + // __syncthreads(); + + if (!w.isNull() && threadId < (R / 2) * (C / 2) * (F / 2)) { + r_sm = (threadId / ((C / 2) * (F / 2))) * 2; + c_sm = ((threadId % ((C / 2) * (F / 2))) / (F / 2)) * 2; + f_sm = ((threadId % ((C / 2) * (F / 2))) % (F / 2)) * 2; + r_gl = r / 2 + threadId / ((C / 2) * (F / 2)); + c_gl = c / 2 + threadId % ((C / 2) * (F / 2)) / (F / 2); + f_gl = f / 2 + threadId % ((C / 2) * (F / 2)) % (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], + // r_sm, c_sm, f_sm); + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + ; + } + if (CALC_COEFF) { + if (in_next && f_gl < nf_c) { + ; + } else { + res -= *w(r_gl, c_gl, f_gl); + } + } + } + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: (%d %d %d) <- %f (%d %d %d)\n", + // r_gl, c_gl, f_gl, *w(r_gl, c_gl, f_gl), + // r_sm, c_sm, f_sm); + } + } + } + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[store coarse] block id %d,%d,%d elapsed %lu\n", + // FunctorBase::GetBlockIdZ(), + // FunctorBase::GetBlockIdY(), + // FunctorBase::GetBlockIdX(), start); start = clock64(); + int base = 0; + // printf("TYPE =%d \n", TYPE); + // printf("%d == %d && %llu >= %d && %llu < %d\n", r + (R/2) * 2, nr_p - 1, + // threadId, base, threadId, base + (C/2) * (F/2)); + + if (!w.isNull() && r + (R / 2) * 2 == nr_p - 1 && threadId >= base && + threadId < base + (C / 2) * (F / 2)) { + r_sm = (R / 2) * 2; + c_sm = ((threadId - base) / (F / 2)) * 2; + f_sm = ((threadId - base) % (F / 2)) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (threadId - base) / (F / 2); + f_gl = f / 2 + (threadId - base) % (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], + // r_sm, c_sm, f_sm); + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + ; + } + if (CALC_COEFF) { + if (in_next && f_gl < nf_c) { + ; + } else { + res -= *w(r_gl, c_gl, f_gl); + } + } + } + + // if (idx[1] == 1 && idx[2] == 0) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w(cf)-store: (%d %d %d) <- %f (%d %d %d)\n", + // r_gl, c_gl, f_gl, *w(r_gl, c_gl, f_gl), + // r_sm, c_sm, f_sm); + // } + } + } + } + + base += (C / 2) * (F / 2); // ROUND_UP_WARP((C/2) * (F/2)) * WARP_SIZE; + if (!w.isNull() && c + (C / 2) * 2 == nc_p - 1 && threadId >= base && + threadId < base + (R / 2) * (F / 2)) { + r_sm = ((threadId - base) / (F / 2)) * 2; + c_sm = (C / 2) * 2; + f_sm = ((threadId - base) % (F / 2)) * 2; + r_gl = r / 2 + (threadId - base) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - base) % (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], + // r_sm, c_sm, f_sm); + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + ; + } + if (CALC_COEFF) { + if (in_next && f_gl < nf_c) { + ; + } else { + res -= *w(r_gl, c_gl, f_gl); + } + } + } + *w(r_gl, c_gl, f_gl) = res; + // printf("w(rf)-store: (%d %d %d) <- %f (%d %d %d)\n", + // r_gl, c_gl, f_gl, *w(r_gl, c_gl, f_gl), + // r_sm, c_sm, f_sm); + } + } + // printf("(%d %d %d) (%d %d %d) %f\n", + // r_sm, c_sm, f_sm, r_gl, c_gl, f_gl, dwork[get_idx(lddv1, lddv2, + // r_gl, c_gl, f_gl)]); + } + + base += (R / 2) * (F / 2); // ROUND_UP_WARP((R/2) * (F/2)) * WARP_SIZE; + // printf("%d %d\n", base, threadId); + if (!w.isNull() && f + (F / 2) * 2 == nf_p - 1 && threadId >= base && + threadId < base + (R / 2) * (C / 2)) { + r_sm = ((threadId - base) / (C / 2)) * 2; + c_sm = ((threadId - base) % (C / 2)) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - base) / (C / 2); + c_gl = c / 2 + (threadId - base) % (C / 2); + f_gl = f / 2 + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], + // r_sm, c_sm, f_sm); + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + ; + } + if (CALC_COEFF) { + if (in_next && f_gl < nf_c) { + ; + } else { + res -= *w(r_gl, c_gl, f_gl); + } + } + } + *w(r_gl, c_gl, f_gl) = res; + // printf("w(rc)-store: (%d %d %d) <- %f (%d %d %d)\n", + // r_gl, c_gl, f_gl, *w(r_gl, c_gl, f_gl), + // r_sm, c_sm, f_sm); + } + } + } + + base += (R / 2) * (C / 2); // ROUND_UP_WARP((R/2) * (C/2)) * WARP_SIZE; + // load extra edges + if (!w.isNull() && c + (C / 2) * 2 == nc_p - 1 && + f + (F / 2) * 2 == nf_p - 1 && threadId >= base && + threadId < base + (R / 2)) { + r_sm = (threadId - base) * 2; + c_sm = (C / 2) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + threadId - base; + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], + // r_sm, c_sm, f_sm); + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + ; + } + if (CALC_COEFF) { + if (in_next && f_gl < nf_c) { + ; + } else { + res -= *w(r_gl, c_gl, f_gl); + } + } + } + *w(r_gl, c_gl, f_gl) = res; + // printf("w(r)-store: (%d %d %d) <- %f (%d %d %d)\n", + // r_gl, c_gl, f_gl, *w(r_gl, c_gl, f_gl), + // r_sm, c_sm, f_sm); + } + } + } + + base += (R / 2); // ROUND_UP_WARP((R/2)) * WARP_SIZE; + // if (TYPE == 2) printf("%d %d, %d, %llu, %d\n",dw == NULL, f + (F/2) * 2, + // nf_p + // - 1, threadId, (C/2)); + if (!w.isNull() && r + (R / 2) * 2 == nr_p - 1 && + f + (F / 2) * 2 == nf_p - 1 && threadId >= base && + threadId < base + (C / 2)) { + r_sm = (R / 2) * 2; + c_sm = (threadId - base) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId - base; + f_gl = f / 2 + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], + // r_sm, c_sm, f_sm); + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + ; + } + if (CALC_COEFF) { + if (in_next && f_gl < nf_c) { + ; + } else { + res -= *w(r_gl, c_gl, f_gl); + } + } + } + *w(r_gl, c_gl, f_gl) = res; + // printf("w(c)-store: (%d %d %d) <- %f (%d %d %d)\n", + // r_gl, c_gl, f_gl, *w(r_gl, c_gl, f_gl), + // r_sm, c_sm, f_sm); + } + } + // printf("store[%d %d %d]: %f\n", r_sm, c_sm, f_sm, v_sm[get_idx(ldsm1, + // ldsm2, r_sm, c_sm, f_sm)]); + } + + base += (C / 2); // ROUND_UP_WARP((C/2)) * WARP_SIZE; + if (!w.isNull() && r + (R / 2) * 2 == nr_p - 1 && + c + (C / 2) * 2 == nc_p - 1 && threadId >= base && + threadId < base + (F / 2)) { + r_sm = (R / 2) * 2; + c_sm = (C / 2) * 2; + f_sm = (threadId - base) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + threadId - base; + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], + // r_sm, c_sm, f_sm); + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + ; + } + if (CALC_COEFF) { + if (in_next && f_gl < nf_c) { + ; + } else { + res -= *w(r_gl, c_gl, f_gl); + } + } + } + *w(r_gl, c_gl, f_gl) = res; + // printf("w(f)-store: (%d %d %d) <- %f (%d %d %d)\n", + // r_gl, c_gl, f_gl, *w(r_gl, c_gl, f_gl), + // r_sm, c_sm, f_sm); + } + } + } + base += (F / 2); // ROUND_UP_WARP((F/2)) * WARP_SIZE; + // // load extra vertex + if (!w.isNull() && r + (R / 2) * 2 == nr_p - 1 && + c + (C / 2) * 2 == nc_p - 1 && f + (F / 2) * 2 == nf_p - 1 && + threadId >= base && threadId < base + 1) { + r_sm = (R / 2) * 2; + c_sm = (C / 2) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl)], + // r_sm, c_sm, f_sm); + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + ; + } + if (CALC_COEFF) { + if (in_next && f_gl < nf_c) { + ; + } else { + res -= *w(r_gl, c_gl, f_gl); + } + } + } + *w(r_gl, c_gl, f_gl) = res; + // printf("w(1)-store: (%d %d %d) <- %f (%d %d %d)\n", + // r_gl, c_gl, f_gl, *w(r_gl, c_gl, f_gl), + // r_sm, c_sm, f_sm); + } + } + } + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[store extra] block id %d,%d,%d elapsed %lu\n", + // FunctorBase::GetBlockIdZ(), + // FunctorBase::GetBlockIdY(), + // FunctorBase::GetBlockIdX(), start); start = clock64(); + + // start = clock64(); + + // printf("wf.isNull(): %d\n", wf.isNull()); + if (!wf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) && + threadId < (R / 2) * (C / 2) * (F / 2) * 2) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2)) / ((C / 2) * (F / 2))) * 2; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) / + (F / 2)) * + 2; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2)) / ((C / 2) * (F / 2)); + c_gl = c / 2 + + ((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + + ((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) % + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + // printf("TYPE: %d\n", TYPE); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + // printf("skip: %d\n", skip); + if (!skip) { + // printf("INTERPOLATION: %d\n", INTERPOLATION); + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + // printf("fw: lerp(%f, %f) -> %f\n", v_sm[get_idx(ldsm1, ldsm2, + // r_sm, c_sm, f_sm - 1)], + // v_sm[get_idx(ldsm1, ldsm2, + // r_sm, c_sm, f_sm + 1)], + // res); + } + if (INTERPOLATION && CALC_COEFF) { // fused + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { // calc_coeff only + res -= *wf(r_gl, c_gl, f_gl); + } + } + *wf(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + ; + } + if (CALC_COEFF) { + if (in_next && f_gl < nf_c) { + ; + } else { + res -= *wf(r_gl, c_gl, f_gl); + } + } + } + *wf(r_gl, c_gl, f_gl) = res; + } + } + + // if (nr == 70) printf("f-store: (%d %d %d) <- %f (%d %d %d)\n", r_gl, + // c_gl, f_gl, v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], r_sm, c_sm, + // f_sm); + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[(F/2)-store] block id %d,%d,%d elapsed %lu\n", + // FunctorBase::GetBlockIdZ(), + // FunctorBase::GetBlockIdY(), + // FunctorBase::GetBlockIdX(), start); start = clock64(); + } + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[(F/2)-store] block id %d,%d,%d elapsed %lu\n", + // FunctorBase::GetBlockIdZ(), + // FunctorBase::GetBlockIdY(), + // FunctorBase::GetBlockIdX(), start); start = clock64(); + + // if (r_sm % 2 == 0 && c_sm % 2 != 0 && f_sm % 2 == 0) { + + if (!wc.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 2 && + threadId < (R / 2) * (C / 2) * (F / 2) * 3) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / ((C / 2) * (F / 2))) * + 2; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) % + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wc(r_gl, c_gl, f_gl); + } + } + *wc(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + if (CALC_COEFF) { // no need to test in_next + res -= *wc(r_gl, c_gl, f_gl); + } + } + *wc(r_gl, c_gl, f_gl) = res; + } + // if (nr == 70) printf("c-store: (%d %d %d) <- %f (%d %d %d)\n", r_gl, + // c_gl, f_gl, v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], r_sm, + // c_sm, f_sm); + } + } + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[(C/2)-store] block id %d,%d,%d elapsed %lu\n", + // FunctorBase::GetBlockIdZ(), + // FunctorBase::GetBlockIdY(), + // FunctorBase::GetBlockIdX(), start); start = clock64(); + + // if (r_sm % 2 != 0 && c_sm % 2 == 0 && f_sm % 2 == 0) { + if (!wr.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 3 && + threadId < (R / 2) * (C / 2) * (F / 2) * 4) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 3) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 3) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) % + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wr(r_gl, c_gl, f_gl); + } + } + *wr(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + if (CALC_COEFF) { // no need to test if in_next + res -= *wr(r_gl, c_gl, f_gl); + } + } + *wr(r_gl, c_gl, f_gl) = res; + } + } + } + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[(R/2)-store] block id %d,%d,%d elapsed %lu\n", + // FunctorBase::GetBlockIdZ(), + // FunctorBase::GetBlockIdY(), + // FunctorBase::GetBlockIdX(), start); start = clock64(); + } + + MGARDX_EXEC void Operation6() { + // __syncthreads(); + if (!wcf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 4 && + threadId < (R / 2) * (C / 2) * (F / 2) * 5) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 4) / ((C / 2) * (F / 2))) * + 2; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 4) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) % + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { + if (!skip) { + if (INTERPOLATION) { + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T tmp = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + res = lerp(f1, f2, ratio_c_sm[c_sm - 1]); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wcf(r_gl, c_gl, f_gl); + } + } + *wcf(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + if (CALC_COEFF) { // not need to test if in_next + res -= *wcf(r_gl, c_gl, f_gl); + } + } + *wcf(r_gl, c_gl, f_gl) = res; + } + } + } + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[CF-store] block id %d,%d,%d elapsed %lu\n", + // FunctorBase::GetBlockIdZ(), + // FunctorBase::GetBlockIdY(), + // FunctorBase::GetBlockIdX(), start); start = clock64(); + + if (!wrf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 5 && + threadId < (R / 2) * (C / 2) * (F / 2) * 6) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 5) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 5) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) % + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + if (!skip) { + if (INTERPOLATION) { + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res = lerp(f1, f2, ratio_r_sm[r_sm - 1]); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wrf(r_gl, c_gl, f_gl); + } + } + *wrf(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + if (CALC_COEFF) { // no need to test if in_next + res -= *wrf(r_gl, c_gl, f_gl); + } + } + *wrf(r_gl, c_gl, f_gl) = res; + } + } + } + + if (!wrc.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 6 && + threadId < (R / 2) * (C / 2) * (F / 2) * 7) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 6) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 6) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) % + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + if (!skip) { + if (INTERPOLATION) { + T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wrc(r_gl, c_gl, f_gl); + } + } + *wrc(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); + } + if (CALC_COEFF) { // no need to test if in_next + res -= *wrc(r_gl, c_gl, f_gl); + } + } + *wrc(r_gl, c_gl, f_gl) = res; + } + } + } + + if (!wrcf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 7 && + threadId < (R / 2) * (C / 2) * (F / 2) * 8) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 7) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 7) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) % + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { + if (!skip) { + if (INTERPOLATION) { + T f1 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f3 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f4 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + + T fc1 = lerp(f1, f2, ratio_c_sm[c_sm - 1]); + T fc2 = lerp(f3, f4, ratio_c_sm[c_sm - 1]); + + res = lerp(fc1, fc2, ratio_r_sm[r_sm - 1]); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wrcf(r_gl, c_gl, f_gl); + } + } + *wrcf(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); + } + if (CALC_COEFF) { // no need to test if in_next + res -= *wrcf(r_gl, c_gl, f_gl); + } + } + *wrcf(r_gl, c_gl, f_gl) = res; + } + } + } + // end = clock64(); + + // asm volatile("membar.cta;"); + // if (threadId < 256 && FunctorBase::GetBlockIdZ() == 0 && + // FunctorBase::GetBlockIdY() == 0 && + // FunctorBase::GetBlockIdX() == 0) printf("threadId %d elapsed + // %lu\n", threadId, end-start); + if (r + (R / 2) * 2 == nr_p - 1) { + // printf("test\n"); + if (threadId < (C / 2) * (F / 2)) { + // printf("test1\n"); + if (!wf.isNull()) { + // printf("test2\n"); + r_sm = (R / 2) * 2; + c_sm = (threadId / (F / 2)) * 2; + f_sm = (threadId % (F / 2)) * 2 + 1; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId / (F / 2); + f_gl = f / 2 + threadId % (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + // printf("test3\n"); + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + // printf("fw: lerp(%f, %f) -> %f\n", v_sm[get_idx(ldsm1, + // ldsm2, r_sm, c_sm, f_sm - 1)], + // v_sm[get_idx(ldsm1, ldsm2, + // r_sm, c_sm, f_sm + 1)], + // res); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wf(r_gl, c_gl, f_gl); + } + } + // printf("dwf (%d %d %d): %f\n", r_gl, c_gl, f_gl, res); + *wf(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + ; + } + if (CALC_COEFF) { // need to test if in_next + if (in_next && f_gl < nf_c) { + ; + } // in_next + else { + res -= *wf(r_gl, c_gl, f_gl); + } + } + } + *wf(r_gl, c_gl, f_gl) = res; + // printf("wf (%d %d %d): %f\n", r_gl, c_gl, f_gl, res); + } + } + } + + if (!wc.isNull()) { + r_sm = (R / 2) * 2; + c_sm = (threadId / (F / 2)) * 2 + 1; + f_sm = (threadId % (F / 2)) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId / (F / 2); + f_gl = f / 2 + threadId % (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wc(r_gl, c_gl, f_gl); + } + } + *wc(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + // if (idx[1]==0 && idx[2]==0) { + // printf("wc-lerp: %f %f (%f) -> %f\n", + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + // ratio_c_sm[c_sm - 1], + // res); + // } + } + if (CALC_COEFF) { // no need to test if in_next + res -= *wc(r_gl, c_gl, f_gl); + } + } + *wc(r_gl, c_gl, f_gl) = res; + + // if (idx[1]==0 && idx[2]==0) { + // printf("wc-store: (%d %d %d) <- %f (%d %d %d)\n", + // r_gl, c_gl, f_gl, *wc(r_gl, c_gl, f_gl), + // r_sm, c_sm, f_sm); + // } + } + } + } + + if (!wcf.isNull()) { + r_sm = (R / 2) * 2; + c_sm = (threadId / (F / 2)) * 2 + 1; + f_sm = (threadId % (F / 2)) * 2 + 1; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId / (F / 2); + f_gl = f / 2 + threadId % (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { + if (!skip) { + if (INTERPOLATION) { + T f1 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res = lerp(f1, f2, ratio_c_sm[c_sm - 1]); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wcf(r_gl, c_gl, f_gl); + } + } + *wcf(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + if (CALC_COEFF) { + res -= *wcf(r_gl, c_gl, f_gl); + } + } + *wcf(r_gl, c_gl, f_gl) = res; + } + } + } + } + } + + if (c + (C / 2) * 2 == nc_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) && + threadId < (R / 2) * (C / 2) * (F / 2) + (R / 2) * (F / 2)) { + if (!wf.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2)) * 2; + c_sm = (C / 2) * 2; + f_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2)) * 2 + 1; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + // printf("fw: lerp(%f, %f) -> %f\n", v_sm[get_idx(ldsm1, + // ldsm2, r_sm, c_sm, f_sm - 1)], + // v_sm[get_idx(ldsm1, ldsm2, + // r_sm, c_sm, f_sm + 1)], + // res); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wf(r_gl, c_gl, f_gl); + } + } + *wf(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + ; + } + if (CALC_COEFF) { // need to test if in_next + if (in_next && f_gl < nf_c) { + ; + } // in_next + else { + res -= *wf(r_gl, c_gl, f_gl); + } + } + } + *wf(r_gl, c_gl, f_gl) = res; + } + } + } + + if (!wr.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2)) * 2 + 1; + c_sm = (C / 2) * 2; + f_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2)) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wr(r_gl, c_gl, f_gl); + } + } + *wr(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + if (CALC_COEFF) { + res -= *wr(r_gl, c_gl, f_gl); + } + } + *wr(r_gl, c_gl, f_gl) = res; + } + } + } + + if (!wrf.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2)) * 2 + 1; + c_sm = (C / 2) * 2; + f_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2)) * 2 + 1; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + if (!skip) { + if (INTERPOLATION) { + T f1 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res = lerp(f1, f2, ratio_r_sm[r_sm - 1]); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wrf(r_gl, c_gl, f_gl); + } + } + *wrf(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + if (CALC_COEFF) { // no need to test if in_next + res -= *wrf(r_gl, c_gl, f_gl); + } + } + *wrf(r_gl, c_gl, f_gl) = res; + } + } + } + } + } + + if (f + (F / 2) * 2 == nf_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 2 && + threadId < (R / 2) * (C / 2) * (F / 2) * 2 + (R / 2) * (C / 2)) { + if (!wc.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2)) * 2; + c_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2)) * 2 + 1; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2); + c_gl = c / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2); + f_gl = f / 2 + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wc(r_gl, c_gl, f_gl); + } + } + *wc(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + if (CALC_COEFF) { + res -= *wc(r_gl, c_gl, f_gl); + } + } + *wc(r_gl, c_gl, f_gl) = res; + } + } + } + + if (!wr.isNull()) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2)) * 2 + 1; + c_sm = ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2)) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2); + c_gl = c / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2); + f_gl = f / 2 + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wr(r_gl, c_gl, f_gl); + } + } + *wr(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + if (CALC_COEFF) { + res -= *wr(r_gl, c_gl, f_gl); + } + } + *wr(r_gl, c_gl, f_gl) = res; + } + } + } + + if (!wrc.isNull()) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2)) * 2 + 1; + c_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2)) * 2 + 1; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2); + c_gl = c / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2); + f_gl = f / 2 + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + if (!skip) { + if (INTERPOLATION) { + T c1 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wrc(r_gl, c_gl, f_gl); + } + } + *wrc(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + T c1 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); + } + if (CALC_COEFF) { + res -= *wrc(r_gl, c_gl, f_gl); + } + } + *wrc(r_gl, c_gl, f_gl) = res; + } + } + } + } + } + + if (!wr.isNull() && c + (C / 2) * 2 == nc_p - 1 && + f + (F / 2) * 2 == nf_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 3 && + threadId < (R / 2) * (C / 2) * (F / 2) * 3 + (R / 2)) { + r_sm = (threadId - (R / 2) * (C / 2) * (F / 2) * 3) * 2 + 1; + c_sm = (C / 2) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + threadId - (R / 2) * (C / 2) * (F / 2) * 3; + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wr(r_gl, c_gl, f_gl); + } + } + *wr(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + if (CALC_COEFF) { + res -= *wr(r_gl, c_gl, f_gl); + } + } + *wr(r_gl, c_gl, f_gl) = res; + } + } + } + } + + if (!wc.isNull() && r + (R / 2) * 2 == nr_p - 1 && + f + (F / 2) * 2 == nf_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 4 && + threadId < (R / 2) * (C / 2) * (F / 2) * 4 + (C / 2)) { + r_sm = (R / 2) * 2; + c_sm = (threadId - (R / 2) * (C / 2) * (F / 2) * 4) * 2 + 1; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId - (R / 2) * (C / 2) * (F / 2) * 4; + f_gl = f / 2 + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wc(r_gl, c_gl, f_gl); + } + } + *wc(r_gl, c_gl, f_gl) = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + if (CALC_COEFF) { + res -= *wc(r_gl, c_gl, f_gl); + } + } + *wc(r_gl, c_gl, f_gl) = res; + } + } + } + } + + // printf("test1\n"); + if (!wf.isNull() && r + (R / 2) * 2 == nr_p - 1 && + c + (C / 2) * 2 == nc_p - 1) { + // printf("test2\n"); + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 5 && + threadId < (R / 2) * (C / 2) * (F / 2) * 5 + (F / 2)) { + // printf("test3\n"); + r_sm = (R / 2) * 2; + c_sm = (C / 2) * 2; + f_sm = (threadId - (R / 2) * (C / 2) * (F / 2) * 5) * 2 + 1; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + threadId - (R / 2) * (C / 2) * (F / 2) * 5; + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + // printf("test4\n"); + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + // printf("fw: lerp(%f, %f) -> %f\n", v_sm[get_idx(ldsm1, ldsm2, + // r_sm, c_sm, f_sm - 1)], + // v_sm[get_idx(ldsm1, ldsm2, + // r_sm, c_sm, f_sm + 1)], + // res); + } + if (INTERPOLATION && CALC_COEFF) { + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + } + if (!INTERPOLATION && CALC_COEFF) { + res -= *wf(r_gl, c_gl, f_gl); + } + } + *wf(r_gl, c_gl, f_gl) = res; + // printf("dwf(%d %d %d): %f\n", r_gl, c_gl, f_gl, + // dwf[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]); + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + if (!skip) { + if (INTERPOLATION) { + ; + } + if (CALC_COEFF) { // do need to test in_next + if (in_next && f_gl < nf_c) { + ; + } // in_next + else { + res -= *wf(r_gl, c_gl, f_gl); + } + } + } + *wf(r_gl, c_gl, f_gl) = res; + } + } + } + } + // if (r == 0 && c == 0 && f == 0 && threadId == 0) { + // printf("out config: %d %d %d (%d %d %d)\n", (R/2), (C/2), (F/2), + // r,c,f); for (int i = 0; i < (R/2) * 2 + 1; i++) { + // for (int j = 0; j < (C/2) * 2 + 1; j++) { + // for (int k = 0; k < (F/2) * 2 + 1; k++) { + // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + + v.reset_offset(); + w.reset_offset(); + wf.reset_offset(); + wc.reset_offset(); + wr.reset_offset(); + wcf.reset_offset(); + wrf.reset_offset(); + wrc.reset_offset(); + wrcf.reset_offset(); + } + + MGARDX_CONT SIZE shared_memory_size() { + SIZE size = 0; + size = ((R + 1) * (C + 1) * (F + 1) + R + C + F) * sizeof(T); + align_byte_offset(size); + size += (D_GLOBAL * 4) * sizeof(SIZE); + align_byte_offset(size); + size += (D_GLOBAL * 1) * sizeof(DIM); + return size; + } + +private: + SubArray<1, SIZE, DeviceType> shape, shape_c; + SubArray<1, T, DeviceType> ratio_r, ratio_c, ratio_f; + SubArray v, w, wf, wc, wr, wcf, wrf, wrc, wrcf; + // SIZE *ldvs; + // SIZE *ldws; + DIM unprocessed_n; + SubArray<1, DIM, DeviceType> unprocessed_dims; + SIZE threadId; + T *v_sm; + T *ratio_f_sm; + T *ratio_c_sm; + T *ratio_r_sm; + DIM curr_dim_r; + DIM curr_dim_c; + DIM curr_dim_f; + + SIZE nr, nc, nf; + SIZE nr_c, nc_c, nf_c; + SIZE r, c, f; + SIZE rest_r, rest_c, rest_f; + SIZE nr_p, nc_p, nf_p; + SIZE rest_r_p, rest_c_p, rest_f_p; + SIZE r_sm, c_sm, f_sm; + SIZE r_sm_ex, c_sm_ex, f_sm_ex; + SIZE r_gl, c_gl, f_gl; + SIZE r_gl_ex, c_gl_ex, f_gl_ex; + T res; + bool in_next; + + SIZE ldsm1, ldsm2; + + SIZE *sm_size; + SIZE *shape_sm; + SIZE *shape_c_sm; + SIZE *lvs_sm; + SIZE *ldws_sm; + + DIM *sm_dim; + DIM *unprocessed_dims_sm; + + SIZE idx[D_GLOBAL]; + + int skip; +}; + +template +class GpkReoKernel : public Kernel { +public: + constexpr static DIM NumDim = D_LOCAL; + using DataType = T; + constexpr static std::string_view Name = "gpk_reo_nd"; + MGARDX_CONT + GpkReoKernel(SubArray<1, SIZE, DeviceType> shape, + SubArray<1, SIZE, DeviceType> shape_c, DIM unprocessed_n, + SubArray<1, DIM, DeviceType> unprocessed_dims, DIM curr_dim_r, + DIM curr_dim_c, DIM curr_dim_f, + SubArray<1, T, DeviceType> ratio_r, + SubArray<1, T, DeviceType> ratio_c, + SubArray<1, T, DeviceType> ratio_f, + SubArray v, + SubArray w, + SubArray wf, + SubArray wc, + SubArray wr, + SubArray wcf, + SubArray wrf, + SubArray wrc, + SubArray wrcf) + : shape(shape), shape_c(shape_c), unprocessed_n(unprocessed_n), + unprocessed_dims(unprocessed_dims), curr_dim_r(curr_dim_r), + curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), ratio_r(ratio_r), + ratio_c(ratio_c), ratio_f(ratio_f), v(v), w(w), wf(wf), wc(wc), wr(wr), + wcf(wcf), wrf(wrf), wrc(wrc), wrcf(wrcf) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + + using FunctorType = + GpkReoFunctor; + FunctorType functor(shape, shape_c, unprocessed_n, unprocessed_dims, + curr_dim_r, curr_dim_c, curr_dim_f, ratio_r, ratio_c, + ratio_f, v, w, wf, wc, wr, wcf, wrf, wrc, wrcf); + + SIZE nr = curr_dim_r < D_GLOBAL ? shape.dataHost()[curr_dim_r] : 1; + SIZE nc = curr_dim_c < D_GLOBAL ? shape.dataHost()[curr_dim_c] : 1; + SIZE nf = curr_dim_f < D_GLOBAL ? shape.dataHost()[curr_dim_f] : 1; + if (D_LOCAL == 2) { + nr = 1; + } + SIZE total_thread_z = std::max(nr - 1, (SIZE)1); + SIZE total_thread_y = std::max(nc - 1, (SIZE)1); + SIZE total_thread_x = std::max(nf - 1, (SIZE)1); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + // printf("sm_size: %llu\n", sm_size); + // printf("RCF: %u %u %u\n", R, C, F); + tbz = R; + tby = C; + tbx = F; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + for (DIM d = 0; d < D_GLOBAL; d++) { + if (D_LOCAL == 3 && d != curr_dim_f && d != curr_dim_c && + d != curr_dim_r) { + gridx *= shape.dataHost()[d]; + } + if (D_LOCAL == 2 && d != curr_dim_f && d != curr_dim_c) { + gridx *= shape.dataHost()[d]; + } + } + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, SIZE, DeviceType> shape, shape_c; + SubArray<1, T, DeviceType> ratio_r, ratio_c, ratio_f; + SubArray v, w, wf, wc, wr, wcf, wrf, wrc, wrcf; + DIM unprocessed_n; + SubArray<1, DIM, DeviceType> unprocessed_dims; + DIM curr_dim_r; + DIM curr_dim_c; + DIM curr_dim_f; +}; + +template +class GpkRevFunctor : public Functor { +public: + MGARDX_CONT GpkRevFunctor() {} + + MGARDX_CONT GpkRevFunctor( + SubArray<1, SIZE, DeviceType> shape, + SubArray<1, SIZE, DeviceType> shape_c, DIM unprocessed_n, + SubArray<1, DIM, DeviceType> unprocessed_dims, DIM curr_dim_r, + DIM curr_dim_c, DIM curr_dim_f, SubArray<1, T, DeviceType> ratio_r, + SubArray<1, T, DeviceType> ratio_c, SubArray<1, T, DeviceType> ratio_f, + SubArray v, SubArray w, + SubArray wf, + SubArray wc, + SubArray wr, + SubArray wcf, + SubArray wrf, + SubArray wrc, + SubArray wrcf, SIZE svr, SIZE svc, SIZE svf, + SIZE nvr, SIZE nvc, SIZE nvf) + : shape(shape), shape_c(shape_c), unprocessed_n(unprocessed_n), + unprocessed_dims(unprocessed_dims), curr_dim_r(curr_dim_r), + curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), ratio_r(ratio_r), + ratio_c(ratio_c), ratio_f(ratio_f), v(v), w(w), wf(wf), wc(wc), wr(wr), + wcf(wcf), wrf(wrf), wrc(wrc), wrcf(wrcf), svr(svr), svc(svc), svf(svf), + nvr(nvr), nvc(nvc), nvf(nvf) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + + threadId = (FunctorBase::GetThreadIdZ() * + (FunctorBase::GetBlockDimX() * + FunctorBase::GetBlockDimY())) + + (FunctorBase::GetThreadIdY() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + + nr, nc, nf; + nr_c, nc_c, nf_c; + r, c, f; + rest_r, rest_c, rest_f; + nr_p, nc_p, nf_p; + rest_r_p, rest_c_p, rest_f_p; + r_sm, c_sm, f_sm; + r_sm_ex, c_sm_ex, f_sm_ex; + r_gl, c_gl, f_gl; + r_gl_ex, c_gl_ex, f_gl_ex; + in_next = true; + + Byte *sm = FunctorBase::GetSharedMemory(); + SIZE offset = 0; + ldsm1 = (F / 2) * 2 + 1; + ldsm2 = (C / 2) * 2 + 1; + + v_sm = (T *)&sm[offset]; + advance_with_align(offset, (F + 1) * (C + 1) * (R + 1)); + ratio_f_sm = (T *)&sm[offset]; + advance_with_align(offset, F); + ratio_c_sm = (T *)&sm[offset]; + advance_with_align(offset, C); + ratio_r_sm = (T *)&sm[offset]; + advance_with_align(offset, R); + + // switching data type + align_byte_offset(offset); + + shape_sm = (SIZE *)&sm[offset]; + advance_with_align(offset, D_GLOBAL); + shape_c_sm = (SIZE *)&sm[offset]; + advance_with_align(offset, D_GLOBAL); + + // switching data type + align_byte_offset(offset); + + unprocessed_dims_sm = (DIM *)&sm[offset]; + advance_with_align(offset, D_GLOBAL); + + // SIZE idx[D_GLOBAL]; + if (threadId < D_GLOBAL) { + shape_sm[threadId] = *shape(threadId); + shape_c_sm[threadId] = *shape_c(threadId); + // ldvs_sm[threadId] = ldvs[threadId]; + // ldws_sm[threadId] = ldws[threadId]; + } + + if (threadId < unprocessed_n) { + unprocessed_dims_sm[threadId] = *unprocessed_dims(threadId); + } + } + + MGARDX_EXEC void Operation2() { + // __syncthreads(); + for (DIM d = 0; d < D_GLOBAL; d++) + idx[d] = 0; + + nr = curr_dim_r < D_GLOBAL ? shape_sm[curr_dim_r] : 1; + nc = curr_dim_c < D_GLOBAL ? shape_sm[curr_dim_c] : 1; + nf = curr_dim_f < D_GLOBAL ? shape_sm[curr_dim_f] : 1; + + nr_c = curr_dim_r < D_GLOBAL ? shape_c_sm[curr_dim_r] : 1; + nc_c = curr_dim_c < D_GLOBAL ? shape_c_sm[curr_dim_c] : 1; + nf_c = curr_dim_f < D_GLOBAL ? shape_c_sm[curr_dim_f] : 1; + + if (D_LOCAL < 3) { + nr = 1; + nr_c = 1; + } + if (D_LOCAL < 2) { + nc = 1; + nc_c = 1; + } + + r = FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ(); + c = FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY(); + SIZE bidx = FunctorBase::GetBlockIdX(); + SIZE firstD = div_roundup(shape_sm[D_GLOBAL - 1] - 1, + FunctorBase::GetBlockDimX()); + f = (bidx % firstD) * FunctorBase::GetBlockDimX(); + + bidx /= firstD; + + rest_r = nr - r; + rest_c = nc - c; + rest_f = nf - f; + + nr_p = nr; + nc_p = nc; + nf_p = nf; + + rest_r_p = rest_r; + rest_c_p = rest_c; + rest_f_p = rest_f; + + if (nr % 2 == 0) { + nr_p = nr + 1; + rest_r_p = nr_p - r; + } + if (nc % 2 == 0) { + nc_p = nc + 1; + rest_c_p = nc_p - c; + } + if (nf % 2 == 0) { + nf_p = nf + 1; + rest_f_p = nf_p - f; + } + + for (int d = 0; d < D_GLOBAL; d++) { + if (D_LOCAL == 3 && d != curr_dim_r && d != curr_dim_c && + d != curr_dim_f) { + idx[d] = bidx % shape_sm[d]; + bidx /= shape_sm[d]; + if ((shape_sm[d] % 2 == 1 && idx[d] % 2 != 0) || + shape_sm[d] % 2 == 0 && + (idx[d] % 2 != 0 && idx[d] != shape_sm[d] - 1)) + in_next = false; + } + if (D_LOCAL == 2 && d != curr_dim_c && d != curr_dim_f) { + idx[d] = bidx % shape_sm[d]; + bidx /= shape_sm[d]; + if ((shape_sm[d] % 2 == 1 && idx[d] % 2 != 0) || + shape_sm[d] % 2 == 0 && + (idx[d] % 2 != 0 && idx[d] != shape_sm[d] - 1)) + in_next = false; + } + } + + skip = 0; + // #pragma unroll 1 + for (DIM t = 0; t < D_GLOBAL; t++) { + for (DIM k = 0; k < unprocessed_n; k++) { + if (t == unprocessed_dims_sm[k] && idx[t] >= shape_c_sm[t]) { + skip = 1; + } + } + } + + // if (FunctorBase::GetBlockIdX() == 0 && + // FunctorBase::GetBlockIdY() == 0 && + // FunctorBase::GetBlockIdZ() == 0) { if + // (FunctorBase::GetThreadIdX() == 0 && + // FunctorBase::GetThreadIdY() == 0 && + // FunctorBase::GetThreadIdZ() == 0) { + // printf("TYPE %d total_idx_sm: %d %d %d %d (skip: %d)\n", TYPE, idx[3], + // idx[2], idx[1], idx[0], skip); + // } + // } + + // SIZE other_offset_v = get_idx(ldvs_sm, idx); + // SIZE other_offset_w = get_idx(ldws_sm, idx); + + // dv = dv + other_offset_v; + // dw = dw + other_offset_w; + // dwr = dwr + other_offset_w; + // dwc = dwc + other_offset_w; + // dwf = dwf + other_offset_w; + // dwrf = dwrf + other_offset_w; + // dwrc = dwrc + other_offset_w; + // dwcf = dwcf + other_offset_w; + // dwrcf = dwrcf + other_offset_w; + + v.offset(idx); + w.offset(idx); + wf.offset(idx); + wc.offset(idx); + wr.offset(idx); + wcf.offset(idx); + wrf.offset(idx); + wrc.offset(idx); + wrcf.offset(idx); + + if (TYPE == 2) { + wf = w; + wcf = wc; + wrf = wr; + wrcf = wrc; + } + } + + MGARDX_EXEC void Operation3() { + // __syncthreads(); + + r_sm = FunctorBase::GetThreadIdZ(); + c_sm = FunctorBase::GetThreadIdY(); + f_sm = FunctorBase::GetThreadIdX(); + + r_sm_ex = (R / 2) * 2; + c_sm_ex = (C / 2) * 2; + f_sm_ex = (F / 2) * 2; + + r_gl = r + r_sm; + r_gl_ex = r + (R / 2) * 2; + c_gl = c + c_sm; + c_gl_ex = c + (C / 2) * 2; + f_gl = f + f_sm; + f_gl_ex = f + (F / 2) * 2; + + // load dist + if (c_sm == 0 && f_sm == 0 && r_sm < rest_r - 2) { + ratio_r_sm[r_sm] = *ratio_r(r + r_sm); + if (nr % 2 == 0 && (R / 2) * 2 + 1 >= rest_r_p && r_sm == 0) { + ratio_r_sm[rest_r_p - 3] = 0.5; + } + } + if (r_sm == 0 && f_sm == 0 && c_sm < rest_c - 2) { + ratio_c_sm[c_sm] = *ratio_c(c + c_sm); + if (nc % 2 == 0 && (C / 2) * 2 + 1 >= rest_c_p && c_sm == 0) { + ratio_c_sm[rest_c_p - 3] = 0.5; + } + } + if (c_sm == 0 && r_sm == 0 && f_sm < rest_f - 2) { + ratio_f_sm[f_sm] = *ratio_f(f + f_sm); + if (nf % 2 == 0 && (F / 2) * 2 + 1 >= rest_f_p && f_sm == 0) { + ratio_f_sm[rest_f_p - 3] = 0.5; + } + } + + if (FunctorBase::GetThreadIdZ() == 0 && + FunctorBase::GetThreadIdY() == 0 && + FunctorBase::GetThreadIdX() == 0) { + for (int i = 0; i < (R / 2) * 2 + 1; i++) { + for (int j = 0; j < (C / 2) * 2 + 1; j++) { + for (int k = 0; k < (F / 2) * 2 + 1; k++) { + v_sm[get_idx(ldsm1, ldsm2, i, j, k)] = 0.0; + } + } + } + } + } + MGARDX_EXEC void Operation4() { + + // __syncthreads(); + + if (!w.isNull() && threadId < (R / 2) * (C / 2) * (F / 2)) { + r_sm = (threadId / ((C / 2) * (F / 2))) * 2; + c_sm = ((threadId % ((C / 2) * (F / 2))) / (F / 2)) * 2; + f_sm = ((threadId % ((C / 2) * (F / 2))) % (F / 2)) * 2; + r_gl = r / 2 + threadId / ((C / 2) * (F / 2)); + c_gl = c / 2 + threadId % ((C / 2) * (F / 2)) / (F / 2); + f_gl = f / 2 + threadId % ((C / 2) * (F / 2)) % (F / 2); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + if (!INTERPOLATION && COEFF_RESTORE) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = + *w(r_gl, c_gl, f_gl); + } + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (debug2) + // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, + // *w(r_gl, c_gl, f_gl), r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + + f_gl += 1; + f_sm += 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (debug2) + // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, + // *w(r_gl, c_gl, f_gl), r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + } + } + + int base = 0; + if (!w.isNull() && threadId >= base && + threadId < base + (C / 2) * (F / 2)) { + r_sm = (R / 2) * 2; + c_sm = ((threadId - base) / (F / 2)) * 2; + f_sm = ((threadId - base) % (F / 2)) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (threadId - base) / (F / 2); + f_gl = f / 2 + (threadId - base) % (F / 2); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + if (!INTERPOLATION && COEFF_RESTORE) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = + *w(r_gl, c_gl, f_gl); + } + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + // printf("nf: %d, f_gl: %d, in_next: %d, f_in_next: %d\n", nf, + // f_gl, in_next, f_in_next); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + + f_gl += 1; + f_sm += 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (debug2) + // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, + // *w(r_gl, c_gl, f_gl), r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + // printf("nf: %d, f_gl: %d, in_next: %d, f_in_next: %d\n", nf, + // f_gl, in_next, f_in_next); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + } + } + base += (C / 2) * (F / 2); // ROUND_UP_WARP((C/2) * (F/2)) * WARP_SIZE; + if (!w.isNull() && threadId >= base && + threadId < base + (R / 2) * (F / 2)) { + r_sm = ((threadId - base) / (F / 2)) * 2; + c_sm = (C / 2) * 2; + f_sm = ((threadId - base) % (F / 2)) * 2; + r_gl = r / 2 + (threadId - base) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - base) % (F / 2); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + if (!INTERPOLATION && COEFF_RESTORE) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = + *w(r_gl, c_gl, f_gl); + } + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + + f_gl += 1; + f_sm += 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (debug2) + // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, + // *w(r_gl, c_gl, f_gl), r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + } + } + base += (R / 2) * (F / 2); // ROUND_UP_WARP((R/2) * (F/2)) * WARP_SIZE; + if (!w.isNull() && threadId >= base && + threadId < base + (R / 2) * (C / 2)) { + r_sm = ((threadId - base) / (C / 2)) * 2; + c_sm = ((threadId - base) % (C / 2)) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - base) / (C / 2); + c_gl = c / 2 + (threadId - base) % (C / 2); + f_gl = f / 2 + (F / 2); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + if (!INTERPOLATION && COEFF_RESTORE) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = + *w(r_gl, c_gl, f_gl); + } + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + + f_gl += 1; + f_sm += 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (debug2) + // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, + // *w(r_gl, c_gl, f_gl), r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + } + } + base += (R / 2) * (C / 2); // ROUND_UP_WARP((R/2) * (C/2)) * WARP_SIZE; + // load extra edges + if (!w.isNull() && threadId >= base && threadId < base + (R / 2)) { + r_sm = (threadId - base) * 2; + c_sm = (C / 2) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + threadId - base; + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (F / 2); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + if (!INTERPOLATION && COEFF_RESTORE) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = + *w(r_gl, c_gl, f_gl); + } + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + + f_gl += 1; + f_sm += 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (debug2) + // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, + // *w(r_gl, c_gl, f_gl), r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + } + } + base += (R / 2); // ROUND_UP_WARP((R/2)) * WARP_SIZE; + if (!w.isNull() && threadId >= base && threadId < base + (C / 2)) { + r_sm = (R / 2) * 2; + c_sm = (threadId - base) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId - base; + f_gl = f / 2 + (F / 2); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + if (!INTERPOLATION && COEFF_RESTORE) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = + *w(r_gl, c_gl, f_gl); + } + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + + f_gl += 1; + f_sm += 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (debug2) + // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, + // *w(r_gl, c_gl, f_gl), r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + } + } + base += (C / 2); // ROUND_UP_WARP((C/2)) * WARP_SIZE; + if (!w.isNull() && threadId >= base && threadId < base + (F / 2)) { + r_sm = (R / 2) * 2; + c_sm = (C / 2) * 2; + f_sm = (threadId - base) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + threadId - base; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + if (!INTERPOLATION && COEFF_RESTORE) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = + *w(r_gl, c_gl, f_gl); + } + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + + f_gl += 1; + f_sm += 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (debug2) + // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, + // *w(r_gl, c_gl, f_gl), r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + } + } + base += (F / 2); // ROUND_UP_WARP((F/2)) * WARP_SIZE; + // // load extra vertex + if (!w.isNull() && threadId >= base && threadId < base + 1) { + r_sm = (R / 2) * 2; + c_sm = (C / 2) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (F / 2); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + if (!INTERPOLATION && COEFF_RESTORE) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = + *w(r_gl, c_gl, f_gl); + } + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + + f_gl += 1; + f_sm += 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (debug2) + // printf("(%d %d %d) %f <- (%d %d %d)\n", r_sm, c_sm, f_sm, + // *w(r_gl, c_gl, f_gl), r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + if (COEFF_RESTORE) { + bool f_in_next = + (nf % 2 == 1 && f_gl % 2 == 0) || + (nf % 2 == 0 && (f_gl % 2 == 0 || f_gl == nf - 1)); + if (in_next && f_in_next) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = 0.0; + } else { + ; + } + } + } + } + } + } + } + + MGARDX_EXEC void Operation5() { + + // __syncthreads(); + + // __syncthreads(); + // if (debug) { + // printf("TYPE: %d %d %d %d\n", TYPE, min(rest_r_p, (R/2) * 2 + 1), + // min(rest_c_p, (C/2) * 2 + 1), min(rest_f_p, (F/2) * 2 + 1)); + // for (int i = 0; i < min(rest_r_p, (R/2) * 2 + 1); i++) { + // for (int j = 0; j < min(rest_c_p, (C/2) * 2 + 1); j++) { + // for (int k = 0; k < min(rest_f_p, (F/2) * 2 + 1); k++) { + // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + + if (!wf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) && + threadId < (R / 2) * (C / 2) * (F / 2) * 2) { + + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2)) / ((C / 2) * (F / 2))) * 2; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) / + (F / 2)) * + 2; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2)) / ((C / 2) * (F / 2)); + c_gl = c / 2 + + ((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + + ((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) % + (F / 2); + + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + + res = *wf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { // fused + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + f_gl = 2 * f_gl + 1; + // res = *wf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + } + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + + if (!wc.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 2 && + threadId < (R / 2) * (C / 2) * (F / 2) * 3) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / ((C / 2) * (F / 2))) * + 2; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) % + (F / 2); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = *wc(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { + res = *wc(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + + if (!wr.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 3 && + threadId < (R / 2) * (C / 2) * (F / 2) * 4) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 3) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 3) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) % + (F / 2); + + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + res = *wr(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { + res = *wr(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + + if (!wcf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 4 && + threadId < (R / 2) * (C / 2) * (F / 2) * 5) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 4) / ((C / 2) * (F / 2))) * + 2; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 4) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) % + (F / 2); + + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { + res = *wcf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res += lerp(f1, f2, ratio_c_sm[c_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res = lerp(f1, f2, ratio_c_sm[c_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { + res = *wcf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + + if (!wrf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 5 && + threadId < (R / 2) * (C / 2) * (F / 2) * 6) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 5) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 5) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) % + (F / 2); + + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + + res = *wrf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + + res += lerp(f1, f2, ratio_r_sm[r_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + + res = lerp(f1, f2, ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { + res = *wrf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + + if (!wrc.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 6 && + threadId < (R / 2) * (C / 2) * (F / 2) * 7) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 6) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 6) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) % + (F / 2); + + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = *wrc(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res += lerp(c1, c2, ratio_r_sm[r_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf) { + res = *wrc(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + + if (!wrcf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 7 && + threadId < (R / 2) * (C / 2) * (F / 2) * 8) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 7) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 7) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) % + (F / 2); + + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { + res = *wrcf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + T f1 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f3 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f4 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + + T fc1 = lerp(f1, f2, ratio_c_sm[c_sm - 1]); + T fc2 = lerp(f3, f4, ratio_c_sm[c_sm - 1]); + + res += lerp(fc1, fc2, ratio_r_sm[r_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + T f1 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f3 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f4 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + + T fc1 = lerp(f1, f2, ratio_c_sm[c_sm - 1]); + T fc2 = lerp(f3, f4, ratio_c_sm[c_sm - 1]); + + res = lerp(fc1, fc2, ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf) { + res = *wrcf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + + if (r + (R / 2) * 2 == nr_p - 1) { + if (threadId < (C / 2) * (F / 2)) { + if (!wf.isNull()) { + r_sm = (R / 2) * 2; + c_sm = (threadId / (F / 2)) * 2; + f_sm = (threadId % (F / 2)) * 2 + 1; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId / (F / 2); + f_gl = f / 2 + threadId % (F / 2); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + res = *wf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + // res = *wf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + } + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + + if (!wc.isNull()) { + r_sm = (R / 2) * 2; + c_sm = (threadId / (F / 2)) * 2 + 1; + f_sm = (threadId % (F / 2)) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId / (F / 2); + f_gl = f / 2 + threadId % (F / 2); + + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = *wc(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { + res = *wc(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + if (!wcf.isNull()) { + r_sm = (R / 2) * 2; + c_sm = (threadId / (F / 2)) * 2 + 1; + f_sm = (threadId % (F / 2)) * 2 + 1; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId / (F / 2); + f_gl = f / 2 + threadId % (F / 2); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { + res = *wcf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + T f1 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res += lerp(f1, f2, ratio_c_sm[c_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + T f1 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res = lerp(f1, f2, ratio_c_sm[c_sm - 1]); + } + } + + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { + res = *wcf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + // if (idx[1] ==0 && idx[2] == 0) { + // printf("%f(%d %d %d) %f(%d %d %d) -> %f\n", + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, + // f_sm)], r_sm, c_sm - 1, f_sm, v_sm[get_idx(ldsm1, + // ldsm2, r_sm, c_sm + 1, f_sm)], r_sm, c_sm + 1, + // f_sm, res); + // } + } + } + + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + } + } + + if (c + (C / 2) * 2 == nc_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) && + threadId < (R / 2) * (C / 2) * (F / 2) + (R / 2) * (F / 2)) { + if (!wf.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2)) * 2; + c_sm = (C / 2) * 2; + f_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2)) * 2 + 1; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2); + + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + res = *wf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + // res = *wf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + } + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + if (!wr.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2)) * 2 + 1; + c_sm = (C / 2) * 2; + f_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2)) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + res = *wr(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { + res = *wr(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + if (!wrf.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2)) * 2 + 1; + c_sm = (C / 2) * 2; + f_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2)) * 2 + 1; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2); + + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + res = *wrf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + T f1 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res += lerp(f1, f2, ratio_r_sm[r_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + T f1 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res = lerp(f1, f2, ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { + res = *wrf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + } + } + + if (f + (F / 2) * 2 == nf_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 2 && + threadId < (R / 2) * (C / 2) * (F / 2) * 2 + (R / 2) * (C / 2)) { + if (!wc.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2)) * 2; + c_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2)) * 2 + 1; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2); + c_gl = c / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2); + f_gl = f / 2 + (F / 2); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = *wc(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { + res = *wc(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + + if (!wr.isNull()) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2)) * 2 + 1; + c_sm = ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2)) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2); + c_gl = c / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2); + f_gl = f / 2 + (F / 2); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + res = *wr(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { + res = *wr(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + + if (!wrc.isNull()) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2)) * 2 + 1; + c_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2)) * 2 + 1; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2); + c_gl = c / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2); + f_gl = f / 2 + (F / 2); + + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = *wrc(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + T c1 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res += lerp(c1, c2, ratio_r_sm[r_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + T c1 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf) { + res = *wrc(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + T c1 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp( + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + } + } + + if (c + (C / 2) * 2 == nc_p - 1 && f + (F / 2) * 2 == nf_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 3 && + threadId < (R / 2) * (C / 2) * (F / 2) * 3 + (R / 2)) { + if (!wr.isNull()) { + r_sm = (threadId - (R / 2) * (C / 2) * (F / 2) * 3) * 2 + 1; + c_sm = (C / 2) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + threadId - (R / 2) * (C / 2) * (F / 2) * 3; + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (F / 2); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + res = *wr(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf) { + res = *wr(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + } + } + + if (r + (R / 2) * 2 == nr_p - 1 && f + (F / 2) * 2 == nf_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 4 && + threadId < (R / 2) * (C / 2) * (F / 2) * 4 + (C / 2)) { + if (!wc.isNull()) { + r_sm = (R / 2) * 2; + c_sm = (threadId - (R / 2) * (C / 2) * (F / 2) * 4) * 2 + 1; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId - (R / 2) * (C / 2) * (F / 2) * 4; + f_gl = f / 2 + (F / 2); + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = *wc(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl *= 2; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf) { + res = *wc(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + } + } + + if (r + (R / 2) * 2 == nr_p - 1 && c + (C / 2) * 2 == nc_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 5 && + threadId < (R / 2) * (C / 2) * (F / 2) * 5 + (F / 2)) { + if (!wf.isNull()) { + r_sm = (R / 2) * 2; + c_sm = (C / 2) * 2; + f_sm = (threadId - (R / 2) * (C / 2) * (F / 2) * 5) * 2 + 1; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + threadId - (R / 2) * (C / 2) * (F / 2) * 5; + if (TYPE == 1) { + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + res = *wf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION && COEFF_RESTORE) { + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + } else if (INTERPOLATION && !COEFF_RESTORE) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + } + } + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } else if (TYPE == 2) { + f_gl = 2 * f_gl + 1; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf) { + // res = *wf(r_gl, c_gl, f_gl); + if (!skip) { + if (INTERPOLATION) { + ; + } + } + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + } + } + + // __syncthreads(); + // if (debug) { + // printf("TYPE: %d %d %d %d\n", TYPE, min(rest_r_p, (R/2) * 2 + 1), + // min(rest_c_p, (C/2) * 2 + 1), min(rest_f_p, (F/2) * 2 + 1)); + // for (int i = 0; i < min(rest_r_p, (R/2) * 2 + 1); i++) { + // for (int j = 0; j < min(rest_c_p, (C/2) * 2 + 1); j++) { + // for (int k = 0; k < min(rest_f_p, (F/2) * 2 + 1); k++) { + // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + + // __syncthreads(); + } + + MGARDX_EXEC void Operation6() { + r_sm = FunctorBase::GetThreadIdZ(); + c_sm = FunctorBase::GetThreadIdY(); + f_sm = FunctorBase::GetThreadIdX(); + + r_sm_ex = FunctorBase::GetBlockDimZ(); + c_sm_ex = FunctorBase::GetBlockDimY(); + f_sm_ex = FunctorBase::GetBlockDimX(); + + r_gl = r + r_sm; + c_gl = c + c_sm; + f_gl = f + f_sm; + + // r_gl_ex = r + (R/2) * 2; + // c_gl_ex = c + (C/2) * 2; + // f_gl_ex = f + (F/2) * 2; + + r_gl_ex = r + rest_r - 1; + c_gl_ex = c + rest_c - 1; + f_gl_ex = f + rest_f - 1; + + int unpadding_r = rest_r; + int unpadding_c = rest_c; + int unpadding_f = rest_f; + if (nr % 2 == 0) + unpadding_r -= 1; + if (nc % 2 == 0) + unpadding_c -= 1; + if (TYPE == 1 && nf % 2 == 0) + unpadding_f -= 1; + + if (r_sm < unpadding_r && c_sm < unpadding_c && f_sm < unpadding_f) { + + // store extra rules + // case 1: input = odd (non-padding required) + // case 1.a: block size + 1 == rest (need to store extra); + // case 1.b: block size + 1 != rest (No need to store extra); + // case 2: input = even (un-padding requried) + // case 2.a: block size + 1 >= rest (No need to store extra, but need + // un-padding first); case 2.b: block size + 1 < rest (No need to store + // extra); + + if (D_LOCAL >= 3 && r_sm == 0) { + if (nr % 2 != 0 && (R / 2) * 2 + 1 == rest_r) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl_ex, c_gl, f_gl) += + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm)]; + } else { + *v(r_gl_ex, c_gl, f_gl) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm)]; + } + } + if (nr % 2 == 0 && (R / 2) * 2 + 1 >= rest_r_p) { + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm)]; + } + } + + if (D_LOCAL >= 2 && c_sm == 0) { + if (nc % 2 != 0 && (C / 2) * 2 + 1 == rest_c) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl, c_gl_ex, f_gl) += + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm)]; + } else { + *v(r_gl, c_gl_ex, f_gl) = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm)]; + } + } + if (nc % 2 == 0 && (C / 2) * 2 + 1 >= rest_c_p) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm)]; + } + } + + if (D_LOCAL >= 1 && f_sm == 0) { + if (nf % 2 != 0 && (F / 2) * 2 + 1 == rest_f) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl, c_gl, f_gl_ex) += + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm_ex)]; + } else { + *v(r_gl, c_gl, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm_ex)]; + } + } + if (nf % 2 == 0 && (F / 2) * 2 + 1 >= rest_f_p && TYPE == 1) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f_p - 1)]; + } + } + + // load extra edges + if (D_LOCAL >= 2 && c_sm == 0 && f_sm == 0) { + if (nc % 2 != 0 && (C / 2) * 2 + 1 == rest_c && nf % 2 != 0 && + (F / 2) * 2 + 1 == rest_f) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl, c_gl_ex, f_gl_ex) += + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm_ex)]; + } else { + *v(r_gl, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm_ex)]; + } + } + if (nc % 2 == 0 && nf % 2 == 0 && (C / 2) * 2 + 1 >= rest_c_p && + (F / 2) * 2 + 1 >= rest_f_p && TYPE == 1) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, rest_f - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, rest_f_p - 1)]; + } + if (nc % 2 == 0 && nf % 2 != 0 && (C / 2) * 2 + 1 >= rest_c_p && + (F / 2) * 2 + 1 == rest_f) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl, c_gl_ex, f_gl_ex) += + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm_ex)]; + } else { + *v(r_gl, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm_ex)]; + } + } + if (nc % 2 != 0 && nf % 2 == 0 && (C / 2) * 2 + 1 == rest_c && + (F / 2) * 2 + 1 >= rest_f_p && TYPE == 1) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl, c_gl_ex, f_gl_ex) += + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f_p - 1)]; + } else { + *v(r_gl, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f_p - 1)]; + // printf("(%d %d %d): %f <- (%d %d %d)\n", + // r_gl, c_gl_ex, f_gl_ex, + // *v(r_gl, c_gl_ex, f_gl_ex), + // r_sm, c_sm_ex, f_gl_ex); + } + } + } + + if (D_LOCAL >= 3 && r_sm == 0 && f_sm == 0) { + if (nr % 2 != 0 && (R / 2) * 2 + 1 == rest_r && nf % 2 != 0 && + (F / 2) * 2 + 1 == rest_f) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl_ex, c_gl, f_gl_ex) += + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm_ex)]; + } else { + *v(r_gl_ex, c_gl, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm_ex)]; + } + } + if (nr % 2 == 0 && nf % 2 == 0 && (R / 2) * 2 + 1 >= rest_r_p && + (F / 2) * 2 + 1 >= rest_f_p && TYPE == 1) { + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, rest_f - 1)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, rest_f_p - 1)]; + } + if (nr % 2 == 0 && nf % 2 != 0 && (R / 2) * 2 + 1 >= rest_r_p && + (F / 2) * 2 + 1 == rest_f) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl_ex, c_gl, f_gl_ex) += + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm_ex)]; + } else { + *v(r_gl_ex, c_gl, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm_ex)]; + } + } + if (nr % 2 != 0 && nf % 2 == 0 && (R / 2) * 2 + 1 == rest_r && + (F / 2) * 2 + 1 >= rest_f_p && TYPE == 1) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl_ex, c_gl, f_gl_ex) += + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f_p - 1)]; + } else { + *v(r_gl_ex, c_gl, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f_p - 1)]; + // printf("(%d %d %d): %f <- (%d %d %d)\n", + // r_gl_ex, c_gl, rest_f-1, + // *v(r_gl_ex-1, c_gl, f_gl_ex), + // r_sm_ex, c_sm, rest_f_p-1); + } + } + } + + if (D_LOCAL >= 3 && r_sm == 0 && c_sm == 0) { + if (nr % 2 != 0 && (R / 2) * 2 + 1 == rest_r && nc % 2 != 0 && + (C / 2) * 2 + 1 == rest_c) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl_ex, c_gl_ex, f_gl) += + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm)]; + } else { + *v(r_gl_ex, c_gl_ex, f_gl) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm)]; + } + } + if (nr % 2 == 0 && nc % 2 == 0 && (R / 2) * 2 + 1 >= rest_r_p && + (C / 2) * 2 + 1 >= rest_c_p) { + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm)]; + } + if (nr % 2 == 0 && nc % 2 != 0 && (R / 2) * 2 + 1 >= rest_r_p && + (C / 2) * 2 + 1 == rest_c) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl_ex, c_gl_ex, f_gl) += + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm)]; + } else { + *v(r_gl_ex, c_gl_ex, f_gl) = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm)]; + } + } + if (nr % 2 != 0 && nc % 2 == 0 && (R / 2) * 2 + 1 == rest_r && + (C / 2) * 2 + 1 >= rest_c_p) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl_ex, c_gl_ex, f_gl) += + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm)]; + } else { + *v(r_gl_ex, c_gl_ex, f_gl) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm)]; + } + } + } + // load extra vertex + + if (D_LOCAL >= 3 && r_sm == 0 && c_sm == 0 && f_sm == 0) { + if (nr % 2 != 0 && (R / 2) * 2 + 1 == rest_r && nc % 2 != 0 && + (C / 2) * 2 + 1 == rest_c && nf % 2 != 0 && + (F / 2) * 2 + 1 == rest_f) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl_ex, c_gl_ex, f_gl_ex) += + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm_ex)]; + } else { + *v(r_gl_ex, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm_ex)]; + } + } + + if (nr % 2 == 0 && nc % 2 == 0 && nf % 2 == 0 && + (R / 2) * 2 + 1 >= rest_r_p && (C / 2) * 2 + 1 >= rest_c_p && + (F / 2) * 2 + 1 >= rest_f_p && TYPE == 1) { + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, rest_f - 1)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, + rest_f_p - 1)]; + } + if (nr % 2 == 0 && nc % 2 == 0 && nf % 2 != 0 && + (R / 2) * 2 + 1 >= rest_r_p && (C / 2) * 2 + 1 >= rest_c_p && + (F / 2) * 2 + 1 == rest_f) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl_ex, c_gl_ex, f_gl_ex) += v_sm[get_idx( + ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm_ex)]; + } else { + *v(r_gl_ex, c_gl_ex, f_gl_ex) = v_sm[get_idx( + ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm_ex)]; + } + } + if (nr % 2 == 0 && nc % 2 != 0 && nf % 2 == 0 && + (R / 2) * 2 + 1 >= rest_r_p && (C / 2) * 2 + 1 == rest_c && + (F / 2) * 2 + 1 >= rest_f_p && TYPE == 1) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl_ex, c_gl_ex, f_gl_ex) += v_sm[get_idx( + ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, rest_f_p - 1)]; + } else { + *v(r_gl_ex, c_gl_ex, f_gl_ex) = v_sm[get_idx( + ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, rest_f_p - 1)]; + } + } + if (nr % 2 != 0 && nc % 2 == 0 && nf % 2 == 0 && + (R / 2) * 2 + 1 == rest_r && (C / 2) * 2 + 1 >= rest_c_p && + (F / 2) * 2 + 1 >= rest_f_p && TYPE == 1) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl_ex, c_gl_ex, f_gl_ex) += v_sm[get_idx( + ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, rest_f_p - 1)]; + } else { + *v(r_gl_ex, c_gl_ex, f_gl_ex) = v_sm[get_idx( + ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, rest_f_p - 1)]; + } + } + if (nr % 2 == 0 && nc % 2 != 0 && nf % 2 != 0 && + (R / 2) * 2 + 1 >= rest_r_p && (C / 2) * 2 + 1 == rest_c && + (F / 2) * 2 + 1 == rest_f) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl_ex, c_gl_ex, f_gl_ex) += + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm_ex)]; + } else { + *v(r_gl_ex, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm_ex)]; + } + } + if (nr % 2 != 0 && nc % 2 == 0 && nf % 2 != 0 && + (R / 2) * 2 + 1 == rest_r && (C / 2) * 2 + 1 >= rest_c_p && + (F / 2) * 2 + 1 == rest_f) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl_ex, c_gl_ex, f_gl_ex) += + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm_ex)]; + } else { + *v(r_gl_ex, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm_ex)]; + } + } + if (nr % 2 != 0 && nc % 2 != 0 && nf % 2 == 0 && + (R / 2) * 2 + 1 == rest_r && (C / 2) * 2 + 1 == rest_c && + (F / 2) * 2 + 1 >= rest_f_p && TYPE == 1) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl_ex, c_gl_ex, f_gl_ex) += + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f_p - 1)]; + } else { + *v(r_gl_ex, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f_p - 1)]; + } + } + } + } + } + + MGARDX_EXEC void Operation7() { + // __syncthreads(); + + if (r_sm < rest_r && c_sm < rest_c && f_sm < rest_f) { + if (r_gl >= svr && r_gl < svr + nvr && c_gl >= svc && c_gl < svc + nvc && + f_gl >= svf && f_gl < svf + nvf) { + if (!INTERPOLATION && COEFF_RESTORE) { + *v(r_gl, c_gl, f_gl) += v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + } else { + *v(r_gl, c_gl, f_gl) = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + } + } + } + + v.reset_offset(); + w.reset_offset(); + wf.reset_offset(); + wc.reset_offset(); + wr.reset_offset(); + wcf.reset_offset(); + wrf.reset_offset(); + wrc.reset_offset(); + wrcf.reset_offset(); + } + + MGARDX_CONT SIZE shared_memory_size() { + SIZE size = 0; + size = ((R + 1) * (C + 1) * (F + 1) + R + C + F) * sizeof(T); + align_byte_offset(size); + size += (D_GLOBAL * 4) * sizeof(SIZE); + align_byte_offset(size); + size += (D_GLOBAL * 1) * sizeof(DIM); + return size; + } + +private: + SubArray<1, SIZE, DeviceType> shape, shape_c; + SubArray<1, T, DeviceType> ratio_r, ratio_c, ratio_f; + SubArray v, w, wf, wc, wr, wcf, wrf, wrc, wrcf; + // SIZE *ldvs; + // SIZE *ldws; + DIM unprocessed_n; + SubArray<1, DIM, DeviceType> unprocessed_dims; + SIZE threadId; + T *v_sm; + T *ratio_f_sm; + T *ratio_c_sm; + T *ratio_r_sm; + DIM curr_dim_r; + DIM curr_dim_c; + DIM curr_dim_f; + + SIZE svr, svc, svf, nvr, nvc, nvf; + + SIZE nr, nc, nf; + SIZE nr_c, nc_c, nf_c; + SIZE r, c, f; + SIZE rest_r, rest_c, rest_f; + SIZE nr_p, nc_p, nf_p; + SIZE rest_r_p, rest_c_p, rest_f_p; + SIZE r_sm, c_sm, f_sm; + SIZE r_sm_ex, c_sm_ex, f_sm_ex; + SIZE r_gl, c_gl, f_gl; + SIZE r_gl_ex, c_gl_ex, f_gl_ex; + T res; + bool in_next; + + SIZE ldsm1, ldsm2; + + // SIZE * sm_size; + SIZE *shape_sm; + SIZE *shape_c_sm; + // SIZE * lvs_sm; + // SIZE * ldws_sm; + + // DIM * sm_dim; + DIM *unprocessed_dims_sm; + + SIZE idx[D_GLOBAL]; + + int skip; +}; + +template +class GpkRevKernel : public Kernel { +public: + constexpr static const DIM NumDim = D_LOCAL; + using DataType = T; + constexpr static std::string_view Name = "gpk_rev_nd"; + MGARDX_CONT + GpkRevKernel(SubArray<1, SIZE, DeviceType> shape, + SubArray<1, SIZE, DeviceType> shape_c, DIM unprocessed_n, + SubArray<1, DIM, DeviceType> unprocessed_dims, DIM curr_dim_r, + DIM curr_dim_c, DIM curr_dim_f, + SubArray<1, T, DeviceType> ratio_r, + SubArray<1, T, DeviceType> ratio_c, + SubArray<1, T, DeviceType> ratio_f, + SubArray v, + SubArray w, + SubArray wf, + SubArray wc, + SubArray wr, + SubArray wcf, + SubArray wrf, + SubArray wrc, + SubArray wrcf, SIZE svr, SIZE svc, + SIZE svf, SIZE nvr, SIZE nvc, SIZE nvf) + : shape(shape), shape_c(shape_c), unprocessed_n(unprocessed_n), + unprocessed_dims(unprocessed_dims), curr_dim_r(curr_dim_r), + curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), ratio_r(ratio_r), + ratio_c(ratio_c), ratio_f(ratio_f), v(v), w(w), wf(wf), wc(wc), wr(wr), + wcf(wcf), wrf(wrf), wrc(wrc), wrcf(wrcf), svr(svr), svc(svc), svf(svf), + nvr(nvr), nvc(nvc), nvf(nvf) {} + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + + using FunctorType = + GpkRevFunctor; + FunctorType functor(shape, shape_c, unprocessed_n, unprocessed_dims, + curr_dim_r, curr_dim_c, curr_dim_f, ratio_r, ratio_c, + ratio_f, v, w, wf, wc, wr, wcf, wrf, wrc, wrcf, svr, + svc, svf, nvr, nvc, nvf); + + SIZE nr = curr_dim_r < D_GLOBAL ? shape.dataHost()[curr_dim_r] : 1; + SIZE nc = curr_dim_c < D_GLOBAL ? shape.dataHost()[curr_dim_c] : 1; + SIZE nf = curr_dim_f < D_GLOBAL ? shape.dataHost()[curr_dim_f] : 1; + if (D_LOCAL == 2) { + nr = 1; + } + SIZE total_thread_z = std::max(nr - 1, (SIZE)1); + SIZE total_thread_y = std::max(nc - 1, (SIZE)1); + SIZE total_thread_x = std::max(nf - 1, (SIZE)1); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + // printf("sm_size: %llu\n", sm_size); + // printf("RCF: %u %u %u\n", R, C, F); + tbz = R; + tby = C; + tbx = F; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + for (DIM d = 0; d < D_GLOBAL; d++) { + if (D_LOCAL == 3 && d != curr_dim_f && d != curr_dim_c && + d != curr_dim_r) { + gridx *= shape.dataHost()[d]; + } + if (D_LOCAL == 2 && d != curr_dim_f && d != curr_dim_c) { + gridx *= shape.dataHost()[d]; + } + } + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, SIZE, DeviceType> shape, shape_c; + SubArray<1, T, DeviceType> ratio_r, ratio_c, ratio_f; + SubArray v, w, wf, wc, wr, wcf, wrf, wrc, wrcf; + DIM unprocessed_n; + SubArray<1, DIM, DeviceType> unprocessed_dims; + DIM curr_dim_r; + DIM curr_dim_c; + DIM curr_dim_f; + SIZE svr, svc, svf, nvr, nvc, nvf; +}; + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel3D.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel3D.hpp new file mode 100644 index 000000000..47c28aafc --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/GridProcessingKernel3D.hpp @@ -0,0 +1,2413 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_GRID_PROCESSING_KERNEL_3D_TEMPLATE +#define MGARD_X_160_GRID_PROCESSING_KERNEL_3D_TEMPLATE + +#include "../../../RuntimeX/RuntimeX.h" +#include "GPKFunctor.h" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +class GpkReo3DFunctor : public Functor { +public: + MGARDX_CONT GpkReo3DFunctor() {} + MGARDX_CONT GpkReo3DFunctor( + SIZE nr, SIZE nc, SIZE nf, SIZE nr_c, SIZE nc_c, SIZE nf_c, + SubArray<1, T, DeviceType> ratio_r, SubArray<1, T, DeviceType> ratio_c, + SubArray<1, T, DeviceType> ratio_f, SubArray v, + SubArray w, SubArray wf, + SubArray wc, SubArray wr, + SubArray wcf, SubArray wrf, + SubArray wrc, SubArray wrcf) + : nr(nr), nc(nc), nf(nf), nr_c(nr_c), nc_c(nc_c), nf_c(nf_c), + ratio_r(ratio_r), ratio_c(ratio_c), ratio_f(ratio_f), v(v), w(w), + wf(wf), wc(wc), wr(wr), wcf(wcf), wrf(wrf), wrc(wrc), wrcf(wrcf) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + + sm = (T *)FunctorBase::GetSharedMemory(); + ldsm1 = (F / 2) * 2 + 1; + ldsm2 = (C / 2) * 2 + 1; + v_sm = sm; + ratio_f_sm = sm + ((F / 2) * 2 + 1) * ((C / 2) * 2 + 1) * ((R / 2) * 2 + 1); + ratio_c_sm = ratio_f_sm + (F / 2) * 2; + ratio_r_sm = ratio_c_sm + (C / 2) * 2; + + r = FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ(); + c = FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY(); + f = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX(); + + rest_r = nr - r; + rest_c = nc - c; + rest_f = nf - f; + + nr_p = nr; + nc_p = nc; + nf_p = nf; + + rest_r_p = rest_r; + rest_c_p = rest_c; + rest_f_p = rest_f; + + if (nr % 2 == 0) { + nr_p = nr + 1; + rest_r_p = nr_p - r; + } + if (nc % 2 == 0) { + nc_p = nc + 1; + rest_c_p = nc_p - c; + } + if (nf % 2 == 0) { + nf_p = nf + 1; + rest_f_p = nf_p - f; + } + + r_sm = FunctorBase::GetThreadIdZ(); + c_sm = FunctorBase::GetThreadIdY(); + f_sm = FunctorBase::GetThreadIdX(); + + r_sm_ex = (R / 2) * 2; + c_sm_ex = (C / 2) * 2; + f_sm_ex = (F / 2) * 2; + + threadId = (FunctorBase::GetThreadIdZ() * + (FunctorBase::GetBlockDimX() * + FunctorBase::GetBlockDimY())) + + (FunctorBase::GetThreadIdY() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + + r_gl = r + r_sm; + r_gl_ex = r + (R / 2) * 2; + c_gl = c + c_sm; + c_gl_ex = c + (C / 2) * 2; + f_gl = f + f_sm; + f_gl_ex = f + (F / 2) * 2; + + /* Load v */ + // loading extra rules + // case 1: input = odd (non-padding required) + // case 1.a: block size < rest (need to load extra); + // case 1.b: block size > rest (NO need to load extra); + // case 2: input = even (padding requried) + // case 2.a: block size < rest (need to load extra); + // case 2.b: block size >= rest (NO need to load extra, but need + // padding); + + // Load from dv + if (r_sm < rest_r && c_sm < rest_c && f_sm < rest_f) { + + // load cubic + // asm volatile("membar.cta;"); + // start = clock64(); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *v(r_gl, c_gl, f_gl); + // if (blockIdx.x==0 && blockIdx.y==0&&blockIdx.z==0) { + // printf("load (%d %d %d) %f <- %d+(%d %d %d) (ld: %d %d)\n", + // r_sm, c_sm, f_sm, + // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl)], + // other_offset_v+r_gl, c_gl, f_gl, lddv1, lddv2); + // } + if (r_sm == 0) { + if (rest_r > (R / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm)] = + *v(r_gl_ex, c_gl, f_gl); + } + } + if (c_sm == 0) { + if (rest_c > (C / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm)] = + *v(r_gl, c_gl_ex, f_gl); + } + } + if (f_sm == 0) { + if (rest_f > (F / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm_ex)] = + *v(r_gl, c_gl, f_gl_ex); + } + } + if (c_sm == 0 && f_sm == 0) { + if (rest_c > (C / 2) * 2 && rest_f > (F / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm_ex)] = + *v(r_gl, c_gl_ex, f_gl_ex); + } + } + if (r_sm == 0 && f_sm == 0) { + if (rest_r > (R / 2) * 2 && rest_f > (F / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm_ex)] = + *v(r_gl_ex, c_gl, f_gl_ex); + } + } + if (r_sm == 0 && c_sm == 0) { + if (rest_r > (R / 2) * 2 && rest_c > (C / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm)] = + *v(r_gl_ex, c_gl_ex, f_gl); + } + } + if (r_sm == 0 && c_sm == 0 && f_sm == 0) { + if (rest_r > (R / 2) * 2 && rest_c > (C / 2) * 2 && + rest_f > (F / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm_ex)] = + *v(r_gl_ex, c_gl_ex, f_gl_ex); + } + } + } + } + + MGARDX_EXEC void Operation2() { + // apply padding is necessary + if (r_sm < rest_r && c_sm < rest_c && f_sm < rest_f) { + + // printf("load main[%d %d %d]:%f --> [%d %d %d] (%d %d %d)\n", r_gl, + // c_gl, f_gl, + // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl)], r_sm, c_sm, f_sm, nr, + // nc, nf); + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[load main] block id %d,%d,%d elapsed %lu\n", blockIdx.z, + // blockIdx.y, blockIdx.x, start); start = clock64(); + + // load extra surface + + if (r_sm == 0) { + if (rest_r > (R / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm)] = + // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl)]; + // printf("load-r[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl, f_gl, + // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl)], r_sm_ex, c_sm, + // f_sm); + } else if (nr % 2 == 0) { + // if (r == 16 && c == 0 && f == 0) { + // printf("padding (%d %d %d) %f <- (%f %f %f)\n", rest_r_p - 1, + // c_sm, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)], rest_r + // - 1, c_sm, f_sm); + // padded = true; + // aa = v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)]; + // bb = v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm)]; + // } + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)]; + } + } + + if (c_sm == 0) { + if (rest_c > (C / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm)] = + // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl)]; + // printf("load-c[%d %d %d]:%f --> [%d %d %d]\n", r_gl, c_gl_ex, f_gl, + // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl)], r_sm, c_sm_ex, + // f_sm); + } else if (nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, f_sm)]; + } + } + + if (f_sm == 0) { + if (rest_f > (F / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm_ex)] = + // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl_ex)]; + // printf("load-f[%d %d %d]:%f --> [%d %d %d]\n", r_gl, c_gl, f_gl_ex, + // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl_ex)], r_sm, c_sm, + // f_sm_ex); + } else if (nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f - 1)]; + } + } + + // load extra edges + if (c_sm == 0 && f_sm == 0) { + if (rest_c > (C / 2) * 2 && rest_f > (F / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm_ex)] = + // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)]; + // printf("load-cf[%d %d %d]:%f --> [%d %d %d]\n", r_gl, c_gl_ex, + // f_gl_ex, dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)], r_sm, + // c_sm_ex, f_sm_ex); + } else if (rest_c <= (C / 2) * 2 && rest_f <= (F / 2) * 2 && + nc % 2 == 0 && nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, rest_f - 1)]; + } else if (rest_c > (C / 2) * 2 && rest_f <= (F / 2) * 2 && + nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f - 1)]; + } else if (rest_c <= (C / 2) * 2 && rest_f > (F / 2) * 2 && + nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm_ex)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, f_sm_ex)]; + } + } + + if (r_sm == 0 && f_sm == 0) { + if (rest_r > (R / 2) * 2 && rest_f > (F / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm_ex)] = + // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)]; + // printf("load-rf[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl, + // f_gl_ex, dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)], + // r_sm_ex, c_sm, f_sm_ex); + } else if (rest_r <= (R / 2) * 2 && rest_f <= (F / 2) * 2 && + nr % 2 == 0 && nf % 2 == 0) { + // printf("padding (%d %d %d) <- (%d %d %d)\n", rest_r_p - 1, c_sm, + // rest_f_p - 1, rest_r - 1, c_sm, rest_f - 1); + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, rest_f - 1)]; + } else if (rest_r > (R / 2) * 2 && rest_f <= (F / 2) * 2 && + nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f - 1)]; + } else if (rest_r <= (R / 2) * 2 && rest_f > (F / 2) * 2 && + nr % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm_ex)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm_ex)]; + } + } + + if (r_sm == 0 && c_sm == 0) { + if (rest_r > (R / 2) * 2 && rest_c > (C / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm)] = + // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)]; + // printf("load-rc[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl_ex, + // f_gl, dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)], r_sm_ex, + // c_sm_ex, f_sm); + } else if (rest_r <= (R / 2) * 2 && rest_c <= (C / 2) * 2 && + nr % 2 == 0 && nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm)]; + // printf("padding (%d %d %d) <- (%d %d %d): %f\n", rest_r_p - 1, + // rest_c_p - 1, f_sm, rest_r - 1, rest_c - 1, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm)]); + } else if (rest_r > (R / 2) * 2 && rest_c <= (C / 2) * 2 && + nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c - 1, f_sm)]; + } else if (rest_r <= (R / 2) * 2 && rest_c > (C / 2) * 2 && + nr % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm_ex, f_sm)]; + } + } + // load extra vertex + + if (r_sm == 0 && c_sm == 0 && f_sm == 0) { + if (rest_r > (R / 2) * 2 && rest_c > (C / 2) * 2 && + rest_f > (F / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm_ex)] = + *v(r_gl_ex, c_gl_ex, f_gl_ex); + // printf("load-rcf[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl_ex, + // f_gl_ex, dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)], + // r_sm_ex, c_sm_ex, f_sm_ex); + } else if (rest_r <= (R / 2) * 2 && rest_c <= (C / 2) * 2 && + rest_f <= (F / 2) * 2 && nr % 2 == 0 && nc % 2 == 0 && + nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, + rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, rest_f - 1)]; + } else if (rest_r > (R / 2) * 2 && rest_c > (C / 2) * 2 && + rest_f <= (F / 2) * 2 && nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f - 1)]; + } else if (rest_r > (R / 2) * 2 && rest_c <= (C / 2) * 2 && + rest_f > (F / 2) * 2 && nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm_ex)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c - 1, f_sm_ex)]; + } else if (rest_r > (R / 2) * 2 && rest_c <= (C / 2) * 2 && + rest_f <= (F / 2) * 2 && nc % 2 == 0 && nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c - 1, rest_f - 1)]; + } else if (rest_r <= (R / 2) * 2 && rest_c > (C / 2) * 2 && + rest_f > (F / 2) * 2 && nr % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm_ex)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm_ex, f_sm_ex)]; + } else if (rest_r <= (R / 2) * 2 && rest_c > (C / 2) * 2 && + rest_f <= (F / 2) * 2 && nr % 2 == 0 && nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm_ex, rest_f - 1)]; + } else if (rest_r <= (R / 2) * 2 && rest_c <= (C / 2) * 2 && + rest_f > (F / 2) * 2 && nr % 2 == 0 && nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm_ex)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm_ex)]; + } + } + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[load extra] block id %d,%d,%d elapsed %lu\n", blockIdx.z, + // blockIdx.y, blockIdx.x, start); start = clock64(); + + // load dist + if (c_sm == 0 && f_sm == 0 && r_sm < rest_r_p - 2) { + // printf("%d/%d load %f\n", r_sm, rest_r - 2, dratio_r[r + r_sm]); + ratio_r_sm[r_sm] = *ratio_r(r + r_sm); + // if (nr % 2 == 0 && (R/2) * 2 + 1 >= rest_r_p && r_sm == 0) { + // ratio_r_sm[rest_r_p - 3] = 0.5; + // } + } + if (r_sm == 0 && f_sm == 0 && c_sm < rest_c_p - 2) { + ratio_c_sm[c_sm] = *ratio_c(c + c_sm); + // if (nc % 2 == 0 && (C/2) * 2 + 1 >= rest_c_p && c_sm == 0) { + // ratio_c_sm[rest_c_p - 3] = 0.5; + // } + } + if (c_sm == 0 && r_sm == 0 && f_sm < rest_f_p - 2) { + ratio_f_sm[f_sm] = *ratio_f(f + f_sm); + // if (nf % 2 == 0 && (F/2) * 2 + 1 >= rest_f_p && f_sm == 0) { + // ratio_f_sm[rest_f_p - 3] = 0.5; + // } + } + + // if (r == 0 && c == 0 && f == 0 && r_sm == 0 && c_sm == 0 && f_sm == 0) + // { + // printf("ratio:"); + // for (int i = 0; i < (R/2) * 2 + 1; i++) { + // printf("%2.2f ", ratio_r_sm[i]); + // } + // printf("\n"); + // } + + } // restrict boundary + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[load ratio] block id %d,%d,%d elapsed %lu\n", blockIdx.z, + // blockIdx.y, blockIdx.x, start); start = clock64(); + // bool debug = false; + // if (threadx == 0 && thready == 0 && threadz == 0) { + // debug = true; + // } + + // __syncthreads(); + // // debug print + // if (debug) { + // printf("in config: %d %d %d (%d %d %d)\n", (R/2), (C/2), (F/2), r,c,f); + // printf("rest_p: %d %d %d\n", rest_r_p, rest_c_p, rest_f_p); + // for (int i = 0; i < (R/2) * 2 + 1; i++) { + // for (int j = 0; j < (C/2) * 2 + 1; j++) { + // for (int k = 0; k < (F/2) * 2 + 1; k++) { + // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + } + + MGARDX_EXEC void Operation3() { + if (!w.isNull() && threadId < (R / 2) * (C / 2) * (F / 2)) { + r_sm = (threadId / ((C / 2) * (F / 2))) * 2; + c_sm = ((threadId % ((C / 2) * (F / 2))) / (F / 2)) * 2; + f_sm = ((threadId % ((C / 2) * (F / 2))) % (F / 2)) * 2; + r_gl = r / 2 + threadId / ((C / 2) * (F / 2)); + c_gl = c / 2 + threadId % ((C / 2) * (F / 2)) / (F / 2); + f_gl = f / 2 + threadId % ((C / 2) * (F / 2)) % (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, !w.isNull()[get_idx(lddw1, lddw2, r_gl, c_gl, + // f_gl)], r_sm, c_sm, f_sm); + } + } + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[store coarse] block id %d,%d,%d elapsed %lu\n", blockIdx.z, + // blockIdx.y, blockIdx.x, start); start = clock64(); + int base = 0; + // printf("TYPE =%d \n", TYPE); + // printf("%d == %d && %llu >= %d && %llu < %d\n", r + (R/2) * 2, nr_p - 1, + // threadId, base, threadId, base + (C/2) * (F/2)); + + if (!w.isNull() && r + (R / 2) * 2 == nr_p - 1 && threadId >= base && + threadId < base + (C / 2) * (F / 2)) { + r_sm = (R / 2) * 2; + c_sm = ((threadId - base) / (F / 2)) * 2; + f_sm = ((threadId - base) % (F / 2)) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (threadId - base) / (F / 2); + f_gl = f / 2 + (threadId - base) % (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, !w.isNull()[get_idx(lddw1, lddw2, r_gl, c_gl, + // f_gl)], r_sm, c_sm, f_sm); + } + } + + base += (C / 2) * (F / 2); // ROUND_UP_WARP((C/2) * (F/2)) * WARP_SIZE; + if (!w.isNull() && c + (C / 2) * 2 == nc_p - 1 && threadId >= base && + threadId < base + (R / 2) * (F / 2)) { + r_sm = ((threadId - base) / (F / 2)) * 2; + c_sm = (C / 2) * 2; + f_sm = ((threadId - base) % (F / 2)) * 2; + r_gl = r / 2 + (threadId - base) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - base) % (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, !w.isNull()[get_idx(lddw1, lddw2, r_gl, c_gl, + // f_gl)], r_sm, c_sm, f_sm); + } + // printf("(%d %d %d) (%d %d %d) %f\n", + // r_sm, c_sm, f_sm, r_gl, c_gl, f_gl, dwork[get_idx(lddv1, lddv2, + // r_gl, c_gl, f_gl)]); + } + + base += (R / 2) * (F / 2); // ROUND_UP_WARP((R/2) * (F/2)) * WARP_SIZE; + // printf("%d %d\n", base, threadId); + if (!w.isNull() && f + (F / 2) * 2 == nf_p - 1 && threadId >= base && + threadId < base + (R / 2) * (C / 2)) { + r_sm = ((threadId - base) / (C / 2)) * 2; + c_sm = ((threadId - base) % (C / 2)) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - base) / (C / 2); + c_gl = c / 2 + (threadId - base) % (C / 2); + f_gl = f / 2 + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, !w.isNull()[get_idx(lddw1, lddw2, r_gl, c_gl, + // f_gl)], r_sm, c_sm, f_sm); + } + } + + base += (R / 2) * (C / 2); // ROUND_UP_WARP((R/2) * (C/2)) * WARP_SIZE; + // load extra edges + if (!w.isNull() && c + (C / 2) * 2 == nc_p - 1 && + f + (F / 2) * 2 == nf_p - 1 && threadId >= base && + threadId < base + (R / 2)) { + r_sm = (threadId - base) * 2; + c_sm = (C / 2) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + threadId - base; + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, !w.isNull()[get_idx(lddw1, lddw2, r_gl, c_gl, + // f_gl)], r_sm, c_sm, f_sm); + } + } + + base += (R / 2); // ROUND_UP_WARP((R/2)) * WARP_SIZE; + // if (TYPE == 2) printf("%d %d, %d, %llu, %d\n",!w.isNull() == NULL, f + + // (F/2) * 2, nf_p + // - 1, threadId, (C/2)); + if (!w.isNull() && r + (R / 2) * 2 == nr_p - 1 && + f + (F / 2) * 2 == nf_p - 1 && threadId >= base && + threadId < base + (C / 2)) { + r_sm = (R / 2) * 2; + c_sm = (threadId - base) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId - base; + f_gl = f / 2 + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, !w.isNull()[get_idx(lddw1, lddw2, r_gl, c_gl, + // f_gl)], r_sm, c_sm, f_sm); + } + // printf("store[%d %d %d]: %f\n", r_sm, c_sm, f_sm, v_sm[get_idx(ldsm1, + // ldsm2, r_sm, c_sm, f_sm)]); + } + + base += (C / 2); // ROUND_UP_WARP((C/2)) * WARP_SIZE; + if (!w.isNull() && r + (R / 2) * 2 == nr_p - 1 && + c + (C / 2) * 2 == nc_p - 1 && threadId >= base && + threadId < base + (F / 2)) { + r_sm = (R / 2) * 2; + c_sm = (C / 2) * 2; + f_sm = (threadId - base) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + threadId - base; + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, !w.isNull()[get_idx(lddw1, lddw2, r_gl, c_gl, + // f_gl)], r_sm, c_sm, f_sm); + } + } + base += (F / 2); // ROUND_UP_WARP((F/2)) * WARP_SIZE; + // // load extra vertex + if (!w.isNull() && r + (R / 2) * 2 == nr_p - 1 && + c + (C / 2) * 2 == nc_p - 1 && f + (F / 2) * 2 == nf_p - 1 && + threadId >= base && threadId < base + 1) { + r_sm = (R / 2) * 2; + c_sm = (C / 2) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (F / 2); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + *w(r_gl, c_gl, f_gl) = res; + // printf("w-store: %d+(%d %d %d) <- %f (%d %d %d)\n", other_offset_w, + // r_gl, c_gl, f_gl, !w.isNull()[get_idx(lddw1, lddw2, r_gl, c_gl, + // f_gl)], r_sm, c_sm, f_sm); + } + } + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[store extra] block id %d,%d,%d elapsed %lu\n", blockIdx.z, + // blockIdx.y, blockIdx.x, start); start = clock64(); + + // start = clock64(); + + if (!wf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) && + threadId < (R / 2) * (C / 2) * (F / 2) * 2) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2)) / ((C / 2) * (F / 2))) * 2; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) / + (F / 2)) * + 2; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2)) / ((C / 2) * (F / 2)); + c_gl = c / 2 + + ((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + + ((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) % + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wf(r_gl, c_gl, f_gl) = res; + } + + // if (nr == 70) + // printf("f-store: (%d %d %d) <- %f (%d %d %d)\n", r_gl, + // c_gl, f_gl, v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], r_sm, c_sm, + // f_sm); + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[(F/2)-store] block id %d,%d,%d elapsed %lu\n", blockIdx.z, + // blockIdx.y, blockIdx.x, start); start = clock64(); + } + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[(F/2)-store] block id %d,%d,%d elapsed %lu\n", blockIdx.z, + // blockIdx.y, blockIdx.x, start); start = clock64(); + + // if (r_sm % 2 == 0 && c_sm % 2 != 0 && f_sm % 2 == 0) { + + if (!wc.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 2 && + threadId < (R / 2) * (C / 2) * (F / 2) * 3) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / ((C / 2) * (F / 2))) * + 2; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) % + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wc(r_gl, c_gl, f_gl) = res; + } + } + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[(C/2)-store] block id %d,%d,%d elapsed %lu\n", blockIdx.z, + // blockIdx.y, blockIdx.x, start); start = clock64(); + + // if (r_sm % 2 != 0 && c_sm % 2 == 0 && f_sm % 2 == 0) { + if (!wr.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 3 && + threadId < (R / 2) * (C / 2) * (F / 2) * 4) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 3) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 3) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) % + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wr(r_gl, c_gl, f_gl) = res; + } + } + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[(R/2)-store] block id %d,%d,%d elapsed %lu\n", blockIdx.z, + // blockIdx.y, blockIdx.x, start); start = clock64(); + // __syncthreads(); + if (!wcf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 4 && + threadId < (R / 2) * (C / 2) * (F / 2) * 5) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 4) / ((C / 2) * (F / 2))) * + 2; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 4) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) % + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res = lerp(f1, f2, ratio_c_sm[c_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wcf(r_gl, c_gl, f_gl) = res; + } + } + + // asm volatile("membar.cta;"); + // start = clock64() - start; + // printf("[CF-store] block id %d,%d,%d elapsed %lu\n", blockIdx.z, + // blockIdx.y, blockIdx.x, start); start = clock64(); + + if (!wrf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 5 && + threadId < (R / 2) * (C / 2) * (F / 2) * 6) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 5) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 5) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) % + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res = lerp(f1, f2, ratio_r_sm[r_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wrf(r_gl, c_gl, f_gl) = res; + } + } + + if (!wrc.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 6 && + threadId < (R / 2) * (C / 2) * (F / 2) * 7) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 6) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 6) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) % + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wrc(r_gl, c_gl, f_gl) = res; + } + } + + if (!wrcf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 7 && + threadId < (R / 2) * (C / 2) * (F / 2) * 8) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 7) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 7) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) % + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f3 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f4 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + + T fc1 = lerp(f1, f2, ratio_c_sm[c_sm - 1]); + T fc2 = lerp(f3, f4, ratio_c_sm[c_sm - 1]); + + res = lerp(fc1, fc2, ratio_r_sm[r_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wrcf(r_gl, c_gl, f_gl) = res; + } + } + // end = clock64(); + + // asm volatile("membar.cta;"); + // if (threadId < 256 && blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == + // 0) printf("threadId %d elapsed %lu\n", threadId, end-start); + if (r + (R / 2) * 2 == nr_p - 1) { + // printf("test\n"); + if (threadId < (C / 2) * (F / 2)) { + // printf("test1\n"); + if (!wf.isNull()) { + // printf("test2\n"); + r_sm = (R / 2) * 2; + c_sm = (threadId / (F / 2)) * 2; + f_sm = (threadId % (F / 2)) * 2 + 1; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId / (F / 2); + f_gl = f / 2 + threadId % (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + // printf("!wf.isNull() (%d %d %d): %f<-(%f %f %f)\n", r_gl, c_gl, + // f_gl, res, + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + // ratio_f_sm[f_sm - 1]); + *wf(r_gl, c_gl, f_gl) = res; + } + } + + if (!wc.isNull()) { + r_sm = (R / 2) * 2; + c_sm = (threadId / (F / 2)) * 2 + 1; + f_sm = (threadId % (F / 2)) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId / (F / 2); + f_gl = f / 2 + threadId % (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wc(r_gl, c_gl, f_gl) = res; + } + } + + if (!wcf.isNull()) { + r_sm = (R / 2) * 2; + c_sm = (threadId / (F / 2)) * 2 + 1; + f_sm = (threadId % (F / 2)) * 2 + 1; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId / (F / 2); + f_gl = f / 2 + threadId % (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res = lerp(f1, f2, ratio_c_sm[c_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wcf(r_gl, c_gl, f_gl) = res; + } + } + } + } + + if (c + (C / 2) * 2 == nc_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) && + threadId < (R / 2) * (C / 2) * (F / 2) + (R / 2) * (F / 2)) { + if (!wf.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2)) * 2; + c_sm = (C / 2) * 2; + f_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2)) * 2 + 1; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wf(r_gl, c_gl, f_gl) = res; + } + } + + if (!wr.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2)) * 2 + 1; + c_sm = (C / 2) * 2; + f_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2)) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wr(r_gl, c_gl, f_gl) = res; + } + } + + if (!wrf.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2)) * 2 + 1; + c_sm = (C / 2) * 2; + f_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2)) * 2 + 1; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res = lerp(f1, f2, ratio_r_sm[r_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wrf(r_gl, c_gl, f_gl) = res; + } + } + } + } + + if (f + (F / 2) * 2 == nf_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 2 && + threadId < (R / 2) * (C / 2) * (F / 2) * 2 + (R / 2) * (C / 2)) { + if (!wc.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2)) * 2; + c_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2)) * 2 + 1; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2); + c_gl = c / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2); + f_gl = f / 2 + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wc(r_gl, c_gl, f_gl) = res; + } + } + + if (!wr.isNull()) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2)) * 2 + 1; + c_sm = ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2)) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2); + c_gl = c / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2); + f_gl = f / 2 + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wr(r_gl, c_gl, f_gl) = res; + } + } + + if (!wrc.isNull()) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2)) * 2 + 1; + c_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2)) * 2 + 1; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2); + c_gl = c / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2); + f_gl = f / 2 + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = lerp(c1, c2, ratio_r_sm[r_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wrc(r_gl, c_gl, f_gl) = res; + } + } + } + } + + if (!wr.isNull() && c + (C / 2) * 2 == nc_p - 1 && + f + (F / 2) * 2 == nf_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 3 && + threadId < (R / 2) * (C / 2) * (F / 2) * 3 + (R / 2)) { + r_sm = (threadId - (R / 2) * (C / 2) * (F / 2) * 3) * 2 + 1; + c_sm = (C / 2) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + threadId - (R / 2) * (C / 2) * (F / 2) * 3; + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wr(r_gl, c_gl, f_gl) = res; + } + } + } + + if (!wc.isNull() && r + (R / 2) * 2 == nr_p - 1 && + f + (F / 2) * 2 == nf_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 4 && + threadId < (R / 2) * (C / 2) * (F / 2) * 4 + (C / 2)) { + r_sm = (R / 2) * 2; + c_sm = (threadId - (R / 2) * (C / 2) * (F / 2) * 4) * 2 + 1; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId - (R / 2) * (C / 2) * (F / 2) * 4; + f_gl = f / 2 + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wc(r_gl, c_gl, f_gl) = res; + } + } + } + + if (!wf.isNull() && r + (R / 2) * 2 == nr_p - 1 && + c + (C / 2) * 2 == nc_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 5 && + threadId < (R / 2) * (C / 2) * (F / 2) * 5 + (F / 2)) { + r_sm = (R / 2) * 2; + c_sm = (C / 2) * 2; + f_sm = (threadId - (R / 2) * (C / 2) * (F / 2) * 5) * 2 + 1; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + threadId - (R / 2) * (C / 2) * (F / 2) * 5; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + res = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] - res; + *wf(r_gl, c_gl, f_gl) = res; + // printf("!wf.isNull()(%d %d %d): %f\n", r_gl, c_gl, f_gl, + // !wf.isNull()[get_idx(lddwf1, lddwf2, r_gl, c_gl, f_gl)]); + } + } + } + + // if (r == 0 && c == 0 && f == 0 && threadId == 0) { + // printf("out config: %d %d %d (%d %d %d)\n", (R/2), (C/2), (F/2), + // r,c,f); for (int i = 0; i < (R/2) * 2 + 1; i++) { + // for (int j = 0; j < (C/2) * 2 + 1; j++) { + // for (int k = 0; k < (F/2) * 2 + 1; k++) { + // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + } + + MGARDX_EXEC void Operation4() {} + + MGARDX_EXEC void Operation5() {} + +private: + // functor parameters + SIZE nr, nc, nf, nr_c, nc_c, nf_c; + SubArray<1, T, DeviceType> ratio_r, ratio_c, ratio_f; + SubArray v, w, wf, wc, wr, wcf, wrf, wrc, wrcf; + + // thread local variables + SIZE r, c, f; + SIZE rest_r, rest_c, rest_f; + SIZE nr_p, nc_p, nf_p; + SIZE rest_r_p, rest_c_p, rest_f_p; + SIZE r_sm, c_sm, f_sm; + SIZE r_sm_ex, c_sm_ex, f_sm_ex; + SIZE r_gl, c_gl, f_gl; + SIZE r_gl_ex, c_gl_ex, f_gl_ex; + SIZE threadId; + T res; + T *sm; + SIZE ldsm1; + SIZE ldsm2; + T *v_sm; + T *ratio_f_sm; + T *ratio_c_sm; + T *ratio_r_sm; +}; + +template +class GpkReo3DKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "gpk_reo_3d"; + + MGARDX_CONT + GpkReo3DKernel(SIZE nr, SIZE nc, SIZE nf, SIZE nr_c, SIZE nc_c, SIZE nf_c, + SubArray<1, T, DeviceType> ratio_r, + SubArray<1, T, DeviceType> ratio_c, + SubArray<1, T, DeviceType> ratio_f, + SubArray v, SubArray w, + SubArray wf, SubArray wc, + SubArray wr, SubArray wcf, + SubArray wrf, SubArray wrc, + SubArray wrcf) + : nr(nr), nc(nc), nf(nf), nr_c(nr_c), nc_c(nc_c), nf_c(nf_c), + ratio_r(ratio_r), ratio_c(ratio_c), ratio_f(ratio_f), v(v), w(w), + wf(wf), wc(wc), wr(wr), wcf(wcf), wrf(wrf), wrc(wrc), wrcf(wrcf) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = GpkReo3DFunctor; + FunctorType functor(nr, nc, nf, nr_c, nc_c, nf_c, ratio_r, ratio_c, ratio_f, + v, w, wf, wc, wr, wcf, wrf, wrc, wrcf); + + SIZE total_thread_z = std::max(nr - 1, (SIZE)1); + SIZE total_thread_y = std::max(nc - 1, (SIZE)1); + SIZE total_thread_x = std::max(nf - 1, (SIZE)1); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size; + tbz = R; + tby = C; + tbx = F; + sm_size = ((R + 1) * (C + 1) * (F + 1) + R + C + F) * sizeof(T); + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE nr, nc, nf, nr_c, nc_c, nf_c; + SubArray<1, T, DeviceType> ratio_r, ratio_c, ratio_f; + SubArray v, w, wf, wc, wr, wcf, wrf, wrc, wrcf; +}; + +template +class GpkRev3DFunctor : public Functor { +public: + MGARDX_CONT GpkRev3DFunctor() {} + MGARDX_CONT GpkRev3DFunctor( + SIZE nr, SIZE nc, SIZE nf, SIZE nr_c, SIZE nc_c, SIZE nf_c, + SubArray<1, T, DeviceType> ratio_r, SubArray<1, T, DeviceType> ratio_c, + SubArray<1, T, DeviceType> ratio_f, SubArray v, + SubArray w, SubArray wf, + SubArray wc, SubArray wr, + SubArray wcf, SubArray wrf, + SubArray wrc, SubArray wrcf, SIZE svr, + SIZE svc, SIZE svf, SIZE nvr, SIZE nvc, SIZE nvf) + : nr(nr), nc(nc), nf(nf), nr_c(nr_c), nc_c(nc_c), nf_c(nf_c), + ratio_r(ratio_r), ratio_c(ratio_c), ratio_f(ratio_f), v(v), w(w), + wf(wf), wc(wc), wr(wr), wrcf(wrcf), wcf(wcf), wrf(wrf), wrc(wrc), + svr(svr), svc(svc), svf(svf), nvr(nvr), nvc(nvc), nvf(nvf) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + r = FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ(); + c = FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY(); + f = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX(); + + r_sm = FunctorBase::GetThreadIdZ(); + c_sm = FunctorBase::GetThreadIdY(); + f_sm = FunctorBase::GetThreadIdX(); + + r_sm_ex = (R / 2) * 2; + c_sm_ex = (C / 2) * 2; + f_sm_ex = (F / 2) * 2; + + sm = (T *)FunctorBase::GetSharedMemory(); + + ldsm1 = (F / 2) * 2 + 1; + ldsm2 = (C / 2) * 2 + 1; + v_sm = sm; + ratio_f_sm = sm + ((F / 2) * 2 + 1) * ((C / 2) * 2 + 1) * ((R / 2) * 2 + 1); + ratio_c_sm = ratio_f_sm + (F / 2) * 2; + ratio_r_sm = ratio_c_sm + (C / 2) * 2; + + rest_r = nr - r; + rest_c = nc - c; + rest_f = nf - f; + + nr_p = nr; + nc_p = nc; + nf_p = nf; + + threadId = (FunctorBase::GetThreadIdZ() * + (FunctorBase::GetBlockDimX() * + FunctorBase::GetBlockDimY())) + + (FunctorBase::GetThreadIdY() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + + rest_r_p = rest_r; + rest_c_p = rest_c; + rest_f_p = rest_f; + + if (nr % 2 == 0) { + nr_p = nr + 1; + rest_r_p = nr_p - r; + } + if (nc % 2 == 0) { + nc_p = nc + 1; + rest_c_p = nc_p - c; + } + if (nf % 2 == 0) { + nf_p = nf + 1; + rest_f_p = nf_p - f; + } + + // load dist + if (c_sm == 0 && f_sm == 0 && r_sm < rest_r - 2) { + ratio_r_sm[r_sm] = *ratio_r(r + r_sm); + if (nr % 2 == 0 && (R / 2) * 2 + 1 >= rest_r_p && r_sm == 0) { + ratio_r_sm[rest_r_p - 3] = 0.5; + } + } + if (r_sm == 0 && f_sm == 0 && c_sm < rest_c - 2) { + ratio_c_sm[c_sm] = *ratio_c(c + c_sm); + if (nc % 2 == 0 && (C / 2) * 2 + 1 >= rest_c_p && c_sm == 0) { + ratio_c_sm[rest_c_p - 3] = 0.5; + } + } + if (c_sm == 0 && r_sm == 0 && f_sm < rest_f - 2) { + ratio_f_sm[f_sm] = *ratio_f(f + f_sm); + if (nf % 2 == 0 && (F / 2) * 2 + 1 >= rest_f_p && f_sm == 0) { + ratio_f_sm[rest_f_p - 3] = 0.5; + } + } + } + + MGARDX_EXEC void Operation2() { + + if (!w.isNull() && threadId < (R / 2) * (C / 2) * (F / 2)) { + r_sm = (threadId / ((C / 2) * (F / 2))) * 2; + c_sm = ((threadId % ((C / 2) * (F / 2))) / (F / 2)) * 2; + f_sm = ((threadId % ((C / 2) * (F / 2))) % (F / 2)) * 2; + r_gl = r / 2 + threadId / ((C / 2) * (F / 2)); + c_gl = c / 2 + threadId % ((C / 2) * (F / 2)) / (F / 2); + f_gl = f / 2 + threadId % ((C / 2) * (F / 2)) % (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (c_gl == nc_c - 1 && f_gl == nf_c-1) + // printf("block: (%d %d %d) thread: (%d %d %d) load0 (%d %d %d): %f + // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, + // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], + // r_gl, c_gl, f_gl); + } + } + + int base = 0; + if (!w.isNull() && threadId >= base && + threadId < base + (C / 2) * (F / 2)) { + r_sm = (R / 2) * 2; + c_sm = ((threadId - base) / (F / 2)) * 2; + f_sm = ((threadId - base) % (F / 2)) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (threadId - base) / (F / 2); + f_gl = f / 2 + (threadId - base) % (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (c_gl == nc_c - 1 && f_gl == nf_c-1) + // printf("block: (%d %d %d) thread: (%d %d %d) load1 (%d %d %d): %f + // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, + // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], + // r_gl, c_gl, f_gl); + } + } + base += (C / 2) * (F / 2); // ROUND_UP_WARP((C/2) * (F/2)) * WARP_SIZE; + if (!w.isNull() && threadId >= base && + threadId < base + (R / 2) * (F / 2)) { + r_sm = ((threadId - base) / (F / 2)) * 2; + c_sm = (C / 2) * 2; + f_sm = ((threadId - base) % (F / 2)) * 2; + r_gl = r / 2 + (threadId - base) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - base) % (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (c_gl == nc_c - 1 && f_gl == nf_c-1) + // printf("block: (%d %d %d) thread: (%d %d %d) load2 (%d %d %d): %f + // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, + // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], + // r_gl, c_gl, f_gl); + } + } + base += (R / 2) * (F / 2); // ROUND_UP_WARP((R/2) * (F/2)) * WARP_SIZE; + if (!w.isNull() && threadId >= base && + threadId < base + (R / 2) * (C / 2)) { + r_sm = ((threadId - base) / (C / 2)) * 2; + c_sm = ((threadId - base) % (C / 2)) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - base) / (C / 2); + c_gl = c / 2 + (threadId - base) % (C / 2); + f_gl = f / 2 + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (c_gl == nc_c - 1 && f_gl == nf_c-1) + // printf("block: (%d %d %d) thread: (%d %d %d) load3 (%d %d %d): %f + // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, + // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], + // r_gl, c_gl, f_gl); + } + } + base += (R / 2) * (C / 2); // ROUND_UP_WARP((R/2) * (C/2)) * WARP_SIZE; + // load extra edges + if (!w.isNull() && threadId >= base && threadId < base + (R / 2)) { + r_sm = (threadId - base) * 2; + c_sm = (C / 2) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + threadId - base; + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (c_gl == nc_c - 1 && f_gl == nf_c-1) + // printf("block: (%d %d %d) thread: (%d %d %d) load4 (%d %d %d): %f + // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, + // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], + // r_gl, c_gl, f_gl); + } + } + base += (R / 2); // ROUND_UP_WARP((R/2)) * WARP_SIZE; + if (!w.isNull() && threadId >= base && threadId < base + (C / 2)) { + r_sm = (R / 2) * 2; + c_sm = (threadId - base) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId - base; + f_gl = f / 2 + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (c_gl == nc_c - 1 && f_gl == nf_c-1) + // printf("block: (%d %d %d) thread: (%d %d %d) load5 (%d %d %d): %f + // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, + // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], + // r_gl, c_gl, f_gl); + } + } + base += (C / 2); // ROUND_UP_WARP((C/2)) * WARP_SIZE; + if (!w.isNull() && threadId >= base && threadId < base + (F / 2)) { + r_sm = (R / 2) * 2; + c_sm = (C / 2) * 2; + f_sm = (threadId - base) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + threadId - base; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (c_gl == nc_c - 1 && f_gl == nf_c-1) + // printf("block: (%d %d %d) thread: (%d %d %d) load6 (%d %d %d): %f + // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, + // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], + // r_gl, c_gl, f_gl); + } + } + base += (F / 2); // ROUND_UP_WARP((F/2)) * WARP_SIZE; + // // load extra vertex + if (!w.isNull() && threadId >= base && threadId < base + 1) { + r_sm = (R / 2) * 2; + c_sm = (C / 2) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = *w(r_gl, c_gl, f_gl); + // if (c_gl == nc_c - 1 && f_gl == nf_c-1) + // printf("block: (%d %d %d) thread: (%d %d %d) load7 (%d %d %d): %f + // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, + // threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], + // r_gl, c_gl, f_gl); + } + } + + // __syncthreads(); + // if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) { + // printf("rest_p: %u %u %u RCF %u %u %u\n", rest_r_p, rest_c_p, rest_f_p, + // R, C, F); + + // for (int i = 0; i < min(rest_r_p, (R/2) * 2 + 1); i++) { + // for (int j = 0; j < min(rest_c_p, (C/2) * 2 + 1); j++) { + // for (int k = 0; k < min(rest_f_p, (F/2) * 2 + 1); k++) { + // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + } + + MGARDX_EXEC void Operation3() { + if (!wf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) && + threadId < (R / 2) * (C / 2) * (F / 2) * 2) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2)) / ((C / 2) * (F / 2))) * 2; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) / + (F / 2)) * + 2; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2)) / ((C / 2) * (F / 2)); + c_gl = c / 2 + + ((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + + ((threadId - (R / 2) * (C / 2) * (F / 2)) % ((C / 2) * (F / 2))) % + (F / 2); + + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + + res = *wf(r_gl, c_gl, f_gl); + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + + if (!wc.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 2 && + threadId < (R / 2) * (C / 2) * (F / 2) * 3) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / ((C / 2) * (F / 2))) * + 2; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % + ((C / 2) * (F / 2))) % + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = *wc(r_gl, c_gl, f_gl); + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + + if (!wr.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 3 && + threadId < (R / 2) * (C / 2) * (F / 2) * 4) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 3) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 3) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 3) % + ((C / 2) * (F / 2))) % + (F / 2); + + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + res = *wr(r_gl, c_gl, f_gl); + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + // if (c_gl == nc_c-1 && f_gl == nf_c - 1) + // printf("block: (%d %d %d) thread: (%d %d %d) calc_coeff0 (%d + // %d %d): %f <- %f %f\n", blockIdx.z, blockIdx.y, blockIdx.x, + // threadIdx.z, threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, + // res, v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, + // f_sm)], + // v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + + if (!wcf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 4 && + threadId < (R / 2) * (C / 2) * (F / 2) * 5) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 4) / ((C / 2) * (F / 2))) * + 2; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 4) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 4) % + ((C / 2) * (F / 2))) % + (F / 2); + + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { + res = *wcf(r_gl, c_gl, f_gl); + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res += lerp(f1, f2, ratio_c_sm[c_sm - 1]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + + if (!wrf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 5 && + threadId < (R / 2) * (C / 2) * (F / 2) * 6) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 5) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 5) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 5) % + ((C / 2) * (F / 2))) % + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + res = *wrf(r_gl, c_gl, f_gl); + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res += lerp(f1, f2, ratio_r_sm[r_sm - 1]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + + if (!wrc.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 6 && + threadId < (R / 2) * (C / 2) * (F / 2) * 7) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 6) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 6) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 6) % + ((C / 2) * (F / 2))) % + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = *wrc(r_gl, c_gl, f_gl); + T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res += lerp(c1, c2, ratio_r_sm[r_sm - 1]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + + if (!wrcf.isNull() && threadId >= (R / 2) * (C / 2) * (F / 2) * 7 && + threadId < (R / 2) * (C / 2) * (F / 2) * 8) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 7) / ((C / 2) * (F / 2))) * + 2 + + 1; + c_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) / + (F / 2)) * + 2 + + 1; + f_sm = (((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) % + (F / 2)) * + 2 + + 1; + r_gl = r / 2 + + (threadId - (R / 2) * (C / 2) * (F / 2) * 7) / ((C / 2) * (F / 2)); + c_gl = c / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) / + (F / 2); + f_gl = f / 2 + ((threadId - (R / 2) * (C / 2) * (F / 2) * 7) % + ((C / 2) * (F / 2))) % + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { + res = *wrcf(r_gl, c_gl, f_gl); + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f3 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f4 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + + T fc1 = lerp(f1, f2, ratio_c_sm[c_sm - 1]); + T fc2 = lerp(f3, f4, ratio_c_sm[c_sm - 1]); + + res += lerp(fc1, fc2, ratio_r_sm[r_sm - 1]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + + if (r + (R / 2) * 2 == nr_p - 1) { + if (threadId < (C / 2) * (F / 2)) { + if (!wf.isNull()) { + r_sm = (R / 2) * 2; + c_sm = (threadId / (F / 2)) * 2; + f_sm = (threadId % (F / 2)) * 2 + 1; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId / (F / 2); + f_gl = f / 2 + threadId % (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + res = *wf(r_gl, c_gl, f_gl); + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + + if (!wc.isNull()) { + r_sm = (R / 2) * 2; + c_sm = (threadId / (F / 2)) * 2 + 1; + f_sm = (threadId % (F / 2)) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId / (F / 2); + f_gl = f / 2 + threadId % (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = *wc(r_gl, c_gl, f_gl); + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + if (!wcf.isNull()) { + r_sm = (R / 2) * 2; + c_sm = (threadId / (F / 2)) * 2 + 1; + f_sm = (threadId % (F / 2)) * 2 + 1; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId / (F / 2); + f_gl = f / 2 + threadId % (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf - nf_c) { + res = *wcf(r_gl, c_gl, f_gl); + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res += lerp(f1, f2, ratio_c_sm[c_sm - 1]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + } + + if (c + (C / 2) * 2 == nc_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) && + threadId < (R / 2) * (C / 2) * (F / 2) + (R / 2) * (F / 2)) { + if (!wf.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2)) * 2; + c_sm = (C / 2) * 2; + f_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2)) * 2 + 1; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + res = *wf(r_gl, c_gl, f_gl); + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + if (!wr.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2)) * 2 + 1; + c_sm = (C / 2) * 2; + f_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2)) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + res = *wr(r_gl, c_gl, f_gl); + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + // if (c_gl == nc_c-1 && f_gl == nf_c - 1) + // printf("block: (%d %d %d) thread: (%d %d %d) calc_coeff1 (%d + // %d %d): %f <- %f %f\n", blockIdx.z, blockIdx.y, blockIdx.x, + // threadIdx.z, threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, + // res, v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, + // f_sm)], + // v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + if (!wrf.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2)) * 2 + 1; + c_sm = (C / 2) * 2; + f_sm = ((threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2)) * 2 + 1; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) / (F / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (threadId - (R / 2) * (C / 2) * (F / 2)) % (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + res = *wrf(r_gl, c_gl, f_gl); + T f1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + T f2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + res += lerp(f1, f2, ratio_r_sm[r_sm - 1]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + } + + if (f + (F / 2) * 2 == nf_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 2 && + threadId < (R / 2) * (C / 2) * (F / 2) * 2 + (R / 2) * (C / 2)) { + if (!wc.isNull()) { + r_sm = ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2)) * 2; + c_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2)) * 2 + 1; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2); + c_gl = c / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2); + f_gl = f / 2 + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = *wc(r_gl, c_gl, f_gl); + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + + if (!wr.isNull()) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2)) * 2 + 1; + c_sm = ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2)) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2); + c_gl = c / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2); + f_gl = f / 2 + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + res = *wr(r_gl, c_gl, f_gl); + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + // if (c_gl == nc_c-1 && f_gl == nf_c - 1) + // printf("block: (%d %d %d) thread: (%d %d %d) calc_coeff2 (%d + // %d %d): %f <- %f %f\n", blockIdx.z, blockIdx.y, blockIdx.x, + // threadIdx.z, threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, + // res, v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, + // f_sm)], + // v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + + if (!wrc.isNull()) { + r_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2)) * 2 + 1; + c_sm = + ((threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2)) * 2 + 1; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) / (C / 2); + c_gl = c / 2 + (threadId - (R / 2) * (C / 2) * (F / 2) * 2) % (C / 2); + f_gl = f / 2 + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = *wrc(r_gl, c_gl, f_gl); + T c1 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + T c2 = lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + res += lerp(c1, c2, ratio_r_sm[r_sm - 1]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + } + + if (c + (C / 2) * 2 == nc_p - 1 && f + (F / 2) * 2 == nf_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 3 && + threadId < (R / 2) * (C / 2) * (F / 2) * 3 + (R / 2)) { + if (!wr.isNull()) { + r_sm = (threadId - (R / 2) * (C / 2) * (F / 2) * 3) * 2 + 1; + c_sm = (C / 2) * 2; + f_sm = (F / 2) * 2; + r_gl = r / 2 + threadId - (R / 2) * (C / 2) * (F / 2) * 3; + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr - nr_c && c_gl < nc_c && f_gl < nf_c) { + res = *wr(r_gl, c_gl, f_gl); + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)], + ratio_r_sm[r_sm - 1]); + // if (c_gl == nc_c-1 && f_gl == nf_c - 1) + // printf("block: (%d %d %d) thread: (%d %d %d) calc_coeff3 (%d + // %d %d): %f <- %f %f\n", blockIdx.z, blockIdx.y, blockIdx.x, + // threadIdx.z, threadIdx.y, threadIdx.x, r_sm, c_sm, f_sm, + // res, v_sm[get_idx(ldsm1, ldsm2, r_sm - 1, c_sm, + // f_sm)], + // v_sm[get_idx(ldsm1, ldsm2, r_sm + 1, c_sm, f_sm)]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + } + + if (r + (R / 2) * 2 == nr_p - 1 && f + (F / 2) * 2 == nf_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 4 && + threadId < (R / 2) * (C / 2) * (F / 2) * 4 + (C / 2)) { + if (!wc.isNull()) { + r_sm = (R / 2) * 2; + c_sm = (threadId - (R / 2) * (C / 2) * (F / 2) * 4) * 2 + 1; + f_sm = (F / 2) * 2; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + threadId - (R / 2) * (C / 2) * (F / 2) * 4; + f_gl = f / 2 + (F / 2); + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc - nc_c && f_gl < nf_c) { + res = *wc(r_gl, c_gl, f_gl); + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm - 1, f_sm)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm + 1, f_sm)], + ratio_c_sm[c_sm - 1]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + } + + if (r + (R / 2) * 2 == nr_p - 1 && c + (C / 2) * 2 == nc_p - 1) { + if (threadId >= (R / 2) * (C / 2) * (F / 2) * 5 && + threadId < (R / 2) * (C / 2) * (F / 2) * 5 + (F / 2)) { + if (!wf.isNull()) { + r_sm = (R / 2) * 2; + c_sm = (C / 2) * 2; + f_sm = (threadId - (R / 2) * (C / 2) * (F / 2) * 5) * 2 + 1; + r_gl = r / 2 + (R / 2); + c_gl = c / 2 + (C / 2); + f_gl = f / 2 + threadId - (R / 2) * (C / 2) * (F / 2) * 5; + if (r_sm < rest_r_p && c_sm < rest_c_p && f_sm < rest_f_p && + r_gl < nr_c && c_gl < nc_c && f_gl < nf - nf_c) { + res = *wf(r_gl, c_gl, f_gl); + res += lerp(v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm - 1)], + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm + 1)], + ratio_f_sm[f_sm - 1]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)] = res; + } + } + } + } + + // __syncthreads(); + // if (debug) { + // printf("TYPE: %d %d %d %d\n", TYPE, + // min(rest_r_p, (R/2) * 2 + 1), + // min(rest_c_p, (C/2) * 2 + 1), + // min(rest_f_p, (F/2) * 2 + 1)); + // for (int i = 0; i < min(rest_r_p, (R/2) * 2 + 1); i++) { + // for (int j = 0; j < min(rest_c_p, (C/2) * 2 + 1); j++) { + // for (int k = 0; k < min(rest_f_p, (F/2) * 2 + 1); k++) { + // printf("%2.2f ", v_sm[get_idx(ldsm1, ldsm2, i, j, k)]); + // } + // printf("\n"); + // } + // printf("\n"); + // } + // } + // __syncthreads(); + } + + MGARDX_EXEC void Operation4() { + r_sm = FunctorBase::GetThreadIdZ(); + c_sm = FunctorBase::GetThreadIdY(); + f_sm = FunctorBase::GetThreadIdX(); + + r_sm_ex = FunctorBase::GetBlockDimZ(); + c_sm_ex = FunctorBase::GetBlockDimY(); + f_sm_ex = FunctorBase::GetBlockDimX(); + + r_gl = r + r_sm; + c_gl = c + c_sm; + f_gl = f + f_sm; + + r_gl_ex = r + rest_r - 1; + c_gl_ex = c + rest_c - 1; + f_gl_ex = f + rest_f - 1; + + int unpadding_r = rest_r; + int unpadding_c = rest_c; + int unpadding_f = rest_f; + if (nr % 2 == 0) + unpadding_r -= 1; + if (nc % 2 == 0) + unpadding_c -= 1; + if (nf % 2 == 0) + unpadding_f -= 1; + + if (r_sm < unpadding_r && c_sm < unpadding_c && f_sm < unpadding_f) { + + // store extra rules + // case 1: input = odd (non-padding required) + // case 1.a: block size + 1 == rest (need to store extra); + // case 1.b: block size + 1 != rest (No need to store extra); + // case 2: input = even (un-padding requried) + // case 2.a: block size + 1 >= rest (No need to store extra, but need + // un-padding first); case 2.b: block size + 1 < rest (No need to store + // extra); + + if (D >= 3 && r_sm == 0) { + if (nr % 2 != 0 && (R / 2) * 2 + 1 == rest_r) { + *v(r_gl_ex, c_gl, f_gl) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm)]; + } + if (nr % 2 == 0 && (R / 2) * 2 + 1 >= rest_r_p) { + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm)]; + // if ( v_sm[get_idx(ldsm1, ldsm2, rest_r_p-1, c_sm, f_sm)] == + // 71177117) printf("un-padding0 error block: (%d %d %d) thread: (%d + // %d %d) un-padding (%d %d %d) %f (%d %d %d)\n", blockIdx.z, + // blockIdx.y, blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, + // rest_r-1, c_sm, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, rest_r_p-1, c_sm, f_sm)], + // rest_r_p-1, c_sm, f_sm); + } + } + + if (D >= 2 && c_sm == 0) { + if (nc % 2 != 0 && (C / 2) * 2 + 1 == rest_c) { + *v(r_gl, c_gl_ex, f_gl) = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm)]; + } + if (nc % 2 == 0 && (C / 2) * 2 + 1 >= rest_c_p) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm)]; + // if (v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm)] == + // 71177117) + // printf("un-padding1 error block: (%d %d %d) thread: (%d %d %d) " + // "un-padding (%d %d %d) %f (%d %d %d)\n", + // blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, + // threadIdx.y, threadIdx.x, r_sm, rest_c - 1, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm)], + // r_sm, rest_c_p - 1, f_sm); + } + } + + if (D >= 1 && f_sm == 0) { + if (nf % 2 != 0 && (F / 2) * 2 + 1 == rest_f) { + *v(r_gl, c_gl, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm_ex)]; + } + if (nf % 2 == 0 && (F / 2) * 2 + 1 >= rest_f_p) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f_p - 1)]; + // if ( v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f_p-1)] == + // 71177117) printf("un-padding2 error block: (%d %d %d) thread: (%d + // %d %d) un-padding (%d %d %d) %f (%d %d %d)\n", blockIdx.z, + // blockIdx.y, blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, + // r_sm, c_sm, rest_f-1, + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f_p-1)], r_sm, + // c_sm, rest_f_p-1); + } + } + + // load extra edges + if (D >= 2 && c_sm == 0 && f_sm == 0) { + if (nc % 2 != 0 && (C / 2) * 2 + 1 == rest_c && nf % 2 != 0 && + (F / 2) * 2 + 1 == rest_f) { + *v(r_gl, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm_ex)]; + } + if (nc % 2 == 0 && nf % 2 == 0 && (C / 2) * 2 + 1 >= rest_c_p && + (F / 2) * 2 + 1 >= rest_f_p) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, rest_f - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, rest_f_p - 1)]; + // printf("block: (%d %d %d) thread: (%d %d %d) un-padding (%d %d %d) + // %f + // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, + // threadIdx.y, threadIdx.x, r_sm, rest_c-1, rest_f-1, + // v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c-1, rest_f-1)], r_sm, + // rest_c_p-1, rest_f_p-1); + } + if (nc % 2 == 0 && nf % 2 != 0 && (C / 2) * 2 + 1 >= rest_c_p && + (F / 2) * 2 + 1 == rest_f) { + *v(r_gl, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm_ex)]; + } + if (nc % 2 != 0 && nf % 2 == 0 && (C / 2) * 2 + 1 == rest_c && + (F / 2) * 2 + 1 >= rest_f_p) { + *v(r_gl, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f_p - 1)]; + // printf("(%d %d %d): %f <- (%d %d %d)\n", + // r_gl, c_gl_ex, f_gl_ex, + // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)], + // r_sm, c_sm_ex, f_gl_ex); + } + } + + if (D >= 3 && r_sm == 0 && f_sm == 0) { + if (nr % 2 != 0 && (R / 2) * 2 + 1 == rest_r && nf % 2 != 0 && + (F / 2) * 2 + 1 == rest_f) { + *v(r_gl_ex, c_gl, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm_ex)]; + } + if (nr % 2 == 0 && nf % 2 == 0 && (R / 2) * 2 + 1 >= rest_r_p && + (F / 2) * 2 + 1 >= rest_f_p) { + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, rest_f - 1)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, rest_f_p - 1)]; + // if ( v_sm[get_idx(ldsm1, ldsm2, rest_r_p-1, c_sm, rest_f_p-1)] == + // 71177117) printf("un-padding3 error block: (%d %d %d) thread: (%d + // %d %d) un-padding (%d %d %d) %f (%d %d %d)\n", blockIdx.z, + // blockIdx.y, blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, + // rest_r-1, c_sm, rest_f-1, + // v_sm[get_idx(ldsm1, ldsm2, rest_r_p-1, c_sm, rest_f_p-1)], + // rest_r_p-1, c_sm, rest_f_p-1); + } + if (nr % 2 == 0 && nf % 2 != 0 && (R / 2) * 2 + 1 >= rest_r_p && + (F / 2) * 2 + 1 == rest_f) { + *v(r_gl_ex, c_gl, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm_ex)]; + } + if (nr % 2 != 0 && nf % 2 == 0 && (R / 2) * 2 + 1 == rest_r && + (F / 2) * 2 + 1 >= rest_f_p) { + *v(r_gl_ex, c_gl, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f_p - 1)]; + // printf("(%d %d %d): %f <- (%d %d %d)\n", + // r_gl_ex, c_gl, rest_f-1, + // dv[get_idx(lddv1, lddv2, r_gl_ex-1, c_gl, f_gl_ex)], + // r_sm_ex, c_sm, rest_f_p-1); + } + } + + if (D >= 3 && r_sm == 0 && c_sm == 0) { + if (nr % 2 != 0 && (R / 2) * 2 + 1 == rest_r && nc % 2 != 0 && + (C / 2) * 2 + 1 == rest_c) { + *v(r_gl_ex, c_gl_ex, f_gl) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm)]; + } + if (nr % 2 == 0 && nc % 2 == 0 && (R / 2) * 2 + 1 >= rest_r_p && + (C / 2) * 2 + 1 >= rest_c_p) { + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm)]; + // if ( v_sm[get_idx(ldsm1, ldsm2, rest_r_p-1, rest_c_p-1, f_sm)] == + // 71177117) printf("un-padding4 error block: (%d %d %d) thread: (%d + // %d %d) un-padding (%d %d %d) %f (%d %d %d)\n", blockIdx.z, + // blockIdx.y, blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, + // rest_r-1, rest_c-1, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, rest_r_p-1, rest_c_p-1, f_sm)], + // rest_r_p-1, rest_c_p-1, f_sm); + } + if (nr % 2 == 0 && nc % 2 != 0 && (R / 2) * 2 + 1 >= rest_r_p && + (C / 2) * 2 + 1 == rest_c) { + *v(r_gl_ex, c_gl_ex, f_gl) = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm)]; + } + if (nr % 2 != 0 && nc % 2 == 0 && (R / 2) * 2 + 1 == rest_r && + (C / 2) * 2 + 1 >= rest_c_p) { + *v(r_gl_ex, c_gl_ex, f_gl) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm)]; + } + } + // load extra vertex + + if (D >= 3 && r_sm == 0 && c_sm == 0 && f_sm == 0) { + if (nr % 2 != 0 && (R / 2) * 2 + 1 == rest_r && nc % 2 != 0 && + (C / 2) * 2 + 1 == rest_c && nf % 2 != 0 && + (F / 2) * 2 + 1 == rest_f) { + *v(r_gl_ex, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm_ex)]; + } + + if (nr % 2 == 0 && nc % 2 == 0 && nf % 2 == 0 && + (R / 2) * 2 + 1 >= rest_r_p && (C / 2) * 2 + 1 >= rest_c_p && + (F / 2) * 2 + 1 >= rest_f_p) { + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, rest_f - 1)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, + rest_f_p - 1)]; + + // printf("block: (%d %d %d) thread: (%d %d %d) un-padding (%d %d %d) + // %f + // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, + // threadIdx.y, threadIdx.x, rest_r-1, rest_c-1, rest_f-1, + // v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c-1, rest_f-1)], + // rest_r_p-1, rest_c_p-1, rest_f_p-1); + } + if (nr % 2 == 0 && nc % 2 == 0 && nf % 2 != 0 && + (R / 2) * 2 + 1 >= rest_r_p && (C / 2) * 2 + 1 >= rest_c_p && + (F / 2) * 2 + 1 == rest_f) { + *v(r_gl_ex, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm_ex)]; + } + if (nr % 2 == 0 && nc % 2 != 0 && nf % 2 == 0 && + (R / 2) * 2 + 1 >= rest_r_p && (C / 2) * 2 + 1 == rest_c && + (F / 2) * 2 + 1 >= rest_f_p) { + *v(r_gl_ex, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, rest_f_p - 1)]; + } + if (nr % 2 != 0 && nc % 2 == 0 && nf % 2 == 0 && + (R / 2) * 2 + 1 == rest_r && (C / 2) * 2 + 1 >= rest_c_p && + (F / 2) * 2 + 1 >= rest_f_p) { + *v(r_gl_ex, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, rest_f_p - 1)]; + } + if (nr % 2 == 0 && nc % 2 != 0 && nf % 2 != 0 && + (R / 2) * 2 + 1 >= rest_r_p && (C / 2) * 2 + 1 == rest_c && + (F / 2) * 2 + 1 == rest_f) { + *v(r_gl_ex, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm_ex)]; + } + if (nr % 2 != 0 && nc % 2 == 0 && nf % 2 != 0 && + (R / 2) * 2 + 1 == rest_r && (C / 2) * 2 + 1 >= rest_c_p && + (F / 2) * 2 + 1 == rest_f) { + *v(r_gl_ex, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm_ex)]; + } + if (nr % 2 != 0 && nc % 2 != 0 && nf % 2 == 0 && + (R / 2) * 2 + 1 == rest_r && (C / 2) * 2 + 1 == rest_c && + (F / 2) * 2 + 1 >= rest_f_p) { + *v(r_gl_ex, c_gl_ex, f_gl_ex) = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f_p - 1)]; + } + } + } + } + + MGARDX_EXEC void Operation5() { + if (r_sm < rest_r && c_sm < rest_c && f_sm < rest_f) { + if (r_gl >= svr && r_gl < svr + nvr && c_gl >= svc && c_gl < svc + nvc && + f_gl >= svf && f_gl < svf + nvf) { + *v(r_gl, c_gl, f_gl) = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)]; + + // if (c_gl == nc - 1 && f_gl == nf - 1) { + // printf("block: (%d %d %d) thread: (%d %d %d) store (%d %d %d) %f + // (%d %d %d)\n", blockIdx.z, blockIdx.y, blockIdx.x, threadIdx.z, + // threadIdx.y, threadIdx.x, r_gl, c_gl, f_gl, + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm)], r_sm, c_sm, f_sm); + // } + } + } + } + +private: + // functor parameters + SIZE nr, nc, nf, nr_c, nc_c, nf_c; + SubArray<1, T, DeviceType> ratio_r, ratio_c, ratio_f; + SubArray v, w, wf, wc, wr, wcf, wrf, wrc, wrcf; + SIZE svr, svc, svf, nvr, nvc, nvf; + + // thread local variables + SIZE r, c, f; + SIZE rest_r, rest_c, rest_f; + SIZE nr_p, nc_p, nf_p; + SIZE rest_r_p, rest_c_p, rest_f_p; + SIZE r_sm, c_sm, f_sm; + SIZE r_sm_ex, c_sm_ex, f_sm_ex; + SIZE r_gl, c_gl, f_gl; + SIZE r_gl_ex, c_gl_ex, f_gl_ex; + SIZE threadId; + T res; + T *sm; + SIZE ldsm1; + SIZE ldsm2; + T *v_sm; + T *ratio_f_sm; + T *ratio_c_sm; + T *ratio_r_sm; +}; + +template +class GpkRev3DKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "gpk_rev_3d"; + MGARDX_CONT + GpkRev3DKernel(SIZE nr, SIZE nc, SIZE nf, SIZE nr_c, SIZE nc_c, SIZE nf_c, + SubArray<1, T, DeviceType> ratio_r, + SubArray<1, T, DeviceType> ratio_c, + SubArray<1, T, DeviceType> ratio_f, + SubArray v, SubArray w, + SubArray wf, SubArray wc, + SubArray wr, SubArray wcf, + SubArray wrf, SubArray wrc, + SubArray wrcf, SIZE svr, SIZE svc, SIZE svf, + SIZE nvr, SIZE nvc, SIZE nvf) + : nr(nr), nc(nc), nf(nf), nr_c(nr_c), nc_c(nc_c), nf_c(nf_c), + ratio_r(ratio_r), ratio_c(ratio_c), ratio_f(ratio_f), v(v), w(w), + wf(wf), wc(wc), wr(wr), wrcf(wrcf), wcf(wcf), wrf(wrf), wrc(wrc), + svr(svr), svc(svc), svf(svf), nvr(nvr), nvc(nvc), nvf(nvf) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = GpkRev3DFunctor; + FunctorType functor(nr, nc, nf, nr_c, nc_c, nf_c, ratio_r, ratio_c, ratio_f, + v, w, wf, wc, wr, wcf, wrf, wrc, wrcf, svr, svc, svf, + nvr, nvc, nvf); + + SIZE total_thread_z = std::max(nr - 1, (SIZE)1); + SIZE total_thread_y = std::max(nc - 1, (SIZE)1); + SIZE total_thread_x = std::max(nf - 1, (SIZE)1); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size; + tbz = R; + tby = C; + tbx = F; + sm_size = ((R + 1) * (C + 1) * (F + 1) + R + C + F) * sizeof(T); + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + // functor parameters + SIZE nr, nc, nf, nr_c, nc_c, nf_c; + SubArray<1, T, DeviceType> ratio_r, ratio_c, ratio_f; + SubArray v, w, wf, wc, wr, wcf, wrf, wrc, wrcf; + SIZE svr, svc, svf, nvr, nvc, nvf; +}; + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/AddND.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/AddND.hpp new file mode 100644 index 000000000..75b0b1470 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/AddND.hpp @@ -0,0 +1,38 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../../Hierarchy/Hierarchy.h" +#include "../../../RuntimeX/RuntimeX.h" + +#include "../DataRefactoring.h" + +#include "LevelwiseProcessingKernel.hpp" + +#ifndef MGARD_X_160_DATA_REFACTORING_ADD_ND +#define MGARD_X_160_DATA_REFACTORING_ADD_ND + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +void AddND(SubArray dinput, + SubArray doutput, int queue_idx) { + + DeviceLauncher::Execute( + LwpkReoKernel(dinput, doutput), queue_idx); +} + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/Copy3D.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/Copy3D.hpp new file mode 100644 index 000000000..0a176b9fd --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/Copy3D.hpp @@ -0,0 +1,49 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../../Hierarchy/Hierarchy.h" +#include "../../../RuntimeX/RuntimeX.h" + +#include "../DataRefactoring.h" + +#include "LevelwiseProcessingKernel.hpp" + +#ifndef MGARD_X_160_DATA_REFACTORING_COPY_ND +#define MGARD_X_160_DATA_REFACTORING_COPY_ND + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +void Copy3D(SubArray dinput, + SubArray doutput, bool padding, int queue_idx) { + + SIZE nf = dinput.shape(D - 1); + SIZE nc = D >= 2 ? dinput.shape(D - 2) : 1; + SIZE nr = D >= 3 ? dinput.shape(D - 3) : 1; + if (padding) { + DeviceLauncher::Execute( + Lwpk3DKernel(nf, nc, nr, dinput, doutput), + queue_idx); + } else { + DeviceLauncher::Execute( + Lwpk3DKernel(nf, nc, nr, dinput, + doutput), + queue_idx); + } +} + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/CopyND.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/CopyND.hpp new file mode 100644 index 000000000..d8d3e836d --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/CopyND.hpp @@ -0,0 +1,38 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../../Hierarchy/Hierarchy.h" +#include "../../../RuntimeX/RuntimeX.h" + +#include "../DataRefactoring.h" + +#include "LevelwiseProcessingKernel.hpp" + +#ifndef MGARD_X_160_DATA_REFACTORING_COPY_ND +#define MGARD_X_160_DATA_REFACTORING_COPY_ND + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +void CopyND(SubArray dinput, + SubArray doutput, int queue_idx) { + + DeviceLauncher::Execute( + LwpkReoKernel(dinput, doutput), queue_idx); +} + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/LevelwiseProcessingKernel.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/LevelwiseProcessingKernel.hpp new file mode 100644 index 000000000..f272a1abb --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/LevelwiseProcessingKernel.hpp @@ -0,0 +1,376 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_LEVELWISE_PROCESSING_KERNEL_TEMPLATE +#define MGARD_X_160_LEVELWISE_PROCESSING_KERNEL_TEMPLATE + +#include "../../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +class LwpkFunctor : public Functor { +public: + MGARDX_CONT LwpkFunctor() {} + MGARDX_CONT LwpkFunctor(SubArray v, + SubArray work) + : v(v), work(work) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + threadId = (FunctorBase::GetThreadIdZ() * + (FunctorBase::GetBlockDimX() * + FunctorBase::GetBlockDimY())) + + (FunctorBase::GetThreadIdY() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + + SIZE idx[D]; + SIZE firstD = div_roundup(v.shape(D - 1), F); + + SIZE bidx = FunctorBase::GetBlockIdX(); + idx[D - 1] = (bidx % firstD) * F + FunctorBase::GetThreadIdX(); + + bidx /= firstD; + if (D >= 2) + idx[D - 2] = FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY() + + FunctorBase::GetThreadIdY(); + if (D >= 3) + idx[D - 3] = FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ() + + FunctorBase::GetThreadIdZ(); + + for (int d = D - 4; d >= 0; d--) { + idx[d] = bidx % v.shape(d); + bidx /= v.shape(d); + } + bool in_range = true; + for (DIM d = 0; d < D; d++) { + if (idx[d] >= v.shape(d)) + in_range = false; + } + if (in_range) { + // printf("%d %d %d %d\n", idx[3], idx[2], idx[1], idx[0]); + if (OP == COPY) { + // *work(idx) = *v(idx); + work[idx] = v[idx]; + } + if (OP == ADD) { + work[idx] += v[idx]; + } + if (OP == SUBTRACT) { + work[idx] -= v[idx]; + } + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + SubArray v, work; + IDX threadId; +}; + +template +class LwpkReoKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "lwpk"; + MGARDX_CONT + LwpkReoKernel(SubArray v, SubArray work) + : v(v), work(work) {} + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = LwpkFunctor; + FunctorType functor(v, work); + + SIZE total_thread_z = 1; + SIZE total_thread_y = 1; + SIZE total_thread_x = 1; + if (D >= 3) + total_thread_z = v.shape(D - 3); + if (D >= 2) + total_thread_y = v.shape(D - 2); + total_thread_x = v.shape(D - 1); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = R; + tby = C; + tbx = F; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + for (int d = D - 4; d >= 0; d--) { + gridx *= v.shape(d); + } + // printf("%u %u %u\n", shape.dataHost()[2], shape.dataHost()[1], + // shape.dataHost()[0]); PrintSubarray("shape", shape); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray v, work; +}; + +template +class Lwpk3DFunctor : public Functor { +public: + MGARDX_CONT Lwpk3DFunctor() {} + MGARDX_CONT Lwpk3DFunctor(SIZE nr, SIZE nc, SIZE nf, + SubArray v, + SubArray work) + : nr(nr), nc(nc), nf(nf), v(v), work(work) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + f_gl = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + if constexpr (D >= 2) + c_gl = FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY() + + FunctorBase::GetThreadIdY(); + if constexpr (D >= 3) + r_gl = FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ() + + FunctorBase::GetThreadIdZ(); + + if (r_gl < nr && c_gl < nc && f_gl < nf) { + // printf("%d %d %d %d\n", idx[3], idx[2], idx[1], idx[0]); + if (OP == COPY) { + // *work(idx) = *v(idx); + *work(r_gl, c_gl, f_gl) = *v(r_gl, c_gl, f_gl); + + /* + if (nr % 2 == 0 && r_gl == 0) { + *work(r_gl+1, c_gl, f_gl) = *v(r_gl, c_gl, f_gl); + } + + if (c_sm == 0) { + if (rest_c > (C / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm)] = + // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl)]; + // printf("load-c[%d %d %d]:%f --> [%d %d %d]\n", r_gl, c_gl_ex, + f_gl, + // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl)], r_sm, c_sm_ex, + // f_sm); + } else if (nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, f_sm)]; + } + } + + if (f_sm == 0) { + if (rest_f > (F / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm_ex)] = + // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl_ex)]; + // printf("load-f[%d %d %d]:%f --> [%d %d %d]\n", r_gl, c_gl, + f_gl_ex, + // dv[get_idx(lddv1, lddv2, r_gl, c_gl, f_gl_ex)], r_sm, c_sm, + // f_sm_ex); + } else if (nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, rest_f - 1)]; + } + } + + // load extra edges + if (c_sm == 0 && f_sm == 0) { + if (rest_c > (C / 2) * 2 && rest_f > (F / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, f_sm_ex)] = + // dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)]; + // printf("load-cf[%d %d %d]:%f --> [%d %d %d]\n", r_gl, c_gl_ex, + // f_gl_ex, dv[get_idx(lddv1, lddv2, r_gl, c_gl_ex, f_gl_ex)], r_sm, + // c_sm_ex, f_sm_ex); + } else if (rest_c <= (C / 2) * 2 && rest_f <= (F / 2) * 2 && + nc % 2 == 0 && nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, rest_f - 1)]; + } else if (rest_c > (C / 2) * 2 && rest_f <= (F / 2) * 2 && + nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm_ex, rest_f - 1)]; + } else if (rest_c <= (C / 2) * 2 && rest_f > (F / 2) * 2 && + nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c_p - 1, f_sm_ex)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm, rest_c - 1, f_sm_ex)]; + } + } + + if (r_sm == 0 && f_sm == 0) { + if (rest_r > (R / 2) * 2 && rest_f > (F / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, f_sm_ex)] = + // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)]; + // printf("load-rf[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl, + // f_gl_ex, dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl, f_gl_ex)], + // r_sm_ex, c_sm, f_sm_ex); + } else if (rest_r <= (R / 2) * 2 && rest_f <= (F / 2) * 2 && + nr % 2 == 0 && nf % 2 == 0) { + // printf("padding (%d %d %d) <- (%d %d %d)\n", rest_r_p - 1, c_sm, + // rest_f_p - 1, rest_r - 1, c_sm, rest_f - 1); + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, rest_f - 1)]; + } else if (rest_r > (R / 2) * 2 && rest_f <= (F / 2) * 2 && + nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm, rest_f - 1)]; + } else if (rest_r <= (R / 2) * 2 && rest_f > (F / 2) * 2 && + nr % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm, f_sm_ex)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm, f_sm_ex)]; + } + } + + if (r_sm == 0 && c_sm == 0) { + if (rest_r > (R / 2) * 2 && rest_c > (C / 2) * 2) { + // v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm)] = + // dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)]; + // printf("load-rc[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, c_gl_ex, + // f_gl, dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl)], r_sm_ex, + // c_sm_ex, f_sm); + } else if (rest_r <= (R / 2) * 2 && rest_c <= (C / 2) * 2 && + nr % 2 == 0 && nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm)]; + // printf("padding (%d %d %d) <- (%d %d %d): %f\n", rest_r_p - 1, + // rest_c_p - 1, f_sm, rest_r - 1, rest_c - 1, f_sm, + // v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm)]); + } else if (rest_r > (R / 2) * 2 && rest_c <= (C / 2) * 2 && + nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c - 1, f_sm)]; + } else if (rest_r <= (R / 2) * 2 && rest_c > (C / 2) * 2 && + nr % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm_ex, f_sm)]; + } + } + // load extra vertex + + if (r_sm == 0 && c_sm == 0 && f_sm == 0) { + if (rest_r > (R / 2) * 2 && rest_c > (C / 2) * 2 && + rest_f > (F / 2) * 2) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, f_sm_ex)] = + *v(r_gl_ex, c_gl_ex, f_gl_ex); + // printf("load-rcf[%d %d %d]:%f --> [%d %d %d]\n", r_gl_ex, + c_gl_ex, + // f_gl_ex, dv[get_idx(lddv1, lddv2, r_gl_ex, c_gl_ex, f_gl_ex)], + // r_sm_ex, c_sm_ex, f_sm_ex); + } else if (rest_r <= (R / 2) * 2 && rest_c <= (C / 2) * 2 && + rest_f <= (F / 2) * 2 && nr % 2 == 0 && nc % 2 == 0 && + nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, + rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, rest_f - 1)]; + } else if (rest_r > (R / 2) * 2 && rest_c > (C / 2) * 2 && + rest_f <= (F / 2) * 2 && nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, c_sm_ex, rest_f - 1)]; + } else if (rest_r > (R / 2) * 2 && rest_c <= (C / 2) * 2 && + rest_f > (F / 2) * 2 && nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, f_sm_ex)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c - 1, f_sm_ex)]; + } else if (rest_r > (R / 2) * 2 && rest_c <= (C / 2) * 2 && + rest_f <= (F / 2) * 2 && nc % 2 == 0 && nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c_p - 1, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, r_sm_ex, rest_c - 1, rest_f - 1)]; + } else if (rest_r <= (R / 2) * 2 && rest_c > (C / 2) * 2 && + rest_f > (F / 2) * 2 && nr % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, f_sm_ex)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm_ex, f_sm_ex)]; + } else if (rest_r <= (R / 2) * 2 && rest_c > (C / 2) * 2 && + rest_f <= (F / 2) * 2 && nr % 2 == 0 && nf % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, c_sm_ex, rest_f_p - 1)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, c_sm_ex, rest_f - 1)]; + } else if (rest_r <= (R / 2) * 2 && rest_c <= (C / 2) * 2 && + rest_f > (F / 2) * 2 && nr % 2 == 0 && nc % 2 == 0) { + v_sm[get_idx(ldsm1, ldsm2, rest_r_p - 1, rest_c_p - 1, f_sm_ex)] = + v_sm[get_idx(ldsm1, ldsm2, rest_r - 1, rest_c - 1, f_sm_ex)]; + } + } + + */ + } + if (OP == ADD) { + *work(r_gl, c_gl, f_gl) += *v(r_gl, c_gl, f_gl); + } + if (OP == SUBTRACT) { + *work(r_gl, c_gl, f_gl) -= *v(r_gl, c_gl, f_gl); + } + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + SIZE nr, nc, nf; + SubArray v, work; + SIZE r_gl, c_gl, f_gl; +}; + +template +class Lwpk3DKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "lwpk"; + MGARDX_CONT + Lwpk3DKernel(SIZE nr, SIZE nc, SIZE nf, SubArray v, + SubArray work) + : nr(nr), nc(nc), nf(nf), v(v), work(work) {} + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Lwpk3DFunctor; + FunctorType functor(nr, nc, nf, v, work); + SIZE total_thread_z = nr; + SIZE total_thread_y = nc; + SIZE total_thread_x = nf; + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = R; + tby = C; + tbx = F; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE nr, nc, nf; + SubArray v, work; +}; + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/SubtractND.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/SubtractND.hpp new file mode 100644 index 000000000..fcf829848 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/CopyND/SubtractND.hpp @@ -0,0 +1,38 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../../Hierarchy/Hierarchy.h" +#include "../../../RuntimeX/RuntimeX.h" + +#include "../DataRefactoring.h" + +#include "LevelwiseProcessingKernel.hpp" + +#ifndef MGARD_X_160_DATA_REFACTORING_SUBTRACT_ND +#define MGARD_X_160_DATA_REFACTORING_SUBTRACT_ND + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +void SubtractND(SubArray dinput, + SubArray doutput, int queue_idx) { + + DeviceLauncher::Execute( + LwpkReoKernel(dinput, doutput), queue_idx); +} + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrection3D.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrection3D.hpp new file mode 100644 index 000000000..50ae540a5 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrection3D.hpp @@ -0,0 +1,193 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../../Hierarchy/Hierarchy.h" +#include "../../../RuntimeX/RuntimeX.h" + +#include "../DataRefactoring.h" + +#include "IterativeProcessingKernel3D.hpp" +#include "LinearProcessingKernel3D.hpp" + +#ifndef MGARD_X_160_DATA_REFACTORING_CALC_CORRECTION_3D +#define MGARD_X_160_DATA_REFACTORING_CALC_CORRECTION_3D + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +void CalcCorrection3D(Hierarchy &hierarchy, + SubArray dcoeff, + SubArray &dcorrection, SIZE l, + int queue_idx) { + + std::string prefix = "decomp_"; + if (sizeof(T) == sizeof(double)) + prefix += "d_"; + if (sizeof(T) == sizeof(float)) + prefix += "f_"; + for (int d = 0; d < D; d++) + prefix += + std::to_string(hierarchy.level_shape(hierarchy.l_target(), d)) + "_"; + + SIZE f = hierarchy.level_shape(l, D - 1); + SIZE c = hierarchy.level_shape(l, D - 2); + SIZE r = hierarchy.level_shape(l, D - 3); + SIZE ff = hierarchy.level_shape(l - 1, D - 1); + SIZE cc = hierarchy.level_shape(l - 1, D - 2); + SIZE rr = hierarchy.level_shape(l - 1, D - 3); + + SubArray dist_f(hierarchy.dist(l, D - 1)); + SubArray dist_c(hierarchy.dist(l, D - 2)); + SubArray dist_r(hierarchy.dist(l, D - 3)); + + SubArray dist_ff(hierarchy.dist(l - 1, D - 1)); + SubArray dist_cc(hierarchy.dist(l - 1, D - 2)); + SubArray dist_rr(hierarchy.dist(l - 1, D - 3)); + + SubArray ratio_f(hierarchy.ratio(l, D - 1)); + SubArray ratio_c(hierarchy.ratio(l, D - 2)); + SubArray ratio_r(hierarchy.ratio(l, D - 3)); + + SubArray am_ff(hierarchy.am(l - 1, D - 1)); + SubArray am_cc(hierarchy.am(l - 1, D - 2)); + SubArray am_rr(hierarchy.am(l - 1, D - 3)); + + SubArray bm_ff(hierarchy.bm(l - 1, D - 1)); + SubArray bm_cc(hierarchy.bm(l - 1, D - 2)); + SubArray bm_rr(hierarchy.bm(l - 1, D - 3)); + + SubArray dw_in1, dw_in2, dw_out; + + if (D >= 1) { + dw_in1 = dcoeff; + dw_in1.resize({r, c, ff}); + dw_in2 = dcoeff; + dw_in2.offset({0, 0, ff}); + dw_in2.resize({r, c, f - ff}); + dw_out = dcorrection; + dw_out.resize({r, c, ff}); + + DeviceLauncher::Execute( + Lpk1Reo3DKernel(r, c, f, ff, rr, cc, ff, dist_f, + ratio_f, dw_in1, dw_in2, dw_out), + queue_idx); + + verify_matrix_cuda(r, c, ff, dw_out.data(), dw_out.ld(D - 1), + dw_out.ld(D - 2), dw_out.ld(D - 1), + prefix + "lpk_reo_1_3d" + "_level_" + std::to_string(l), + multidim_refactoring_store, multidim_refactoring_verify); + + if (multidim_refactoring_debug_print) { + PrintSubarray("after mass_trans_multiply_1_cpt", dw_out); + } + } + + if (D >= 2) { + dw_in1 = dw_out; + dw_in1.resize({r, cc, ff}); + dw_in2 = dw_out; + dw_in2.offset({0, cc, 0}); + dw_in2.resize({r, c - cc, ff}); + dw_out.offset({0, 0, ff}); + dw_out.resize({r, cc, ff}); + + DeviceLauncher::Execute( + Lpk2Reo3DKernel(r, c, ff, cc, dist_c, ratio_c, dw_in1, + dw_in2, dw_out), + queue_idx); + + verify_matrix_cuda(r, cc, ff, dw_out.data(), dw_out.ld(D - 1), + dw_out.ld(D - 2), dw_out.ld(D - 1), + prefix + "lpk_reo_2_3d" + "_level_" + std::to_string(l), + multidim_refactoring_store, multidim_refactoring_verify); + + if (multidim_refactoring_debug_print) { + PrintSubarray("after mass_trans_multiply_2_cpt", dw_out); + } + } + + if (D == 3) { + dw_in1 = dw_out; + dw_in1.resize({rr, cc, ff}); + dw_in2 = dw_out; + dw_in2.offset({rr, 0, 0}); + dw_in2.resize({r - rr, cc, r - ff}); + dw_out.offset({0, cc, ff}); + dw_out.resize({rr, cc, ff}); + + DeviceLauncher::Execute( + Lpk3Reo3DKernel(r, cc, ff, rr, dist_r, ratio_r, + dw_in1, dw_in2, dw_out), + queue_idx); + + verify_matrix_cuda(rr, cc, ff, dw_out.data(), dw_out.ld(D - 1), + dw_out.ld(D - 2), dw_out.ld(D - 1), + prefix + "lpk_reo_3_3d" + "_level_" + std::to_string(l), + multidim_refactoring_store, multidim_refactoring_verify); + + if (multidim_refactoring_debug_print) { + PrintSubarray("after mass_trans_multiply_3_cpt", dw_out); + } + } + + if (D >= 1) { + DeviceLauncher::Execute( + Ipk1Reo3DKernel(rr, cc, ff, am_ff, bm_ff, dist_ff, + dw_out), + queue_idx); + verify_matrix_cuda(rr, cc, ff, dw_out.data(), dw_out.ld(D - 1), + dw_out.ld(D - 2), dw_out.ld(D - 1), + prefix + "ipk_1_3d" + "_level_" + std::to_string(l), + multidim_refactoring_store, multidim_refactoring_verify); + + if (multidim_refactoring_debug_print) { + PrintSubarray("after solve_tridiag_1_cpt", dw_out); + } + } + if (D >= 2) { + DeviceLauncher::Execute( + Ipk2Reo3DKernel(rr, cc, ff, am_cc, bm_cc, dist_cc, + dw_out), + queue_idx); + verify_matrix_cuda(rr, cc, ff, dw_out.data(), dw_out.ld(D - 1), + dw_out.ld(D - 2), dw_out.ld(D - 1), + prefix + "ipk_2_3d" + "_level_" + std::to_string(l), + multidim_refactoring_store, multidim_refactoring_verify); + + if (multidim_refactoring_debug_print) { + PrintSubarray("after solve_tridiag_2_cpt", dw_out); + } + } + if (D == 3) { + DeviceLauncher::Execute( + Ipk3Reo3DKernel(rr, cc, ff, am_rr, bm_rr, dist_rr, + dw_out), + queue_idx); + verify_matrix_cuda(rr, cc, ff, dw_out.data(), dw_out.ld(D - 1), + dw_out.ld(D - 2), dw_out.ld(D - 1), + prefix + "ipk_3_3d" + "_level_" + std::to_string(l), + multidim_refactoring_store, multidim_refactoring_verify); + + if (multidim_refactoring_debug_print) { + PrintSubarray("after solve_tridiag_3_cpt", dw_out); + } + } + // final correction output + dcorrection = dw_out; +} + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrectionND.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrectionND.hpp new file mode 100644 index 000000000..cde7f3a43 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrectionND.hpp @@ -0,0 +1,269 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../../Hierarchy/Hierarchy.h" +#include "../../../RuntimeX/RuntimeX.h" + +#include "../DataRefactoring.h" + +#include "IterativeProcessingKernel.hpp" +#include "LinearProcessingKernel.hpp" + +#ifndef MGARD_X_160_DATA_REFACTORING_CALC_CORRECTION_ND +#define MGARD_X_160_DATA_REFACTORING_CALC_CORRECTION_ND + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +void CalcCorrectionND(Hierarchy &hierarchy, + SubArray dcoeff, + SubArray &dcorrection, SIZE l, + int queue_idx) { + + std::string prefix = "decomp_"; + if (sizeof(T) == sizeof(double)) + prefix += "d_"; + if (sizeof(T) == sizeof(float)) + prefix += "f_"; + for (int d = 0; d < D; d++) + prefix += + std::to_string(hierarchy.level_shape(hierarchy.l_target(), d)) + "_"; + + SubArray dw_in1 = dcoeff; + SubArray dw_in2 = dcoeff; + SubArray dw_out = dcorrection; + + SubArray<1, SIZE, DeviceType> shape, shape_c; + shape = SubArray<1, SIZE, DeviceType>(hierarchy.level_shape_array(l)); + shape_c = SubArray<1, SIZE, DeviceType>(hierarchy.level_shape_array(l - 1)); + + DIM processed_n; + SubArray<1, DIM, DeviceType> processed_dims; + + SubArray<1, T, DeviceType> dist, ratio, am, bm; + + // start correction calculation + int prev_dim_f, prev_dim_c, prev_dim_r; + int curr_dim_f, curr_dim_c, curr_dim_r; + + curr_dim_f = D - 1, curr_dim_c = D - 2, curr_dim_r = D - 3; + + dw_in1.resize(curr_dim_f, hierarchy.level_shape(l - 1, curr_dim_f)); + dw_in2.offset_dim(curr_dim_f, hierarchy.level_shape(l - 1, curr_dim_f)); + dw_in2.resize(curr_dim_f, hierarchy.level_shape(l, curr_dim_f) - + hierarchy.level_shape(l - 1, curr_dim_f)); + dw_out.resize(curr_dim_f, hierarchy.level_shape(l - 1, curr_dim_f)); + prev_dim_f = curr_dim_f; + prev_dim_c = curr_dim_c; + prev_dim_r = curr_dim_r; + + dw_in1.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_in2.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_out.project(curr_dim_r, curr_dim_c, curr_dim_f); + + dist = SubArray(hierarchy.dist(l, curr_dim_f)); + ratio = SubArray(hierarchy.ratio(l, curr_dim_f)); + + processed_dims = SubArray(hierarchy.processed(0, processed_n)); + + DeviceLauncher::Execute( + Lpk1ReoKernel( + shape, shape_c, processed_n, processed_dims, curr_dim_r, curr_dim_c, + curr_dim_f, dist, ratio, dw_in1, dw_in2, dw_out), + queue_idx); + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D(format("decomposition: after MR-1D[{}]", l), dw_out); + } + + // mass trans 2D + dw_in1 = dw_out; + dw_in2 = dw_out; + + curr_dim_f = D - 1, curr_dim_c = D - 2, curr_dim_r = D - 3; + dw_in1.resize(curr_dim_c, hierarchy.level_shape(l - 1, curr_dim_c)); + dw_in2.offset_dim(curr_dim_c, hierarchy.level_shape(l - 1, curr_dim_c)); + dw_in2.resize(curr_dim_c, hierarchy.level_shape(l, curr_dim_c) - + hierarchy.level_shape(l - 1, curr_dim_c)); + dw_out.offset_dim(prev_dim_f, hierarchy.level_shape(l - 1, curr_dim_f)); + dw_out.resize(curr_dim_c, hierarchy.level_shape(l - 1, curr_dim_c)); + prev_dim_f = curr_dim_f; + prev_dim_c = curr_dim_c; + prev_dim_r = curr_dim_r; + + dw_in1.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_in2.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_out.project(curr_dim_r, curr_dim_c, curr_dim_f); + + dist = SubArray(hierarchy.dist(l, curr_dim_c)); + ratio = SubArray(hierarchy.ratio(l, curr_dim_c)); + + processed_dims = SubArray(hierarchy.processed(1, processed_n)); + + DeviceLauncher::Execute( + Lpk2ReoKernel( + shape, shape_c, processed_n, processed_dims, curr_dim_r, curr_dim_c, + curr_dim_f, dist, ratio, dw_in1, dw_in2, dw_out), + queue_idx); + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D(format("decomposition: after MR-2D[{}]", l), dw_out); + } + + // mass trans 3D + dw_in1 = dw_out; + dw_in2 = dw_out; + + curr_dim_f = D - 1, curr_dim_c = D - 2, curr_dim_r = D - 3; + dw_in1.resize(curr_dim_r, hierarchy.level_shape(l - 1, curr_dim_r)); + dw_in2.offset_dim(curr_dim_r, hierarchy.level_shape(l - 1, curr_dim_r)); + dw_in2.resize(curr_dim_r, hierarchy.level_shape(l, curr_dim_r) - + hierarchy.level_shape(l - 1, curr_dim_r)); + dw_out.offset_dim(prev_dim_c, hierarchy.level_shape(l - 1, curr_dim_c)); + dw_out.resize(curr_dim_r, hierarchy.level_shape(l - 1, curr_dim_r)); + prev_dim_f = curr_dim_f; + prev_dim_c = curr_dim_c; + prev_dim_r = curr_dim_r; + + dw_in1.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_in2.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_out.project(curr_dim_r, curr_dim_c, curr_dim_f); + + dist = SubArray(hierarchy.dist(l, curr_dim_r)); + ratio = SubArray(hierarchy.ratio(l, curr_dim_r)); + + processed_dims = SubArray(hierarchy.processed(2, processed_n)); + + DeviceLauncher::Execute( + Lpk3ReoKernel( + shape, shape_c, processed_n, processed_dims, curr_dim_r, curr_dim_c, + curr_dim_f, dist, ratio, dw_in1, dw_in2, dw_out), + queue_idx); + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D(format("decomposition: after MR-3D[{}]", l), dw_out); + } + + // mass trans 4D+ + for (int i = 3; i < D; i++) { + dw_in1 = dw_out; + dw_in2 = dw_out; + + curr_dim_f = D - 1, curr_dim_c = D - 2, curr_dim_r = D - (i + 1); + dw_in1.resize(curr_dim_r, hierarchy.level_shape(l - 1, curr_dim_r)); + dw_in2.offset_dim(curr_dim_r, hierarchy.level_shape(l - 1, curr_dim_r)); + dw_in2.resize(curr_dim_r, hierarchy.level_shape(l, curr_dim_r) - + hierarchy.level_shape(l - 1, curr_dim_r)); + dw_out.offset_dim(prev_dim_r, hierarchy.level_shape(l - 1, prev_dim_r)); + dw_out.resize(curr_dim_r, hierarchy.level_shape(l - 1, curr_dim_r)); + prev_dim_f = curr_dim_f; + prev_dim_c = curr_dim_c; + prev_dim_r = curr_dim_r; + + dw_in1.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_in2.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_out.project(curr_dim_r, curr_dim_c, curr_dim_f); + + dist = SubArray(hierarchy.dist(l, curr_dim_r)); + ratio = SubArray(hierarchy.ratio(l, curr_dim_r)); + + processed_dims = SubArray(hierarchy.processed(i, processed_n)); + + DeviceLauncher::Execute( + Lpk3ReoKernel( + shape, shape_c, processed_n, processed_dims, curr_dim_r, curr_dim_c, + curr_dim_f, dist, ratio, dw_in1, dw_in2, dw_out), + queue_idx); + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D(format("decomposition: after MR-{}D[{}]", i + 1, l), + dw_out); + } + } + + curr_dim_f = D - 1, curr_dim_c = D - 2, curr_dim_r = D - 3; + dw_in1.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_in2.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_out.project(curr_dim_r, curr_dim_c, curr_dim_f); + am = hierarchy.am(l - 1, curr_dim_f); + bm = hierarchy.bm(l - 1, curr_dim_f); + + DeviceLauncher::Execute( + Ipk1ReoKernel(curr_dim_r, curr_dim_c, curr_dim_f, am, + bm, dw_out), + queue_idx); + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D(format("decomposition: after TR-1D[{}]", l), dw_out); + } // debug + + curr_dim_f = D - 1, curr_dim_c = D - 2, curr_dim_r = D - 3; + dw_in1.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_in2.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_out.project(curr_dim_r, curr_dim_c, curr_dim_f); + am = hierarchy.am(l - 1, curr_dim_c); + bm = hierarchy.bm(l - 1, curr_dim_c); + + DeviceLauncher::Execute( + Ipk2ReoKernel(curr_dim_r, curr_dim_c, curr_dim_f, am, + bm, dw_out), + queue_idx); + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D(format("decomposition: after TR-2D[{}]", l), dw_out); + } // debug + + curr_dim_f = D - 1, curr_dim_c = D - 2, curr_dim_r = D - 3; + dw_in1.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_in2.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_out.project(curr_dim_r, curr_dim_c, curr_dim_f); + am = hierarchy.am(l - 1, curr_dim_r); + bm = hierarchy.bm(l - 1, curr_dim_r); + + DeviceLauncher::Execute( + Ipk3ReoKernel(curr_dim_r, curr_dim_c, curr_dim_f, am, + bm, dw_out), + queue_idx); + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D(format("decomposition: after TR-3D[{}]", l), dw_out); + } // debug + + // mass trans 4D+ + for (int i = 3; i < D; i++) { + curr_dim_f = D - 1, curr_dim_c = D - 2, curr_dim_r = D - (i + 1); + dw_in1.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_in2.project(curr_dim_r, curr_dim_c, curr_dim_f); + dw_out.project(curr_dim_r, curr_dim_c, curr_dim_f); + am = hierarchy.am(l - 1, curr_dim_r); + bm = hierarchy.bm(l - 1, curr_dim_r); + + DeviceLauncher::Execute( + Ipk3ReoKernel(curr_dim_r, curr_dim_c, curr_dim_f, am, + bm, dw_out), + queue_idx); + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D(format("decomposition: after TR-{}D[{}]", i + 1, l), + dw_out); + } // debug + } + + dcorrection = dw_out; +} + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/IPKFunctor.h b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/IPKFunctor.h new file mode 100644 index 000000000..06014750a --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/IPKFunctor.h @@ -0,0 +1,55 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_IPK_FUNCTOR +#define MGARD_X_160_IPK_FUNCTOR + +namespace mgard_x_160 { + +template +MGARDX_EXEC T tridiag_forward2(T prev, T am, T bm, T curr) { + +#ifdef MGARD_X_160_FMA + if (sizeof(T) == sizeof(double)) { + // printf("forward: %f < %f %f %f %f\n", fma(prev, am * bm, curr), + // curr, prev, am , bm); + return fma(prev, am * bm, curr); + } else if (sizeof(T) == sizeof(float)) { + // printf("forward: %f < %f %f %f %f\n", fmaf(prev, am * bm, curr), + // curr, prev, am , bm); + return fmaf(prev, am * bm, curr); + } +#else + // printf("forward: %f < %f %f %f %f\n", curr - prev * (am * bm), + // curr, prev, am , bm); + return curr - prev * (am / bm); +#endif +} + +template +MGARDX_EXEC T tridiag_backward2(T prev, T am, T bm, T curr) { + +#ifdef MGARD_X_160_FMA + if (sizeof(T) == sizeof(double)) { + // printf("backward: %f < %f %f %f %f\n", fma(am, prev, curr) * bm, + // curr, prev, am , bm); + return fma(am, prev, curr) * bm; + } else if (sizeof(T) == sizeof(float)) { + // printf("backward: %f < %f %f %f %f\n", fmaf(am, prev, curr) * bm, + // curr, prev, am , bm); + return fmaf(am, prev, curr) * bm; + } +#else + // printf("backward: %f < %f %f %f %f\n", (curr - am * prev) * bm, + // curr, prev, am , bm); + return (curr - am * prev) / bm; +#endif +} + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel.hpp new file mode 100644 index 000000000..367f2309d --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel.hpp @@ -0,0 +1,1993 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_ITERATIVE_PROCESSING_KERNEL_TEMPLATE +#define MGARD_X_160_ITERATIVE_PROCESSING_KERNEL_TEMPLATE + +#include "../../../RuntimeX/RuntimeX.h" +#include "IPKFunctor.h" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +class Ipk1ReoFunctor : public IterFunctor { +public: + MGARDX_CONT Ipk1ReoFunctor() {} + MGARDX_CONT + Ipk1ReoFunctor(DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, + SubArray<1, T, DeviceType> am, SubArray<1, T, DeviceType> bm, + SubArray v) + : curr_dim_r(curr_dim_r), curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), + am(am), bm(bm), v(v) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + // bool debug = false; + // if (blockIdx.z == 0 && FunctorBase::GetBlockIdY() == 0 && + // FunctorBase::GetBlockIdX() == 0 && + // FunctorBase::GetThreadIdZ() == 0 && + // FunctorBase::GetThreadIdY() == 0) + // debug = false; + + // bool debug2 = false; + // if (FunctorBase::GetThreadIdZ() == 0 && + // FunctorBase::GetThreadIdY() == 0 && + // FunctorBase::GetThreadIdX() == 0) + // debug2 = false; + + threadId = (FunctorBase::GetThreadIdZ() * + (FunctorBase::GetBlockDimX() * + FunctorBase::GetBlockDimY())) + + (FunctorBase::GetThreadIdY() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + + T *sm = (T *)FunctorBase::GetSharedMemory(); + ldsm1 = F + G; + ldsm2 = C; + vec_sm = sm; + sm += R * ldsm1 * ldsm2; + am_sm = sm; + sm += ldsm1; + bm_sm = sm; + sm += ldsm1; + } + + MGARDX_EXEC void Operation2() { + // __syncthreads(); + + for (DIM d = 0; d < D; d++) + idx[d] = 0; + + nr = v.shape(curr_dim_r); + nc = v.shape(curr_dim_c); + nf = v.shape(curr_dim_f); + + if (D < 3) + nr = 1; + if (D < 2) + nc = 1; + + SIZE bidx = FunctorBase::GetBlockIdX(); + SIZE firstD = div_roundup(nc, C); + SIZE blockId = bidx % firstD; + // if (debug2) { + // printf("FunctorBase::GetBlockIdX() %u nc %u + // FunctorBase::GetBlockDimX() %u firstD: %u blockId %u\n", + // FunctorBase::GetBlockIdX(), nc, + // FunctorBase::GetBlockDimX(), firstD, blockId); + // } + bidx /= firstD; + + for (DIM d = 0; d < D; d++) { + if (d != curr_dim_r && d != curr_dim_c && d != curr_dim_f) { + SIZE t = v.shape(d); + idx[d] = bidx % t; + bidx /= t; + } + } + + v.offset(idx); + // size_t other_offset_v = get_idx(ldvs_sm, idx); + // v = v + other_offset_v; + + // if (debug2) { + // printf("ipk1 idx: %u %u %u %u %u ld: %u %u %u %u %u\n", idx[4], idx[3], + // idx[2], idx[1], idx[0], ldvs_sm[4], ldvs_sm[3], ldvs_sm[2], ldvs_sm[1], + // ldvs_sm[0]); printf("ipk1 other_offset_v: %llu\n", other_offset_v); + + // SIZE curr_stride = 1; + // SIZE ret_idx = 0; + // for (DIM i = 0; i < D; i++) { + // ret_idx += idx[i] * curr_stride; + // printf("%llu * %llu = %llu\n", curr_stride, ldvs_sm[i], + // curr_stride*ldvs_sm[i]); curr_stride *= ldvs_sm[i]; + + // } + // } + + c_gl = blockId * C; + r_gl = FunctorBase::GetBlockIdY() * R; + f_gl = FunctorBase::GetThreadIdX(); + + c_sm = FunctorBase::GetThreadIdX(); + r_sm = FunctorBase::GetThreadIdY(); + f_sm = FunctorBase::GetThreadIdX(); + + // if (idx[3] == 0 && idx[4] == 1 && r_gl+r_sm == 0 && c_gl+c_sm == 0) { + // printf("offset = %llu\n", other_offset_v); + // debug2 = false; + // } + + // T *vec = v + get_idx(ldv1, ldv2, r_gl, c_gl, 0); + + v.offset_3d(r_gl, c_gl, 0); + + prev_vec_sm = 0.0; + + c_rest = Math::Min(C, nc - blockId * C); + r_rest = Math::Min( + R, nr - FunctorBase::GetBlockIdY() * R); + + f_rest = nf; + f_ghost = Math::Min(nf, G); + f_main = F; + + // printf("r_sm: %d, r_rest: %d, c_sm: %d, c_rest: %d f_sm: %d, f_rest %d , + // nf: %d\n", r_sm, r_rest, c_sm, c_rest, f_sm, f_rest, nf); + + // printf("test %f", vec_sm[get_idx(ldsm1, ldsm2, 0, 1, 0)]); + /* Load first ghost */ + if (r_sm < r_rest && f_sm < f_ghost) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = *v(r_sm, i, f_gl); + // if (r_sm == 0) printf("r0_stride = %d, vec_sm[%d] = %f\n", r0_stride, + // i, vec_sm[i * ldsm + c_sm]); + } + } + + if (r_sm == 0 && f_sm < f_ghost) { + am_sm[f_sm] = *am(f_gl); + bm_sm[f_sm] = *bm(f_gl); + } + + f_rest -= f_ghost; + } + // __syncthreads(); + + MGARDX_EXEC bool LoopCondition1() { return f_rest > F - f_ghost; } + + MGARDX_EXEC void Operation3() { + // while (f_rest > F - f_ghost) { + // if (c_gl == 0 && c_sm == 0 && r_gl == 0 && r_sm == 0) printf("%d %d\n", + // f_rest, F - f_ghost); + f_main = Math::Min(F, f_rest); + if (r_sm < r_rest && f_sm < f_main) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = + *v(r_sm, i, f_gl + f_ghost); + } + } + if (r_sm == 0 && f_sm < f_main) { + am_sm[f_sm + f_ghost] = *am(f_gl + f_ghost); + bm_sm[f_sm + f_ghost] = *bm(f_gl + f_ghost); + } + } + + // __syncthreads(); + MGARDX_EXEC void Operation4() { + + /* Computation of v in parallel*/ + if (r_sm < r_rest && c_sm < c_rest) { + // if (debug) printf("forward %f <- %f %f %f %f\n", + // tridiag_forward2( + // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, r_sm, + // c_sm, 0)]), + // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, + // ldsm2, r_sm, c_sm, 0)]); + + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); + + // #pragma unroll 32 + for (SIZE i = 1; i < F; i++) { + // if (debug) printf("forward %f <- %f %f %f %f\n", + // tridiag_forward2( + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], + // bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]), + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], + // bm_sm[i], + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); + + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_forward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); + } + + /* Store last v */ + prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, F - 1)]; + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation5() { + + /* flush results to v */ + if (r_sm < r_rest && f_sm < F) { + for (SIZE i = 0; i < c_rest; i++) { + // if (FunctorBase::GetBlockIdX() == 0 && + // FunctorBase::GetBlockIdY() == 0 && r_sm == 0 && i == 1) { + // printf("store [%d %d %d] %f<-%f [%d %d %d]\n", + // r_sm, i, f_gl, *v(r_sm, i, f_gl), + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)], r_sm, i, f_sm); + // } + *v(r_sm, i, f_gl) = vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + // if (FunctorBase::GetBlockIdX() == 0 && + // FunctorBase::GetBlockIdY() == 0 && r_sm == 0 && i == 1) { + // printf("store [%d %d %d] %f<-%f [%d %d %d]\n", + // r_sm, i, f_gl, *v(r_sm, i, f_gl), + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)], r_sm, i, f_sm); + // } + } + } + } + + // __syncthreads(); + + MGARDX_EXEC void Operation6() { + + /* Update unloaded col */ + f_rest -= f_main; + + /* Advance c */ + f_gl += F; + + /* Copy next ghost to main */ + f_ghost = Math::Min(G, f_main - (F - G)); + if (r_sm < r_rest && f_sm < f_ghost) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + F)]; + } + } + if (r_sm == 0 && f_sm < f_ghost) { + am_sm[f_sm] = am_sm[f_sm + FunctorBase::GetBlockDimX()]; + bm_sm[f_sm] = bm_sm[f_sm + FunctorBase::GetBlockDimX()]; + } + } + // __syncthreads(); + // } // end of while + MGARDX_EXEC void Operation7() { + /* Load all rest col */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = + *v(r_sm, i, f_gl + f_ghost); + } + // if (r_sm == 0) { + // bm_sm[f_sm + f_ghost] = *bm(f_gl + f_ghost); + // } + } + if (r_sm == 0 && f_sm < f_rest) { + am_sm[f_sm + f_ghost] = *am(f_gl + f_ghost); + bm_sm[f_sm + f_ghost] = *bm(f_gl + f_ghost); + } + } + + // __syncthreads(); + + MGARDX_EXEC void Operation8() { + + /* Only 1 col remain */ + if (f_ghost + f_rest == 1) { + if (r_sm < r_rest && c_sm < c_rest) { + // if (debug) printf("forward %f <- %f %f %f %f\n", + // tridiag_forward2( + // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, + // r_sm, c_sm, 0)]), + // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, + // ldsm2, r_sm, c_sm, 0)]); + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); + // printf ("prev_vec_sm = %f\n", prev_vec_sm ); + // printf ("vec_sm[r_sm * ldsm + 0] = %f\n", vec_sm[r_sm * ldsm + 0] ); + } + //__syncthreads(); + + } else { + if (r_sm < r_rest && c_sm < c_rest) { + // if (debug) printf("forward %f <- %f %f %f %f\n", + // tridiag_forward2( + // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, + // r_sm, c_sm, 0)]), + // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, + // ldsm2, r_sm, c_sm, 0)]); + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); + for (SIZE i = 1; i < f_ghost + f_rest; i++) { + // if (debug) printf("forward %f <- %f %f %f %f\n", + // tridiag_forward2( + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], + // bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]), + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], + // am_sm[i], bm_sm[i], + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_forward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); + } + } + } + } + + MGARDX_EXEC void Operation9() { + // __syncthreads(); + /* flush results to v */ + if (r_sm < r_rest && f_sm < f_ghost + f_rest) { + for (SIZE i = 0; i < c_rest; i++) { + *v(r_sm, i, f_gl) = vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + // printf("c_stride = %d, c_sm = %d, vec_sm = %f, vec[%d] = + // %f\n",c_stride, c_sm, vec_sm[r_sm * ldsm + 0],i * row_stride * lddv + + // c_stride, vec[i * row_stride * lddv + c_stride]); + } + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation10() { + /* backward */ + f_rest = nf; + f_ghost = Math::Min(nf, G); + f_main = F; + f_gl = FunctorBase::GetThreadIdX(); + prev_vec_sm = 0.0; + + /* Load first ghost */ + if (r_sm < r_rest && f_sm < f_ghost) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + *v(r_sm, i, (nf - 1) - f_gl); + // if (r_sm == 0) printf("r0_stride = %d, vec_sm[%d] = %f\n", r0_stride, + // i, vec_sm[i * ldsm + c_sm]); + } + } + if (r_sm == 0 && f_sm < f_ghost) { + am_sm[f_sm] = *am(nf - f_gl); + bm_sm[f_sm] = *bm(nf - f_gl); + } + f_rest -= f_ghost; + } + // __syncthreads(); + MGARDX_EXEC bool LoopCondition2() { return f_rest > F - f_ghost; } + + MGARDX_EXEC void Operation11() { + // while (f_rest > F - f_ghost) { + f_main = Math::Min(F, f_rest); + if (r_sm < r_rest && f_sm < f_main) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = + *v(r_sm, i, (nf - 1) - f_gl - f_ghost); + } + } + if (r_sm == 0 && f_sm < f_main) { + am_sm[f_sm + f_ghost] = *am(nf - f_gl - f_ghost); + bm_sm[f_sm + f_ghost] = *bm(nf - f_gl - f_ghost); + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation12() { + /* Computation of v in parallel*/ + if (r_sm < r_rest && c_sm < c_rest) { + // if (debug) printf("backward %f <- %f %f %f %f\n", + // tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]), + // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, + // ldsm2, r_sm, c_sm, 0)]); + + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); + // #pragma unroll 32 + for (SIZE i = 1; i < F; i++) { + + // if (debug) printf("backward %f <- %f %f %f %f\n", + // tridiag_backward2( + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], + // am_sm[i], bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, + // i)]), + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], + // am_sm[i], bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, + // i)]); + + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_backward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); + } + /* Store last v */ + prev_vec_sm = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, + FunctorBase::GetBlockDimX() - 1)]; + } + } + // __syncthreads(); + MGARDX_EXEC void Operation13() { + + /* flush results to v */ + if (r_sm < r_rest && f_sm < F) { + for (SIZE i = 0; i < c_rest; i++) { + *v(r_sm, i, (nf - 1) - f_gl) = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + } + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation14() { + /* Update unloaded col */ + f_rest -= f_main; + + /* Advance c */ + f_gl += F; + + /* Copy next ghost to main */ + f_ghost = Math::Min(G, f_main - (F - G)); + if (r_sm < r_rest && f_sm < f_ghost) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + F)]; + } + if (r_sm == 0) { + am_sm[f_sm] = am_sm[f_sm + F]; + bm_sm[f_sm] = bm_sm[f_sm + F]; + } + } + } + // __syncthreads(); + // } // end of while + + MGARDX_EXEC void Operation15() { + /* Load all rest col */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = + *v(r_sm, i, (nf - 1) - f_gl - f_ghost); + } + } + if (r_sm == 0 && f_gl + f_ghost <= nf) { + am_sm[f_sm + f_ghost] = *am(nf - f_gl - f_ghost); + bm_sm[f_sm + f_ghost] = *bm(nf - f_gl - f_ghost); + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation16() { + /* Only 1 col remain */ + if (f_ghost + f_rest == 1) { + if (r_sm < r_rest && c_sm < c_rest) { + // if (debug) printf("backward %f <- %f %f %f %f\n", + // tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]), + // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, + // ldsm2, r_sm, c_sm, 0)]); + + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); + // printf ("prev_vec_sm = %f\n", prev_vec_sm ); + // printf ("vec_sm[r_sm * ldsm + 0] = %f\n", vec_sm[r_sm * ldsm + 0] ); + } + //__syncthreads(); + + } else { + if (r_sm < r_rest && c_sm < c_rest) { + // if (debug) printf("backward %f <- %f %f %f %f\n", + // tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]), + // prev_vec_sm, am_sm[0], bm_sm[0], vec_sm[get_idx(ldsm1, + // ldsm2, r_sm, c_sm, 0)]); + + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); + for (SIZE i = 1; i < f_ghost + f_rest; i++) { + + // if (debug) printf("backward %f <- %f %f %f %f\n", + // tridiag_backward2( + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], + // am_sm[i], bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, + // i)]), + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], + // am_sm[i], bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, + // i)]); + + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_backward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); + } + } + } + } + + MGARDX_EXEC void Operation17() { + // __syncthreads(); + /* flush results to v */ + if (r_sm < r_rest && f_sm < f_ghost + f_rest) { + for (SIZE i = 0; i < c_rest; i++) { + *v(r_sm, i, (nf - 1) - f_gl) = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + // printf("c_stride = %d, c_sm = %d, vec_sm = %f, vec[%d] = + // %f\n",c_stride, c_sm, vec_sm[r_sm * ldsm + 0],i * row_stride * lddv + + // c_stride, vec[i * row_stride * lddv + c_stride]); + } + } + // __syncthreads(); + v.reset_offset(); + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + size = (R * C + 2) * (F + G) * sizeof(T); + return size; + } + +private: + // functor parameters + DIM curr_dim_r, curr_dim_c, curr_dim_f; + SubArray<1, T, DeviceType> am, bm; + SubArray v; + + // thread local variables + SIZE threadId; + + T *vec_sm; + SIZE ldsm1, ldsm2; + T *am_sm; + T *bm_sm; + + SIZE idx[D]; + + SIZE nr; + SIZE nc; + SIZE nf; + + SIZE c_gl, r_gl, f_gl; + SIZE c_sm, r_sm, f_sm; + + T prev_vec_sm; + SIZE c_rest, r_rest; + SIZE f_rest, f_ghost, f_main; +}; + +template +class Ipk1ReoKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "ipk1_nd"; + constexpr static SIZE G = 2; + MGARDX_CONT + Ipk1ReoKernel(DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, + SubArray<1, T, DeviceType> am, SubArray<1, T, DeviceType> bm, + SubArray v) + : curr_dim_r(curr_dim_r), curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), + am(am), bm(bm), v(v) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Ipk1ReoFunctor; + FunctorType functor(curr_dim_r, curr_dim_c, curr_dim_f, am, bm, v); + + SIZE nr = 1, nc = 1; + if (D >= 3) + nr = v.shape(curr_dim_r); + if (D >= 2) + nc = v.shape(curr_dim_c); + + SIZE total_thread_x = nc; + SIZE total_thread_y = nr; + SIZE total_thread_z = 1; + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbx = C; + tby = R; + tbz = 1; + gridx = ceil((double)total_thread_x / tbx); + gridy = ceil((double)total_thread_y / tby); + gridz = 1; + tbx = F; // necessary to ensure width is enough + + for (DIM d = 0; d < D; d++) { + if (d != curr_dim_f && d != curr_dim_c && d != curr_dim_r) { + SIZE t = v.shape(d); + gridx *= t; + } + } + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + DIM curr_dim_r, curr_dim_c, curr_dim_f; + SubArray<1, T, DeviceType> am, bm; + SubArray v; +}; + +template +class Ipk2ReoFunctor : public IterFunctor { +public: + MGARDX_CONT Ipk2ReoFunctor() {} + MGARDX_CONT + Ipk2ReoFunctor(DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, + SubArray<1, T, DeviceType> am, SubArray<1, T, DeviceType> bm, + SubArray v) + : curr_dim_r(curr_dim_r), curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), + am(am), bm(bm), v(v) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + threadId = (FunctorBase::GetThreadIdZ() * + (FunctorBase::GetBlockDimX() * + FunctorBase::GetBlockDimY())) + + (FunctorBase::GetThreadIdY() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + + T *sm = (T *)FunctorBase::GetSharedMemory(); + ldsm1 = F; + ldsm2 = C + G; + vec_sm = sm; + sm += R * ldsm1 * ldsm2; + am_sm = sm; + sm += ldsm2; + bm_sm = sm; + sm += ldsm2; + } + + MGARDX_EXEC void Operation2() { + // __syncthreads(); + + for (DIM d = 0; d < D; d++) + idx[d] = 0; + + nr = v.shape(curr_dim_r); + nc = v.shape(curr_dim_c); + nf = v.shape(curr_dim_f); + + if (D < 3) + nr = 1; + + SIZE bidx = FunctorBase::GetBlockIdX(); + SIZE firstD = div_roundup(nf, FunctorBase::GetBlockDimX()); + SIZE blockId = bidx % firstD; + + bidx /= firstD; + + for (DIM d = 0; d < D; d++) { + if (d != curr_dim_r && d != curr_dim_c && d != curr_dim_f) { + SIZE t = v.shape(d); + idx[d] = bidx % t; + bidx /= t; + } + } + + v.offset(idx); + // size_t other_offset_v = get_idx(ldvs_sm, idx); + // v = v + other_offset_v; + + f_gl = blockId * F; + r_gl = FunctorBase::GetBlockIdY() * R; + c_gl = 0; + + f_sm = FunctorBase::GetThreadIdX(); + r_sm = FunctorBase::GetThreadIdY(); + c_sm = FunctorBase::GetThreadIdX(); + + v.offset_3d(r_gl, 0, f_gl); + // T *vec = v + get_idx(ldv1, ldv2, r_gl, 0, f_gl); + + prev_vec_sm = 0.0; + + f_rest = Math::Min(F, nf - blockId * F); + r_rest = Math::Min( + R, nr - FunctorBase::GetBlockIdY() * R); + + // if (FunctorBase::GetBlockIdX() == 1 && + // FunctorBase::GetBlockIdY() == 0 && f_sm == 0 && r_sm == 0) { + // prSIZEf("f_rest: %d r_rest: %d\n", f_rest, r_rest); + // } + + c_rest = nc; + c_ghost = Math::Min(nc, G); + c_main = C; + + /* Load first ghost */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = *v(r_sm, c_gl + i, f_sm); + // if (r_sm == 0) printf("r0_stride = %d, vec_sm[%d] = %f\n", r0_stride, + // i, vec_sm[i * ldsm + c_sm]); + } + } + if (r_sm == 0 && c_sm < c_ghost) { + am_sm[c_sm] = *am(c_gl + c_sm); + bm_sm[c_sm] = *bm(c_gl + c_sm); + } + c_rest -= c_ghost; + } + // __syncthreads(); + + MGARDX_EXEC bool LoopCondition1() { return c_rest > C - c_ghost; } + // while (c_rest > C - c_ghost) { + // printf("%d %d %d\n", c_rest, C, c_ghost); + MGARDX_EXEC void Operation3() { + c_main = Math::Min(C, c_rest); + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_main; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = + *v(r_sm, c_gl + i + c_ghost, f_sm); + } + } + if (r_sm == 0 && c_sm < c_main) { + am_sm[c_sm + c_ghost] = *am(c_gl + c_sm + c_ghost); + bm_sm[c_sm + c_ghost] = *bm(c_gl + c_sm + c_ghost); + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation4() { + + /* Computation of v in parallel*/ + if (r_sm < r_rest && f_sm < f_rest) { + + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = + // __fma_rn(prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, + // r_sm, 0, f_sm)]); + // #else + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] -= prev_vec_sm * + // bm_sm[0]; + // #endif + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); + + for (SIZE i = 1; i < C; i++) { + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + // __fma_rn(vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], + // bm_sm[i], + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); + // #else + // // if (FunctorBase::GetBlockIdX() == 1 && + // FunctorBase::GetBlockIdY() == 0 && f_sm == 0 && + // r_sm + // == 0) { + // // printf("calc: %f %f %f -> %f \n", vec_sm[get_idx(ldsm1, + // ldsm2, r_sm, i, f_sm)], + // // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], + // bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] - + // // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)] * + // bm_sm[i]); + // // } + + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] -= + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)] * bm_sm[i]; + // #endif + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_forward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); + } + /* Store last v */ + prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, r_sm, C - 1, f_sm)]; + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation5() { + + /* flush results to v */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < C; i++) { + // if (FunctorBase::GetBlockIdX() == 1 && + // FunctorBase::GetBlockIdY() == 0 && f_sm == 0 && r_sm == + // 0) { + // printf("store: %f\n", vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, + // f_sm)]); + // } + *v(r_sm, c_gl + i, f_sm) = vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + } + } + } + // __syncthreads(); + MGARDX_EXEC void Operation6() { + /* Update unloaded col */ + c_rest -= c_main; + + /* Advance c */ + c_gl += C; + + /* Copy next ghost to main */ + c_ghost = Math::Min(G, c_main - (C - G)); + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + C, f_sm)]; + } + } + if (r_sm == 0 && c_sm < c_ghost) { + am_sm[c_sm] = am_sm[c_sm + C]; + bm_sm[c_sm] = bm_sm[c_sm + C]; + } + } + // __syncthreads(); + + // } // end of while + + MGARDX_EXEC void Operation7() { + /* Load all rest col */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = + *v(r_sm, c_gl + i + c_ghost, f_sm); + } + } + if (r_sm == 0 && c_sm < c_rest) { + am_sm[c_sm + c_ghost] = *am(c_gl + c_sm + c_ghost); + bm_sm[c_sm + c_ghost] = *bm(c_gl + c_sm + c_ghost); + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation8() { + /* Only 1 col remain */ + if (c_ghost + c_rest == 1) { + if (r_sm < r_rest && f_sm < f_rest) { + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] -= prev_vec_sm * + // bm_sm[0]; #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = + // __fma_rn(prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, + // r_sm, 0, f_sm)]); + // #else + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] -= prev_vec_sm * + // bm_sm[0]; + // #endif + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); + // printf ("prev_vec_sm = %f\n", prev_vec_sm ); + // printf ("vec_sm[r_sm * ldsm + 0] = %f\n", vec_sm[r_sm * ldsm + 0] ); + } + //__syncthreads(); + + } else { + if (r_sm < r_rest && f_sm < f_rest) { + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = + // __fma_rn(prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, + // r_sm, 0, f_sm)]); + // #else + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] -= prev_vec_sm * + // bm_sm[0]; + // #endif + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); + for (SIZE i = 1; i < c_ghost + c_rest; i++) { + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + // __fma_rn(vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], + // bm_sm[i], + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); + // #else + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] -= + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)] * + // bm_sm[i]; + // #endif + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_forward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); + } + } + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation9() { + /* flush results to v */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_ghost + c_rest; i++) { + *v(r_sm, c_gl + i, f_sm) = vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + // printf("c_stride = %d, c_sm = %d, vec_sm = %f, vec[%d] = + // %f\n",c_stride, c_sm, vec_sm[r_sm * ldsm + 0],i * row_stride * lddv + + // c_stride, vec[i * row_stride * lddv + c_stride]); + } + } + } + // __syncthreads(); + MGARDX_EXEC void Operation10() { + /* backward */ + c_rest = nc; + c_ghost = Math::Min(nc, G); + c_main = C; + c_gl = 0; + prev_vec_sm = 0.0; + + // if (f_gl + f_sm == 0 && r_gl + r_sm == 0 && idx[3] == 0) + // debug = false; + // if (debug) + // printf("block id: (%d %d %d) thread id: (%d %d %d)\n", + // FunctorBase::GetBlockIdX(), + // FunctorBase::GetBlockIdY(), blockIdx.z, + // FunctorBase::GetThreadIdX(), + // FunctorBase::GetThreadIdY(), + // FunctorBase::GetThreadIdZ()); + + /* Load first ghost */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + *v(r_sm, (nc - 1) - (c_gl + i), f_sm); + // if (debug) + // printf("load vec_sm[%d] = %f\n", get_idx(ldsm1, ldsm2, r_sm, i, + // f_sm), + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); + } + } + if (r_sm == 0 && c_sm < c_ghost) { + am_sm[c_sm] = *am(nc - (c_gl + c_sm)); + bm_sm[c_sm] = *bm(nc - (c_gl + c_sm)); + } + c_rest -= c_ghost; + } + // __syncthreads(); + + MGARDX_EXEC bool LoopCondition2() { return c_rest > C - c_ghost; } + // while (c_rest > C - c_ghost) { + // printf("%d %d %d\n", c_rest, C, c_ghost); + MGARDX_EXEC void Operation11() { + c_main = Math::Min(C, c_rest); + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_main; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = + *v(r_sm, (nc - 1) - (c_gl + i + c_ghost), f_sm); + // if (debug) + // printf("load vec_sm[%d] = %f\n", + // get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm), + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)]); + } + } + if (r_sm == 0 && c_sm < c_main) { + am_sm[c_sm + c_ghost] = *am(nc - (c_gl + c_sm + c_ghost)); + bm_sm[c_sm + c_ghost] = *bm(nc - (c_gl + c_sm + c_ghost)); + } + } + // __syncthreads(); + MGARDX_EXEC void Operation12() { + // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) + // printf("*****test\n"); + /* Computation of v in parallel*/ + if (r_sm < r_rest && f_sm < f_rest) { + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = + // __fma_rn(dist_sm[0], prev_vec_sm, vec_sm[get_idx(ldsm1, ldsm2, + // r_sm, 0, f_sm)]) * am_sm[0]; + // #else + // // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) + // // printf("(%f + %f * %f) * %f -> %f\n", + // // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)], + // // dist_sm[0], prev_vec_sm, am_sm[0], + // // (vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] - + // dist_sm[0] * prev_vec_sm) / am_sm[0]); vec_sm[get_idx(ldsm1, + // ldsm2, r_sm, 0, f_sm)] = (vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, + // f_sm)] - dist_sm[0] * prev_vec_sm) / am_sm[0]; + // #endif + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)]); + // if (debug) + // printf("calc vec_sm[%d] = %f\n", get_idx(ldsm1, ldsm2, r_sm, 0, + // f_sm), + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); + + for (SIZE i = 1; i < C; i++) { + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + // __fma_rn(dist_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, + // f_sm)], + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)]) * am_sm[i]; + // #else + // // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) + // // printf("(%f + %f * %f) * %f -> %f\n", + // // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)], + // // dist_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, + // i-1, f_sm)], am_sm[i], + // // (vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] - + // // dist_sm[i] * vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, + // f_sm)]) / am_sm[i]); + + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + // (vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] - + // dist_sm[i] * vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, + // f_sm)]) / am_sm[i]; + + // #endif + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_backward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); + + // if (debug) + // printf("calc vec_sm[%d] = %f\n", get_idx(ldsm1, ldsm2, r_sm, i, + // f_sm), + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); + } + + /* Store last v */ + prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, r_sm, C - 1, f_sm)]; + } + } + // __syncthreads(); + MGARDX_EXEC void Operation13() { + /* flush results to v */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < C; i++) { + *v(r_sm, (nc - 1) - (c_gl + i), f_sm) = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + } + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation14() { + /* Update unloaded col */ + c_rest -= c_main; + + /* Advance c */ + c_gl += C; + + /* Copy next ghost to main */ + c_ghost = Math::Min(G, c_main - (C - G)); + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + C, f_sm)]; + } + } + if (r_sm == 0 && c_sm < c_ghost) { + am_sm[c_sm] = am_sm[c_sm + C]; + bm_sm[c_sm] = bm_sm[c_sm + C]; + } + } + // __syncthreads(); + + // } // end of while + + MGARDX_EXEC void Operation15() { + // Load all rest col + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = + *v(r_sm, (nc - 1) - (c_gl + i + c_ghost), f_sm); + + // if (debug) + // printf("load ec_sm[%d] = %f\n", + // get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm), + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)]); + } + } + if (r_sm == 0 && c_sm < c_rest) { + am_sm[c_sm + c_ghost] = *am(nc - (c_gl + c_sm + c_ghost)); + bm_sm[c_sm + c_ghost] = *bm(nc - (c_gl + c_sm + c_ghost)); + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation16() { + /* Only 1 col remain */ + if (c_ghost + c_rest == 1) { + if (r_sm < r_rest && f_sm < f_rest) { + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = + // __fma_rn(dist_sm[0], prev_vec_sm, vec_sm[get_idx(ldsm1, ldsm2, + // r_sm, 0, f_sm)]) * am_sm[0]; + // #else + // // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) + // // printf("(%f + %f * %f) * %f -> %f\n", + // // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)], + // // dist_sm[0], prev_vec_sm, am_sm[0], + // // (vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] - + // dist_sm[0] * prev_vec_sm) / am_sm[0]); vec_sm[get_idx(ldsm1, + // ldsm2, r_sm, 0, f_sm)] = (vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, + // f_sm)] - dist_sm[0] * prev_vec_sm) / am_sm[0]; + // #endif + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)]); + // if (debug) + // printf("calc vec_sm[%d] = %f\n", get_idx(ldsm1, ldsm2, r_sm, 0, + // f_sm), + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); + // printf ("prev_vec_sm = %f\n", prev_vec_sm ); + // printf ("vec_sm[r_sm * ldsm + 0] = %f\n", vec_sm[r_sm * ldsm + 0] ); + } + //__syncthreads(); + + } else { + if (r_sm < r_rest && f_sm < f_rest) { + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = + // __fma_rn(dist_sm[0], prev_vec_sm, vec_sm[get_idx(ldsm1, ldsm2, + // r_sm, 0, f_sm)]) * am_sm[0]; + // #else + // // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) + // // printf("(%f + %f * %f) * %f -> %f\n", + // // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)], + // // dist_sm[0], prev_vec_sm, am_sm[0], + // // (vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] - + // dist_sm[0] * prev_vec_sm) / am_sm[0]); vec_sm[get_idx(ldsm1, + // ldsm2, r_sm, 0, f_sm)] = (vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, + // f_sm)] - dist_sm[0] * prev_vec_sm) / am_sm[0]; + // #endif + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)]); + // if (debug) + // printf("calc vec_sm[%d] = %f\n", get_idx(ldsm1, ldsm2, r_sm, 0, + // f_sm), + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); + for (SIZE i = 1; i < c_ghost + c_rest; i++) { + + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + // __fma_rn(dist_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, + // f_sm)], + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)]) * + // am_sm[i]; + // #else + // // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) + // // printf("(%f + %f * %f) * %f -> %f\n", + // // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)], + // // dist_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, + // i-1, f_sm)], am_sm[i], + // // (vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] + // - + // // dist_sm[i] * vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, + // f_sm)]) / am_sm[i]); + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + // (vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] - + // dist_sm[i] * vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, + // f_sm)]) / am_sm[i]; + // #endif + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_backward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); + // if (debug) + // printf("calc vec_sm[%d] = %f\n", get_idx(ldsm1, ldsm2, r_sm, i, + // f_sm), + // vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); + } + } + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation17() { + /* flush results to v */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_ghost + c_rest; i++) { + *v(r_sm, (nc - 1) - (c_gl + i), f_sm) = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + // printf("c_stride = %d, c_sm = %d, vec_sm = %f, vec[%d] = + // %f\n",c_stride, c_sm, vec_sm[r_sm * ldsm + 0],i * row_stride * lddv + + // c_stride, vec[i * row_stride * lddv + c_stride]); + } + } + // __syncthreads(); + v.reset_offset(); + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + size = (R * F + 2) * (C + G) * sizeof(T); + return size; + } + +private: + // functor parameters + DIM curr_dim_r, curr_dim_c, curr_dim_f; + SubArray<1, T, DeviceType> am, bm; + SubArray v; + + // thread local variables + SIZE threadId; + + T *vec_sm; + SIZE ldsm1, ldsm2; + T *am_sm; + T *bm_sm; + + SIZE idx[D]; + + SIZE nr; + SIZE nc; + SIZE nf; + + SIZE c_gl, r_gl, f_gl; + SIZE c_sm, r_sm, f_sm; + + T prev_vec_sm; + SIZE f_rest, r_rest; + SIZE c_rest, c_ghost, c_main; +}; + +template +class Ipk2ReoKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "ipk2_nd"; + constexpr static SIZE G = 2; + MGARDX_CONT + Ipk2ReoKernel(DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, + SubArray<1, T, DeviceType> am, SubArray<1, T, DeviceType> bm, + SubArray v) + : curr_dim_r(curr_dim_r), curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), + am(am), bm(bm), v(v) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Ipk2ReoFunctor; + FunctorType functor(curr_dim_r, curr_dim_c, curr_dim_f, am, bm, v); + + SIZE nr = 1, nf = 1; + if (D >= 3) + nr = v.shape(curr_dim_r); + nf = v.shape(curr_dim_f); + + SIZE total_thread_x = nf; + SIZE total_thread_y = nr; + SIZE total_thread_z = 1; + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbx = F; + tby = R; + tbz = 1; + gridx = ceil((double)total_thread_x / tbx); + gridy = ceil((double)total_thread_y / tby); + gridz = 1; + // tbx = F; // necessary to ensure width is enough + + for (DIM d = 0; d < D; d++) { + if (d != curr_dim_f && d != curr_dim_c && d != curr_dim_r) { + SIZE t = v.shape(d); + gridx *= t; + } + } + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + DIM curr_dim_r, curr_dim_c, curr_dim_f; + SubArray<1, T, DeviceType> am, bm; + SubArray v; +}; + +template +class Ipk3ReoFunctor : public IterFunctor { +public: + MGARDX_CONT Ipk3ReoFunctor() {} + MGARDX_CONT + Ipk3ReoFunctor(DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, + SubArray<1, T, DeviceType> am, SubArray<1, T, DeviceType> bm, + SubArray v) + : curr_dim_r(curr_dim_r), curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), + am(am), bm(bm), v(v) { + IterFunctor(); + } + + MGARDX_EXEC void Operation1() { + threadId = (FunctorBase::GetThreadIdZ() * + (FunctorBase::GetBlockDimX() * + FunctorBase::GetBlockDimY())) + + (FunctorBase::GetThreadIdY() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + + T *sm = (T *)FunctorBase::GetSharedMemory(); + ldsm1 = F; + ldsm2 = C; + + vec_sm = sm; + sm += (R + G) * ldsm1 * ldsm2; + am_sm = sm; + sm += (R + G); + bm_sm = sm; + sm += (R + G); + } + + // __syncthreads(); + + MGARDX_EXEC void Operation2() { + for (DIM d = 0; d < D; d++) + idx[d] = 0; + + nr = v.shape(curr_dim_r); + nc = v.shape(curr_dim_c); + nf = v.shape(curr_dim_f); + + SIZE bidx = FunctorBase::GetBlockIdX(); + SIZE firstD = div_roundup(nf, FunctorBase::GetBlockDimX()); + SIZE blockId = bidx % firstD; + + bidx /= firstD; + + for (DIM d = 0; d < D; d++) { + if (d != curr_dim_r && d != curr_dim_c && d != curr_dim_f) { + SIZE t = v.shape(d); + idx[d] = bidx % t; + bidx /= t; + } + } + + v.offset(idx); + // size_t other_offset_v = get_idx(ldvs_sm, idx); + // v = v + other_offset_v; + + f_gl = blockId * F; + c_gl = FunctorBase::GetBlockIdY() * C; + r_gl = 0; + + f_sm = FunctorBase::GetThreadIdX(); + c_sm = FunctorBase::GetThreadIdY(); + r_sm = FunctorBase::GetThreadIdX(); + + v.offset_3d(0, c_gl, f_gl); + // T *vec = v + get_idx(ldv1, ldv2, 0, c_gl, f_gl); + + prev_vec_sm = 0.0; + + f_rest = Math::Min(F, (SIZE)(nf - blockId * F)); + c_rest = Math::Min( + C, (SIZE)(nc - FunctorBase::GetBlockIdY() * C)); + + r_rest = nr; + r_ghost = Math::Min(nr, G); + r_main = R; + + // if (f_gl + f_sm == 32 && c_gl + c_sm == 1 ) debug = false; + + /* Load first ghost */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = *v(r_gl + i, c_sm, f_sm); + // if (debug) printf("load first sm[%d] %f [%d]\n", i, + // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)], r_gl + i); + } + } + + if (c_sm == 0 && r_sm < r_ghost) { + am_sm[r_sm] = *am(r_gl + r_sm); + bm_sm[r_sm] = *bm(r_gl + r_sm); + } + r_rest -= r_ghost; + } + + MGARDX_EXEC bool LoopCondition1() { return r_rest > R - r_ghost; } + // __syncthreads(); + + MGARDX_EXEC void Operation3() { + // while (r_rest > R - r_ghost) { + r_main = Math::Min(R, r_rest); + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_main; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = + *v(r_gl + i + r_ghost, c_sm, f_sm); + // if (debug) printf("load ghost sm[%d] %f [%d]\n", i + r_ghost, + // vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)], + // r_gl + i + r_ghost); + } + } + if (c_sm == 0 && r_sm < r_main) { + am_sm[r_sm + r_ghost] = *am(r_gl + r_sm + r_ghost); + bm_sm[r_sm + r_ghost] = *bm(r_gl + r_sm + r_ghost); + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation4() { + /* Computation of v in parallel*/ + if (c_sm < c_rest && f_sm < f_rest) { + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); + + for (SIZE i = 1; i < R; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_forward2( + vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); + } + + /* Store last v */ + prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, R - 1, c_sm, f_sm)]; + } + } + // __syncthreads(); + MGARDX_EXEC void Operation5() { + /* flush results to v */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < R; i++) { + *v(r_gl + i, c_sm, f_sm) = vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; + // if (debug) printf("store[%d] %f [%d]\n", r_gl + i, + // *v(r_gl + i, c_sm, f_sm), i); + } + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation6() { + + // /* Update unloaded col */ + r_rest -= r_main; + + /* Advance c */ + r_gl += R; + + /* Copy next ghost to main */ + r_ghost = Math::Min(G, r_main - (R - G)); + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = + vec_sm[get_idx(ldsm1, ldsm2, i + R, c_sm, f_sm)]; + + // if (debug) printf("copy next ghost[%d] %f [%d]\n", i, + // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)], i+R); + } + } + if (c_sm == 0 && r_sm < r_ghost) { + am_sm[r_sm] = am_sm[r_sm + R]; + bm_sm[r_sm] = bm_sm[r_sm + R]; + } + } + + // // __syncthreads(); + + // // } // end of while + + MGARDX_EXEC void Operation7() { + /* Load all rest col */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = + *v(r_gl + i + r_ghost, c_sm, f_sm); + + // if (debug) printf("load ghost-rest sm[%d] %f [%d]\n", i + r_ghost, + // vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)], + // r_gl + i + r_ghost); + } + } + + if (c_sm == 0 && r_sm < r_rest) { + am_sm[r_sm + r_ghost] = *am(r_gl + r_sm + r_ghost); + bm_sm[r_sm + r_ghost] = *bm(r_gl + r_sm + r_ghost); + } + } + + // __syncthreads(); + + MGARDX_EXEC void Operation8() { + + /* Only 1 col remain */ + if (r_ghost + r_rest == 1) { + if (c_sm < c_rest && f_sm < f_rest) { + + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + // __fma_rn(prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, 0, + // c_sm, f_sm)]); + // #else + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] -= prev_vec_sm * + // bm_sm[0]; + // #endif + // if (debug) printf("compute-rest sm[%d] %f <- %f %f %f\n", 0, + // tridiag_forward(prev_vec_sm, bm_sm[0], + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]), + // prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, 0, + // c_sm, f_sm)]); + + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); + // printf ("prev_vec_sm = %f\n", prev_vec_sm ); + // printf ("vec_sm[r_sm * ldsm + 0] = %f\n", vec_sm[r_sm * ldsm + 0] ); + } + //__syncthreads(); + + } else { + if (c_sm < c_rest && f_sm < f_rest) { + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + // __fma_rn(prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, 0, + // c_sm, f_sm)]); + // #else + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] -= prev_vec_sm * + // bm_sm[0]; + // #endif + + // if (debug) printf("compute-rest sm[%d] %f <- %f %f %f\n", 0, + // tridiag_forward(prev_vec_sm, bm_sm[0], + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]), + // prev_vec_sm, bm_sm[0], vec_sm[get_idx(ldsm1, ldsm2, 0, + // c_sm, f_sm)]); + + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); + for (SIZE i = 1; i < r_ghost + r_rest; i++) { + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = + // __fma_rn(vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], + // bm_sm[i], + // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); + // #else + // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] -= + // vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)] * + // bm_sm[i]; + // #endif + // if (debug) printf("compute-rest sm[%d] %f <- %f %f %f\n", i, + // tridiag_forward(vec_sm[get_idx(ldsm1, ldsm2, i - 1, + // c_sm, f_sm)], + // bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, + // f_sm)]), + // vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], + // bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, + // f_sm)]); + + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_forward2( + vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); + } + } + } + } + + MGARDX_EXEC void Operation9() { + // __syncthreads(); + /* flush results to v */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_ghost + r_rest; i++) { + + *v(r_gl + i, c_sm, f_sm) = vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; + + // if (debug) printf("store-rest[%d] %f [%d]\n", r_gl + i, + // *v(r_gl + i, c_sm, f_sm), i); + // printf("c_stride = %d, c_sm = %d, vec_sm = %f, vec[%d] = + // %f\n",c_stride, c_sm, vec_sm[r_sm * ldsm + 0],i * row_stride * lddv + + // c_stride, vec[i * row_stride * lddv + c_stride]); + } + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation10() { + + /* backward */ + r_rest = nr; + r_ghost = Math::Min(nr, G); + r_main = R; + r_gl = 0; + prev_vec_sm = 0.0; + + /* Load first ghost */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = + *v((nr - 1) - (r_gl + i), c_sm, f_sm); + } + } + + if (c_sm == 0 && r_sm < r_ghost) { + am_sm[r_sm] = *am(nr - (r_gl + r_sm)); + bm_sm[r_sm] = *bm(nr - (r_gl + r_sm)); + } + r_rest -= r_ghost; + } + // __syncthreads(); + + MGARDX_EXEC bool LoopCondition2() { return r_rest > R - r_ghost; } + + MGARDX_EXEC void Operation11() { + // while (r_rest > R - r_ghost) { + r_main = Math::Min(R, r_rest); + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_main; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = + *v((nr - 1) - (r_gl + i + r_ghost), c_sm, f_sm); + // if (debug) printf("load ghost sm[%d] %f [%d]\n", i + r_ghost, + // vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)], + // (nr - 1) - (r_gl + i + r_ghost)); + } + } + if (c_sm == 0 && r_sm < r_main) { + am_sm[r_sm + r_ghost] = *am(nr - (r_gl + r_sm + r_ghost)); + bm_sm[r_sm + r_ghost] = *bm(nr - (r_gl + r_sm + r_ghost)); + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation12() { + /* Computation of v in parallel*/ + if (c_sm < c_rest && f_sm < f_rest) { + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + // __fma_rn(dist_sm[0], prev_vec_sm, vec_sm[get_idx(ldsm1, ldsm2, 0, + // c_sm, f_sm)]) * am_sm[0]; + // #else + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + // (vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] - dist_sm[0] * + // prev_vec_sm) / am_sm[0]; + // #endif + + // if (debug) printf("compute sm[%d] %f <- %f %f %f %f\n", 0, + // tridiag_backward(prev_vec_sm, dist_sm[0], am_sm[0], + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]), + // prev_vec_sm, dist_sm[0], am_sm[0], + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); + + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); + for (SIZE i = 1; i < R; i++) { + + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = + // __fma_rn(dist_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, + // f_sm)], + // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]) * am_sm[i]; + // #else + // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = + // (vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] - + // dist_sm[i] * vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, + // f_sm)]) / am_sm[i]; + // #endif + + // if (debug) printf("compute sm[%d] %f <- %f %f %f %f\n", i, + // tridiag_backward(vec_sm[get_idx(ldsm1, ldsm2, i - 1, + // c_sm, f_sm)], + // dist_sm[i], am_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]), + // vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], + // dist_sm[i], am_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); + + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_backward2( + vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); + } + + /* Store last v */ + prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, R - 1, c_sm, f_sm)]; + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation13() { + /* flush results to v */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < R; i++) { + // if (FunctorBase::GetBlockIdX() == 0 && + // FunctorBase::GetBlockIdY() == 0 && + // FunctorBase::GetThreadIdX() == 0 && + // FunctorBase::GetThreadIdY() == 0) { + // printf("%d %d %d (%f) <- %d %d %d\n", (nr - 1) - (r_gl + i), c_sm, + // f_sm, + // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)], i, c_sm, + // f_sm); + // } + *v((nr - 1) - (r_gl + i), c_sm, f_sm) = + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; + + // if (debug) printf("store[%d] %f [%d]\n", (nr - 1) - (r_gl + i), + // *v((nr - 1) - (r_gl + i), c_sm, f_sm), i); + } + } + } + // __syncthreads(); + + MGARDX_EXEC void Operation14() { + // /* Update unloaded col */ + r_rest -= r_main; + + /* Advance c */ + r_gl += R; + + /* Copy next ghost to main */ + r_ghost = Math::Min(G, r_main - (R - G)); + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = + vec_sm[get_idx(ldsm1, ldsm2, i + R, c_sm, f_sm)]; + + // if (debug) printf("copy next ghost[%d] %f [%d]\n", i, + // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)], i+R); + } + } + if (c_sm == 0 && r_sm < r_ghost) { + am_sm[r_sm] = am_sm[r_sm + R]; + bm_sm[r_sm] = bm_sm[r_sm + R]; + } + } + // __syncthreads(); + + // } // end of while + + MGARDX_EXEC void Operation15() { + /* Load all rest col */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = + *v((nr - 1) - (r_gl + i + r_ghost), c_sm, f_sm); + + // if (debug) printf("load ghost-rest sm[%d] %f [%d]\n", i + r_ghost, + // vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)], + // (nr - 1) - (r_gl + i + r_ghost)); + } + } + if (c_sm == 0 && r_sm < r_rest) { + am_sm[r_sm + r_ghost] = *am(nr - (r_gl + r_sm + r_ghost)); + bm_sm[r_sm + r_ghost] = *bm(nr - (r_gl + r_sm + r_ghost)); + } + // __syncthreads(); + } + + MGARDX_EXEC void Operation16() { + /* Only 1 col remain */ + if (r_ghost + r_rest == 1) { + if (c_sm < c_rest && f_sm < f_rest) { + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + // __fma_rn(dist_sm[0], prev_vec_sm, vec_sm[get_idx(ldsm1, ldsm2, + // 0, c_sm, f_sm)]) * am_sm[0]; + // #else + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + // (vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] - dist_sm[0] * + // prev_vec_sm) / am_sm[0]; + // #endif + // if (FunctorBase::GetBlockIdX() == 0 && + // FunctorBase::GetBlockIdY() == 0 && + // FunctorBase::GetThreadIdX() == 0 && + // FunctorBase::GetThreadIdY() == 0) { + // printf("backward 1 (%f) %f %f %f %f\n", + // tridiag_backward(prev_vec_sm, dist_sm[0], am_sm[0], + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]), prev_vec_sm, + // dist_sm[0], am_sm[0], vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, + // f_sm)]); + + // } + // if (debug) printf("compute sm[%d] %f <- %f %f %f %f\n", 0, + // tridiag_backward(prev_vec_sm, dist_sm[0], am_sm[0], + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]), + // prev_vec_sm, dist_sm[0], am_sm[0], + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); + + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); + // printf ("prev_vec_sm = %f\n", prev_vec_sm ); + // printf ("vec_sm[r_sm * ldsm + 0] = %f\n", vec_sm[r_sm * ldsm + 0] ); + } + //__syncthreads(); + + } else { + if (c_sm < c_rest && f_sm < f_rest) { + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + // __fma_rn(dist_sm[0], prev_vec_sm, vec_sm[get_idx(ldsm1, ldsm2, + // 0, c_sm, f_sm)]) * am_sm[0]; + // #else + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + // (vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] - dist_sm[0] * + // prev_vec_sm) / am_sm[0]; + // #endif + // if (FunctorBase::GetBlockIdX() == 0 && + // FunctorBase::GetBlockIdY() == 0 && + // FunctorBase::GetThreadIdX() == 0 && + // FunctorBase::GetThreadIdY() == 0) { + // printf("backward 1 (%f) %f %f %f %f\n", + // tridiag_backward(prev_vec_sm, dist_sm[0], am_sm[0], + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]), prev_vec_sm, + // dist_sm[0], am_sm[0], vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, + // f_sm)]); + + // } + + // if (debug) printf("compute sm[%d] %f <- %f %f %f %f\n", 0, + // tridiag_backward(prev_vec_sm, dist_sm[0], am_sm[0], + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]), + // prev_vec_sm, dist_sm[0], am_sm[0], + // vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); + + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); + for (SIZE i = 1; i < r_ghost + r_rest; i++) { + + // #ifdef MGARD_X_160_FMA + // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = + // __fma_rn(dist_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i - 1, + // c_sm, f_sm)], + // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]) * am_sm[i]; + // #else + // vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = + // (vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] - + // dist_sm[i] * vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, + // f_sm)]) / am_sm[i]; + // #endif + // if (FunctorBase::GetBlockIdX() == 0 && + // FunctorBase::GetBlockIdY() == 0 && + // FunctorBase::GetThreadIdX() == 0 && + // FunctorBase::GetThreadIdY() == 0) { printf("backward + // R=%d (%f) %f %f %f %f\n", i, + // tridiag_backward(vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, + // f_sm)], + // dist_sm[i], am_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, + // f_sm)]), vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], + // dist_sm[i], am_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, + // f_sm)]); + + // } + + // if (debug) printf("compute sm[%d] %f <- %f %f %f %f\n", i, + // tridiag_backward(vec_sm[get_idx(ldsm1, ldsm2, i - 1, + // c_sm, f_sm)], + // dist_sm[i], am_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, + // f_sm)]), + // vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], + // dist_sm[i], am_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, + // f_sm)]); + + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_backward2( + vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); + } + } + } + } + + MGARDX_EXEC void Operation17() { + // __syncthreads(); + /* flush results to v */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_ghost + r_rest; i++) { + *v((nr - 1) - (r_gl + i), c_sm, f_sm) = + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; + + // if (debug) printf("store-rest[%d] %f [%d]\n", (nr - 1) - (r_gl + i), + // *v((nr - 1) - (r_gl + i), c_sm, f_sm), i); + + // printf("c_stride = %d, c_sm = %d, vec_sm = %f, vec[%d] = + // %f\n",c_stride, c_sm, vec_sm[r_sm * ldsm + 0],i * row_stride * lddv + + // c_stride, vec[i * row_stride * lddv + c_stride]); + } + } + // __syncthreads(); + v.reset_offset(); + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + size = (C * F + 2) * (R + G) * sizeof(T); + return size; + } + +private: + // functor parameters + DIM curr_dim_r, curr_dim_c, curr_dim_f; + SubArray<1, T, DeviceType> am, bm; + SubArray v; + + // thread local variables + SIZE threadId; + + T *vec_sm; + SIZE ldsm1, ldsm2; + T *am_sm; + T *bm_sm; + + SIZE idx[D]; + + SIZE nr; + SIZE nc; + SIZE nf; + + SIZE c_gl, r_gl, f_gl; + SIZE c_sm, r_sm, f_sm; + + T prev_vec_sm; + SIZE c_rest, f_rest; + SIZE r_rest, r_ghost, r_main; +}; + +template +class Ipk3ReoKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "ipk3_nd"; + constexpr static SIZE G = 2; + MGARDX_CONT + Ipk3ReoKernel(DIM curr_dim_r, DIM curr_dim_c, DIM curr_dim_f, + SubArray<1, T, DeviceType> am, SubArray<1, T, DeviceType> bm, + SubArray v) + : curr_dim_r(curr_dim_r), curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), + am(am), bm(bm), v(v) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Ipk3ReoFunctor; + FunctorType functor(curr_dim_r, curr_dim_c, curr_dim_f, am, bm, v); + + SIZE nc = 1, nf = 1; + if (D >= 2) + nc = v.shape(curr_dim_c); + nf = v.shape(curr_dim_f); + + SIZE total_thread_x = nf; + SIZE total_thread_y = nc; + SIZE total_thread_z = 1; + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbx = F; + tby = C; + tbz = 1; + gridx = ceil((double)total_thread_x / tbx); + gridy = ceil((double)total_thread_y / tby); + gridz = 1; + // tbx = F; // necessary to ensure width is enough + + for (DIM d = 0; d < D; d++) { + if (d != curr_dim_f && d != curr_dim_c && d != curr_dim_r) { + SIZE t = v.shape(d); + gridx *= t; + } + } + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + DIM curr_dim_r, curr_dim_c, curr_dim_f; + SubArray<1, T, DeviceType> am, bm; + SubArray v; +}; + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel3D.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel3D.hpp new file mode 100644 index 000000000..d3d3de2d5 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/IterativeProcessingKernel3D.hpp @@ -0,0 +1,1169 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_ITERATIVE_PROCESSING_KERNEL_3D_TEMPLATE +#define MGARD_X_160_ITERATIVE_PROCESSING_KERNEL_3D_TEMPLATE + +#include "../../../RuntimeX/RuntimeX.h" +#include "IPKFunctor.h" +// #include "IterativeProcessingKernel3D.h" + +// #include "../../Functor.h" +// #include "../../AutoTuners/AutoTuner.h" +// #include "../../Task.h" +// #include "../../DeviceAdapters/DeviceAdapter.h" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +class Ipk1Reo3DFunctor : public IterFunctor { +public: + MGARDX_CONT Ipk1Reo3DFunctor() {} + MGARDX_CONT Ipk1Reo3DFunctor(SIZE nr, SIZE nc, SIZE nf, + SubArray<1, T, DeviceType> am, + SubArray<1, T, DeviceType> bm, + SubArray<1, T, DeviceType> dist_f, + SubArray v) + : nr(nr), nc(nc), nf(nf), am(am), bm(bm), dist_f(dist_f), v(v) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + c_gl = FunctorBase::GetBlockIdX() * C; + r_gl = FunctorBase::GetBlockIdY() * R; + f_gl = FunctorBase::GetThreadIdX(); + + c_sm = FunctorBase::GetThreadIdX(); + r_sm = FunctorBase::GetThreadIdY(); + f_sm = FunctorBase::GetThreadIdX(); + + v.offset_3d(r_gl, c_gl, 0); + T *sm = (T *)FunctorBase::GetSharedMemory(); + ldsm1 = F + G; + ldsm2 = C; + vec_sm = sm; + am_sm = sm + R * ldsm1 * ldsm2; + bm_sm = am_sm + ldsm1; + + prev_vec_sm = 0.0; + + c_rest = Math::Min( + C, nc - FunctorBase::GetBlockIdX() * C); + r_rest = Math::Min( + R, nr - FunctorBase::GetBlockIdY() * R); + + // printf("r_rest: %u, c_rest: %u\n", r_rest, c_rest); + // printf("RCF: %u %u %u\n", R,C,F); + // printf("n: %u %u %u\n", nr, nc, nf_c); + f_rest = nf; + f_ghost = Math::Min(nf, G); + // printf("G%u, f_ghost:%u\n ", G, f_ghost); + f_main = F; + + /* Load first ghost */ + if (r_sm < r_rest && f_sm < f_ghost) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = *v(r_sm, i, f_gl); + } + if (r_sm == 0) { + am_sm[f_sm] = *am(f_gl); + bm_sm[f_sm] = *bm(f_gl); + // printf("am[%u]: %f, bm[%u]: %f\n", f_sm, f_sm, am_sm[f_sm], + // bm_sm[f_sm]); + } + } + + f_rest -= f_ghost; + } + + MGARDX_EXEC void Operation2() {} + + MGARDX_EXEC bool LoopCondition1() { return f_rest > F - f_ghost; } + + MGARDX_EXEC void Operation3() { + f_main = Math::Min(F, f_rest); + if (r_sm < r_rest && f_sm < f_main) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = + *v(r_sm, i, f_gl + f_ghost); + } + if (r_sm == 0) { + am_sm[f_sm + f_ghost] = *am(f_gl + f_ghost); + bm_sm[f_sm + f_ghost] = *bm(f_gl + f_ghost); + // printf("am[%u]: %f, bm[%u]: %f\n", f_sm + f_ghost, f_sm + f_ghost, + // am_sm[f_sm + f_ghost], bm_sm[f_sm + f_ghost]); + } + } + } + + MGARDX_EXEC void Operation4() { + /* Computation of v in parallel*/ + if (r_sm < r_rest && c_sm < c_rest) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); + + // #pragma unroll 32 + for (SIZE i = 1; i < F; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_forward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); + } + + /* Store last v */ + prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, F - 1)]; + } + } + + MGARDX_EXEC void Operation5() { + /* flush results to v */ + if (r_sm < r_rest && f_sm < F) { + for (SIZE i = 0; i < c_rest; i++) { + *v(r_sm, i, f_gl) = vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + } + } + } + + MGARDX_EXEC void Operation6() { + /* Update unloaded col */ + f_rest -= f_main; + + /* Advance c */ + f_gl += F; + + /* Copy next ghost to main */ + f_ghost = Math::Min(G, f_main - (F - G)); + if (r_sm < r_rest && f_sm < f_ghost) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + F)]; + } + if (r_sm == 0) { + am_sm[f_sm] = am_sm[f_sm + FunctorBase::GetBlockDimX()]; + bm_sm[f_sm] = bm_sm[f_sm + FunctorBase::GetBlockDimX()]; + } + } + } + + MGARDX_EXEC void Operation7() { + /* Load all rest col */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = + *v(r_sm, i, f_gl + f_ghost); + } + if (r_sm == 0) { + am_sm[f_sm + f_ghost] = *am(f_gl + f_ghost); + bm_sm[f_sm + f_ghost] = *bm(f_gl + f_ghost); + // printf("am-ghost[%u]: %f, bm-ghost[%u]: %f\n", f_sm + f_ghost, f_sm + + // f_ghost, am_sm[f_sm + f_ghost], bm_sm[f_sm + f_ghost]); + } + } + } + + MGARDX_EXEC void Operation8() { + /* Only 1 col remain */ + if (f_ghost + f_rest == 1) { + if (r_sm < r_rest && c_sm < c_rest) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); + } + //__syncthreads(); + + } else { + if (r_sm < r_rest && c_sm < c_rest) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); + // printf("am: %f, bm: %f\n", am_sm[0], bm_sm[0]); + for (SIZE i = 1; i < f_ghost + f_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_forward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); + // printf("am: %f, bm: %f\n", am_sm[i], bm_sm[i]); + } + } + } + } + + MGARDX_EXEC void Operation9() { + /* flush results to v */ + if (r_sm < r_rest && f_sm < f_ghost + f_rest) { + for (SIZE i = 0; i < c_rest; i++) { + *v(r_sm, i, f_gl) = vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + } + } + } + + MGARDX_EXEC void Operation10() { + /* backward */ + f_rest = nf; + f_ghost = Math::Min(nf, G); + f_main = F; + f_gl = FunctorBase::GetThreadIdX(); + prev_vec_sm = 0.0; + + /* Load first ghost */ + if (r_sm < r_rest && f_sm < f_ghost) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + *v(r_sm, i, (nf - 1) - f_gl); + } + } + if (r_sm == 0 && f_gl <= nf) { + am_sm[f_sm] = *am(nf - f_gl); + bm_sm[f_sm] = *bm(nf - f_gl); // * -1; + } + f_rest -= f_ghost; + } + + MGARDX_EXEC bool LoopCondition2() { return f_rest > F - f_ghost; } + + MGARDX_EXEC void Operation11() { + f_main = Math::Min(F, f_rest); + if (r_sm < r_rest && f_sm < f_main) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = + *v(r_sm, i, (nf - 1) - f_gl - f_ghost); + } + } + if (r_sm == 0 && f_gl + f_ghost <= nf) { + am_sm[f_sm + f_ghost] = *am(nf - f_gl - f_ghost); + bm_sm[f_sm + f_ghost] = *bm(nf - f_gl - f_ghost); // * -1; + } + } + + MGARDX_EXEC void Operation12() { + /* Computation of v in parallel*/ + if (r_sm < r_rest && c_sm < c_rest) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); + // #pragma unroll 32 + for (SIZE i = 1; i < F; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_backward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); + } + /* Store last v */ + prev_vec_sm = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, + FunctorBase::GetBlockDimX() - 1)]; + } + } + + MGARDX_EXEC void Operation13() { + /* flush results to v */ + if (r_sm < r_rest && f_sm < F) { + for (SIZE i = 0; i < c_rest; i++) { + *v(r_sm, i, (nf - 1) - f_gl) = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + } + } + } + + MGARDX_EXEC void Operation14() { + /* Update unloaded col */ + f_rest -= f_main; + + /* Advance c */ + f_gl += F; + + /* Copy next ghost to main */ + f_ghost = Math::Min(G, f_main - (F - G)); + if (r_sm < r_rest && f_sm < f_ghost) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + F)]; + } + if (r_sm == 0) { + am_sm[f_sm] = am_sm[f_sm + F]; + bm_sm[f_sm] = bm_sm[f_sm + F]; + } + } + } + + MGARDX_EXEC void Operation15() { + /* Load all rest col */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm + f_ghost)] = + *v(r_sm, i, (nf - 1) - f_gl - f_ghost); + } + } + if (r_sm == 0 && f_gl + f_ghost <= nf) { + am_sm[f_sm + f_ghost] = *am(nf - f_gl - f_ghost); + bm_sm[f_sm + f_ghost] = *bm(nf - f_gl - f_ghost); + // printf("%u %u\n", f_gl, f_ghost); + } + } + + MGARDX_EXEC void Operation16() { + /* Only 1 col remain */ + if (f_ghost + f_rest == 1) { + if (r_sm < r_rest && c_sm < c_rest) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); + } + //__syncthreads(); + + } else { + if (r_sm < r_rest && c_sm < c_rest) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)]); + for (SIZE i = 1; i < f_ghost + f_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)] = tridiag_backward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i - 1)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, i)]); + } + } + } + } + + MGARDX_EXEC void Operation17() { + /* flush results to v */ + if (r_sm < r_rest && f_sm < f_ghost + f_rest) { + for (SIZE i = 0; i < c_rest; i++) { + *v(r_sm, i, (nf - 1) - f_gl) = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + } + } + + v.reset_offset(); + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + size = (R * C + 2) * (F + G) * sizeof(T); + return size; + } + +private: + // functor parameters + SIZE nr, nc, nf; + SubArray<1, T, DeviceType> am, bm; + SubArray<1, T, DeviceType> dist_f; + SubArray v; + + // thread local variables + SIZE c_gl, r_gl, f_gl; + SIZE c_sm, r_sm, f_sm; + SIZE ldsm1, ldsm2; + T *vec_sm; + T *am_sm; + T *bm_sm; + T prev_vec_sm; + SIZE c_rest, r_rest; + SIZE f_rest, f_ghost, f_main; +}; + +template +class Ipk1Reo3DKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "ipk1_3d"; + constexpr static SIZE G = 2; + MGARDX_CONT + Ipk1Reo3DKernel(SIZE nr, SIZE nc, SIZE nf, SubArray<1, T, DeviceType> am, + SubArray<1, T, DeviceType> bm, + SubArray<1, T, DeviceType> dist_f, + SubArray v) + : nr(nr), nc(nc), nf(nf), am(am), bm(bm), dist_f(dist_f), v(v) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Ipk1Reo3DFunctor; + FunctorType functor(nr, nc, nf, am, bm, dist_f, v); + + SIZE total_thread_x = nc; + SIZE total_thread_y = nr; + SIZE total_thread_z = 1; + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbx = C; + tby = R; + tbz = 1; + gridx = ceil((double)total_thread_x / tbx); + gridy = ceil((double)total_thread_y / tby); + gridz = 1; + tbx = F; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE nr, nc, nf; + SubArray<1, T, DeviceType> am, bm; + SubArray<1, T, DeviceType> dist_f; + SubArray v; +}; + +template +class Ipk2Reo3DFunctor : public IterFunctor { +public: + MGARDX_CONT Ipk2Reo3DFunctor() {} + MGARDX_CONT Ipk2Reo3DFunctor(SIZE nr, SIZE nc, SIZE nf, + SubArray<1, T, DeviceType> am, + SubArray<1, T, DeviceType> bm, + SubArray<1, T, DeviceType> dist_c, + SubArray v) + : nr(nr), nc(nc), nf(nf), am(am), bm(bm), dist_c(dist_c), v(v) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + f_gl = FunctorBase::GetBlockIdX() * F; + r_gl = FunctorBase::GetBlockIdY() * R; + c_gl = 0; + + f_sm = FunctorBase::GetThreadIdX(); + r_sm = FunctorBase::GetThreadIdY(); + c_sm = FunctorBase::GetThreadIdX(); + + v.offset_3d(r_gl, 0, f_gl); + T *sm = (T *)FunctorBase::GetSharedMemory(); + ldsm1 = F; + ldsm2 = C + G; + vec_sm = sm; + am_sm = sm + R * ldsm1 * ldsm2; + bm_sm = am_sm + ldsm2; + + prev_vec_sm = 0.0; + + f_rest = Math::Min( + F, nf - FunctorBase::GetBlockIdX() * F); + r_rest = Math::Min( + R, nr - FunctorBase::GetBlockIdY() * R); + + c_rest = nc; + c_ghost = Math::Min(nc, G); + c_main = C; + + /* Load first ghost */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = *v(r_sm, c_gl + i, f_sm); + // if (r_sm == 0) printf("r0_stride = %d, vec_sm[%d] = %f\n", r0_stride, + // i, vec_sm[i * ldsm + c_sm]); + } + } + if (r_sm == 0 && c_sm < c_ghost) { + am_sm[c_sm] = *am(c_gl + c_sm); + bm_sm[c_sm] = *bm(c_gl + c_sm); + } + c_rest -= c_ghost; + } + + MGARDX_EXEC void Operation2() {} + + MGARDX_EXEC bool LoopCondition1() { return c_rest > C - c_ghost; } + + MGARDX_EXEC void Operation3() { + c_main = Math::Min(C, c_rest); + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_main; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = + *v(r_sm, c_gl + i + c_ghost, f_sm); + } + } + if (r_sm == 0 && c_sm < c_main) { + am_sm[c_sm + c_ghost] = *am(c_gl + c_sm + c_ghost); + bm_sm[c_sm + c_ghost] = *bm(c_gl + c_sm + c_ghost); + } + } + + MGARDX_EXEC void Operation4() { + /* Computation of v in parallel*/ + if (r_sm < r_rest && f_sm < f_rest) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); + + for (SIZE i = 1; i < C; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_forward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); + } + /* Store last v */ + prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, r_sm, C - 1, f_sm)]; + } + } + + MGARDX_EXEC void Operation5() { + /* flush results to v */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < C; i++) { + *v(r_sm, c_gl + i, f_sm) = vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + } + } + } + + MGARDX_EXEC void Operation6() { + /* Update unloaded col */ + c_rest -= c_main; + + /* Advance c */ + c_gl += C; + + /* Copy next ghost to main */ + c_ghost = Math::Min(G, c_main - (C - G)); + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + C, f_sm)]; + } + } + if (r_sm == 0 && c_sm < c_ghost) { + am_sm[c_sm] = am_sm[c_sm + C]; + bm_sm[c_sm] = bm_sm[c_sm + C]; + } + } + + MGARDX_EXEC void Operation7() { + /* Load all rest col */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = + *v(r_sm, c_gl + i + c_ghost, f_sm); + } + } + if (r_sm == 0 && c_sm < c_rest) { + am_sm[c_sm + c_ghost] = *am(c_gl + c_sm + c_ghost); + bm_sm[c_sm + c_ghost] = *bm(c_gl + c_sm + c_ghost); + } + } + + MGARDX_EXEC void Operation8() { + /* Only 1 col remain */ + if (c_ghost + c_rest == 1) { + if (r_sm < r_rest && f_sm < f_rest) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); + } + //__syncthreads(); + + } else { + if (r_sm < r_rest && f_sm < f_rest) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)]); + for (SIZE i = 1; i < c_ghost + c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_forward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); + } + } + } + } + + MGARDX_EXEC void Operation9() { + /* flush results to v */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_ghost + c_rest; i++) { + *v(r_sm, c_gl + i, f_sm) = vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + } + } + } + + MGARDX_EXEC void Operation10() { + /* backward */ + c_rest = nc; + c_ghost = Math::Min(nc, G); + c_main = C; + c_gl = 0; + prev_vec_sm = 0.0; + + /* Load first ghost */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + *v(r_sm, (nc - 1) - (c_gl + i), f_sm); + } + } + if (r_sm == 0 && c_sm < c_ghost) { + am_sm[c_sm] = *am(nc - (c_gl + c_sm)); + bm_sm[c_sm] = *bm(nc - (c_gl + c_sm)); + } + c_rest -= c_ghost; + } + + MGARDX_EXEC bool LoopCondition2() { return c_rest > C - c_ghost; } + + MGARDX_EXEC void Operation11() { + c_main = Math::Min(C, c_rest); + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_main; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = + *v(r_sm, (nc - 1) - (c_gl + i + c_ghost), f_sm); + } + } + if (r_sm == 0 && c_sm < c_main) { + am_sm[c_sm + c_ghost] = *am(nc - (c_gl + c_sm + c_ghost)); + bm_sm[c_sm + c_ghost] = *bm(nc - (c_gl + c_sm + c_ghost)); + } + } + + MGARDX_EXEC void Operation12() { + /* Computation of v in parallel*/ + if (r_sm < r_rest && f_sm < f_rest) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)]); + + for (SIZE i = 1; i < C; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_backward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); + } + + /* Store last v */ + prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, r_sm, C - 1, f_sm)]; + } + } + + MGARDX_EXEC void Operation13() { + /* flush results to v */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < C; i++) { + *v(r_sm, (nc - 1) - (c_gl + i), f_sm) = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + } + } + } + + MGARDX_EXEC void Operation14() { + /* Update unloaded col */ + c_rest -= c_main; + + /* Advance c */ + c_gl += C; + + /* Copy next ghost to main */ + c_ghost = Math::Min(G, c_main - (C - G)); + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + C, f_sm)]; + } + } + if (r_sm == 0 && c_sm < c_ghost) { + am_sm[c_sm] = am_sm[c_sm + C]; + bm_sm[c_sm] = bm_sm[c_sm + C]; + } + } + + MGARDX_EXEC void Operation15() { + // Load all rest col + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i + c_ghost, f_sm)] = + *v(r_sm, (nc - 1) - (c_gl + i + c_ghost), f_sm); + } + } + if (r_sm == 0 && c_sm < c_rest) { + am_sm[c_sm + c_ghost] = *am(nc - (c_gl + c_sm + c_ghost)); + bm_sm[c_sm + c_ghost] = *bm(nc - (c_gl + c_sm + c_ghost)); + } + } + + MGARDX_EXEC void Operation16() { + /* Only 1 col remain */ + if (c_ghost + c_rest == 1) { + if (r_sm < r_rest && f_sm < f_rest) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)]); + } + //__syncthreads(); + + } else { + if (r_sm < r_rest && f_sm < f_rest) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, r_sm, 0, c_sm)]); + for (SIZE i = 1; i < c_ghost + c_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)] = tridiag_backward2( + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i - 1, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]); + } + } + } + } + + MGARDX_EXEC void Operation17() { + /* flush results to v */ + if (r_sm < r_rest && f_sm < f_rest) { + for (SIZE i = 0; i < c_ghost + c_rest; i++) { + *v(r_sm, (nc - 1) - (c_gl + i), f_sm) = + vec_sm[get_idx(ldsm1, ldsm2, r_sm, i, f_sm)]; + } + } + v.reset_offset(); + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + size = (R * F + 2) * (C + G) * sizeof(T); + return size; + } + +private: + // functor parameters + SIZE nr, nc, nf; + SubArray<1, T, DeviceType> am, bm; + SubArray<1, T, DeviceType> dist_c; + SubArray v; + + // thread local variables + SIZE c_gl, r_gl, f_gl; + SIZE c_sm, r_sm, f_sm; + SIZE ldsm1, ldsm2; + T *vec_sm; + T *am_sm; + T *bm_sm; + T prev_vec_sm; + SIZE f_rest, r_rest; + SIZE c_rest, c_ghost, c_main; +}; + +template +class Ipk2Reo3DKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "ipk2_3d"; + constexpr static SIZE G = 2; + MGARDX_CONT + Ipk2Reo3DKernel(SIZE nr, SIZE nc, SIZE nf, SubArray<1, T, DeviceType> am, + SubArray<1, T, DeviceType> bm, + SubArray<1, T, DeviceType> dist_c, + SubArray v) + : nr(nr), nc(nc), nf(nf), am(am), bm(bm), dist_c(dist_c), v(v) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Ipk2Reo3DFunctor; + FunctorType functor(nr, nc, nf, am, bm, dist_c, v); + + SIZE total_thread_x = nf; + SIZE total_thread_y = nr; + SIZE total_thread_z = 1; + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbx = F; + tby = R; + tbz = 1; + gridx = ceil((double)total_thread_x / tbx); + gridy = ceil((double)total_thread_y / tby); + gridz = 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + // functor parameters + SIZE nr, nc, nf; + SubArray<1, T, DeviceType> am, bm; + SubArray<1, T, DeviceType> dist_c; + SubArray v; +}; + +template +class Ipk3Reo3DFunctor : public IterFunctor { +public: + MGARDX_CONT Ipk3Reo3DFunctor() {} + MGARDX_CONT Ipk3Reo3DFunctor(SIZE nr, SIZE nc, SIZE nf, + SubArray<1, T, DeviceType> am, + SubArray<1, T, DeviceType> bm, + SubArray<1, T, DeviceType> dist_r, + SubArray v) + : nr(nr), nc(nc), nf(nf), am(am), bm(bm), dist_r(dist_r), v(v) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + f_gl = FunctorBase::GetBlockIdX() * F; + c_gl = FunctorBase::GetBlockIdY() * C; + r_gl = 0; + + f_sm = FunctorBase::GetThreadIdX(); + c_sm = FunctorBase::GetThreadIdY(); + r_sm = FunctorBase::GetThreadIdX(); + + v.offset_3d(0, c_gl, f_gl); + T *sm = (T *)FunctorBase::GetSharedMemory(); + ldsm1 = F; + ldsm2 = C; + vec_sm = sm; + am_sm = sm + (R + G) * ldsm1 * ldsm2; + bm_sm = am_sm + (R + G); + + prev_vec_sm = 0.0; + + f_rest = Math::Min( + F, nf - FunctorBase::GetBlockIdX() * F); + c_rest = Math::Min( + C, nc - FunctorBase::GetBlockIdY() * C); + + r_rest = nr; + r_ghost = Math::Min(nr, G); + r_main = R; + + /* Load first ghost */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = *v(r_gl + i, c_sm, f_sm); + } + } + + if (c_sm == 0 && r_sm < r_ghost) { + am_sm[r_sm] = *am(r_gl + r_sm); + bm_sm[r_sm] = *bm(r_gl + r_sm); + } + r_rest -= r_ghost; + } + + MGARDX_EXEC void Operation2() {} + + MGARDX_EXEC bool LoopCondition1() { return r_rest > R - r_ghost; } + + MGARDX_EXEC void Operation3() { + r_main = Math::Min(R, r_rest); + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_main; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = + *v(r_gl + i + r_ghost, c_sm, f_sm); + } + } + if (c_sm == 0 && r_sm < r_main) { + am_sm[r_sm + r_ghost] = *am(r_gl + r_sm + r_ghost); + bm_sm[r_sm + r_ghost] = *bm(r_gl + r_sm + r_ghost); + } + } + + MGARDX_EXEC void Operation4() { + /* Computation of v in parallel*/ + if (c_sm < c_rest && f_sm < f_rest) { + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); + for (SIZE i = 1; i < R; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_forward2( + vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); + } + + /* Store last v */ + prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, R - 1, c_sm, f_sm)]; + } + } + + MGARDX_EXEC void Operation5() { + /* flush results to v */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < R; i++) { + *v(r_gl + i, c_sm, f_sm) = vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; + } + } + } + + MGARDX_EXEC void Operation6() { + // /* Update unloaded col */ + r_rest -= r_main; + + /* Advance c */ + r_gl += R; + + /* Copy next ghost to main */ + r_ghost = Math::Min(G, r_main - (R - G)); + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = + vec_sm[get_idx(ldsm1, ldsm2, i + R, c_sm, f_sm)]; + } + } + if (c_sm == 0 && r_sm < r_ghost) { + am_sm[r_sm] = am_sm[r_sm + R]; + bm_sm[r_sm] = bm_sm[r_sm + R]; + } + } + + MGARDX_EXEC void Operation7() { + /* Load all rest col */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = + *v(r_gl + i + r_ghost, c_sm, f_sm); + } + } + + if (c_sm == 0 && r_sm < r_rest) { + am_sm[r_sm + r_ghost] = *am(r_gl + r_sm + r_ghost); + bm_sm[r_sm + r_ghost] = *bm(r_gl + r_sm + r_ghost); + } + } + + MGARDX_EXEC void Operation8() { + /* Only 1 col remain */ + if (r_ghost + r_rest == 1) { + if (c_sm < c_rest && f_sm < f_rest) { + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); + } + //__syncthreads(); + + } else { + if (c_sm < c_rest && f_sm < f_rest) { + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + tridiag_forward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); + for (SIZE i = 1; i < r_ghost + r_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_forward2( + vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); + } + } + } + } + + MGARDX_EXEC void Operation9() { + /* flush results to v */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_ghost + r_rest; i++) { + *v(r_gl + i, c_sm, f_sm) = vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; + } + } + } + + MGARDX_EXEC void Operation10() { + /* backward */ + r_rest = nr; + r_ghost = Math::Min(nr, G); + r_main = R; + r_gl = 0; + prev_vec_sm = 0.0; + + /* Load first ghost */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = + *v((nr - 1) - (r_gl + i), c_sm, f_sm); + } + } + + if (c_sm == 0 && r_sm < r_ghost) { + am_sm[r_sm] = *am(nr - (r_gl + r_sm)); + bm_sm[r_sm] = *bm(nr - (r_gl + r_sm)); + } + r_rest -= r_ghost; + } + + MGARDX_EXEC bool LoopCondition2() { return r_rest > R - r_ghost; } + + MGARDX_EXEC void Operation11() { + r_main = Math::Min(R, r_rest); + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_main; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = + *v((nr - 1) - (r_gl + i + r_ghost), c_sm, f_sm); + } + } + if (c_sm == 0 && r_sm < r_main) { + am_sm[r_sm + r_ghost] = *am(nr - (r_gl + r_sm + r_ghost)); + bm_sm[r_sm + r_ghost] = *bm(nr - (r_gl + r_sm + r_ghost)); + } + } + + MGARDX_EXEC void Operation12() { + /* Computation of v in parallel*/ + if (c_sm < c_rest && f_sm < f_rest) { + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); + for (SIZE i = 1; i < R; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_backward2( + vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); + } + + /* Store last v */ + prev_vec_sm = vec_sm[get_idx(ldsm1, ldsm2, R - 1, c_sm, f_sm)]; + } + } + + MGARDX_EXEC void Operation13() { + /* flush results to v */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < R; i++) { + *v((nr - 1) - (r_gl + i), c_sm, f_sm) = + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; + } + } + } + + MGARDX_EXEC void Operation14() { + // /* Update unloaded col */ + r_rest -= r_main; + + /* Advance c */ + r_gl += R; + + /* Copy next ghost to main */ + r_ghost = Math::Min(G, r_main - (R - G)); + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_ghost; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = + vec_sm[get_idx(ldsm1, ldsm2, i + R, c_sm, f_sm)]; + } + } + if (c_sm == 0 && r_sm < r_ghost) { + am_sm[r_sm] = am_sm[r_sm + R]; + bm_sm[r_sm] = bm_sm[r_sm + R]; + } + } + + MGARDX_EXEC void Operation15() { + /* Load all rest col */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i + r_ghost, c_sm, f_sm)] = + *v((nr - 1) - (r_gl + i + r_ghost), c_sm, f_sm); + } + } + if (c_sm == 0 && r_sm < r_rest) { + am_sm[r_sm + r_ghost] = *am(nr - (r_gl + r_sm + r_ghost)); + bm_sm[r_sm + r_ghost] = *bm(nr - (r_gl + r_sm + r_ghost)); + } + } + + MGARDX_EXEC void Operation16() { + /* Only 1 col remain */ + if (r_ghost + r_rest == 1) { + if (c_sm < c_rest && f_sm < f_rest) { + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); + } + //__syncthreads(); + + } else { + if (c_sm < c_rest && f_sm < f_rest) { + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + tridiag_backward2(prev_vec_sm, am_sm[0], bm_sm[0], + vec_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)]); + for (SIZE i = 1; i < r_ghost + r_rest; i++) { + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)] = tridiag_backward2( + vec_sm[get_idx(ldsm1, ldsm2, i - 1, c_sm, f_sm)], am_sm[i], + bm_sm[i], vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]); + } + } + } + } + + MGARDX_EXEC void Operation17() { + /* flush results to v */ + if (c_sm < c_rest && f_sm < f_rest) { + for (SIZE i = 0; i < r_ghost + r_rest; i++) { + *v((nr - 1) - (r_gl + i), c_sm, f_sm) = + vec_sm[get_idx(ldsm1, ldsm2, i, c_sm, f_sm)]; + } + } + v.reset_offset(); + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + size = (C * F + 2) * (R + G) * sizeof(T); + return size; + } + +private: + // functor parameters + SIZE nr, nc, nf; + SubArray<1, T, DeviceType> am, bm; + SubArray<1, T, DeviceType> dist_r; + SubArray v; + + // thread local variables + SIZE c_gl, r_gl, f_gl; + SIZE c_sm, r_sm, f_sm; + SIZE ldsm1, ldsm2; + T *vec_sm; + T *am_sm; + T *bm_sm; + T prev_vec_sm; + SIZE f_rest, c_rest; + SIZE r_rest, r_ghost, r_main; +}; + +template +class Ipk3Reo3DKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "ipk3_3d"; + constexpr static SIZE G = 2; + MGARDX_CONT + Ipk3Reo3DKernel(SIZE nr, SIZE nc, SIZE nf, SubArray<1, T, DeviceType> am, + SubArray<1, T, DeviceType> bm, + SubArray<1, T, DeviceType> dist_r, + SubArray v) + : nr(nr), nc(nc), nf(nf), am(am), bm(bm), dist_r(dist_r), v(v) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Ipk3Reo3DFunctor; + FunctorType functor(nr, nc, nf, am, bm, dist_r, v); + + SIZE total_thread_x = nf; + SIZE total_thread_y = nc; + SIZE total_thread_z = 1; + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbx = F; + tby = C; + tbz = 1; + gridx = ceil((double)total_thread_x / tbx); + gridy = ceil((double)total_thread_y / tby); + gridz = 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE nr, nc, nf; + SubArray<1, T, DeviceType> am, bm; + SubArray<1, T, DeviceType> dist_r; + SubArray v; +}; + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/LPKFunctor.h b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/LPKFunctor.h new file mode 100644 index 000000000..8e7c55be5 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/LPKFunctor.h @@ -0,0 +1,71 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_LPK_FUNCTOR +#define MGARD_X_160_LPK_FUNCTOR + +namespace mgard_x_160 { + +template +MGARDX_EXEC T mass_trans(T a, T b, T c, T d, T e, T h1, T h2, T h3, T h4, T r1, + T r2, T r3, T r4) { + T tb, tc, td, tb1, tb2, tc1, tc2, td1, td2; +#ifdef MGARD_X_160_FMA + if (sizeof(T) == sizeof(double)) { + tb1 = fma(c, h2 / 6, a * h1 / 6); + tb2 = fma(b, h2 / 6, b * h1 / 6); + + tc1 = fma(d, h3 / 6, b * h2 / 6); + tc2 = fma(c, h3 / 6, c * h2 / 6); + + td1 = fma(c, h4 / 6, e * h3 / 6); + td2 = fma(d, h4 / 6, d * h3 / 6); + + tb = fma(2, tb2, tb1); + tc = fma(2, tc2, tc1); + td = fma(2, td2, td1); + return fma(td, r4, fma(tb, r1, tc)); + } else if (sizeof(T) == sizeof(float)) { + tb1 = fmaf(c, h2 / 6, a * h1 / 6); + tb2 = fmaf(b, h2 / 6, b * h1 / 6); + + tc1 = fmaf(d, h3 / 6, b * h2 / 6); + tc2 = fmaf(c, h3 / 6, c * h2 / 6); + + td1 = fmaf(c, h4 / 6, e * h3 / 6); + td2 = fmaf(d, h4 / 6, d * h3 / 6); + + tb = fmaf(2, tb2, tb1); + tc = fmaf(2, tc2, tc1); + td = fmaf(2, td2, td1); + return fmaf(td, r4, fmaf(tb, r1, tc)); + } +#else + + if (h1 + h2 != 0) { + r1 = h1 / (h1 + h2); + } else { + r1 = 0.0; + } + if (h3 + h4 != 0) { + r4 = h4 / (h3 + h4); + } else { + r4 = 0.0; + } + + // printf("%f %f %f %f %f (%f %f %f %f)\n", a, b, c, d, e, h1, h2, h3, h4); + tb = a * (h1 / 6) + b * ((h1 + h2) / 3) + c * (h2 / 6); + tc = b * (h2 / 6) + c * ((h2 + h3) / 3) + d * (h3 / 6); + td = c * (h3 / 6) + d * ((h3 + h4) / 3) + e * (h4 / 6); + tc += tb * r1 + td * r4; + return tc; +#endif +} + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel.hpp new file mode 100644 index 000000000..695f408ed --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel.hpp @@ -0,0 +1,1513 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_LINEAR_PROCESSING_KERNEL_TEMPLATE +#define MGARD_X_160_LINEAR_PROCESSING_KERNEL_TEMPLATE + +#include "../../../RuntimeX/RuntimeX.h" +#include "LPKFunctor.h" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +class Lpk1ReoFunctor : public Functor { +public: + MGARDX_CONT Lpk1ReoFunctor() {} + MGARDX_CONT Lpk1ReoFunctor( + SubArray<1, SIZE, DeviceType> shape, + SubArray<1, SIZE, DeviceType> shape_c, DIM processed_n, + SubArray<1, DIM, DeviceType> processed_dims, DIM curr_dim_r, + DIM curr_dim_c, DIM curr_dim_f, SubArray<1, T, DeviceType> dist_f, + SubArray<1, T, DeviceType> ratio_f, SubArray v1, + SubArray v2, SubArray w) + : shape(shape), shape_c(shape_c), processed_n(processed_n), + processed_dims(processed_dims), curr_dim_r(curr_dim_r), + curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), dist_f(dist_f), + ratio_f(ratio_f), v1(v1), v2(v2), w(w) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + threadId = (FunctorBase::GetThreadIdZ() * + (FunctorBase::GetBlockDimX() * + FunctorBase::GetBlockDimY())) + + (FunctorBase::GetThreadIdY() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + + Byte *sm = FunctorBase::GetSharedMemory(); + SIZE offset = 0; + ldsm1 = F * 2 + 3; + ldsm2 = C; + v_sm = (T *)&sm[offset]; + advance_with_align(offset, ldsm1 * ldsm2 * R); + + dist_f_sm = (T *)&sm[offset]; + advance_with_align(offset, ldsm1); + ratio_f_sm = (T *)&sm[offset]; + advance_with_align(offset, ldsm1); + + // switching data type + align_byte_offset(offset); + + shape_sm = (SIZE *)&sm[offset]; + advance_with_align(offset, D); + shape_c_sm = (SIZE *)&sm[offset]; + advance_with_align(offset, D); + + // switching data type + align_byte_offset(offset); + + processed_dims_sm = (DIM *)&sm[offset]; + advance_with_align(offset, D); + + if (threadId < D) { + shape_sm[threadId] = *shape(threadId); + shape_c_sm[threadId] = *shape_c(threadId); + } + if (threadId < processed_n) { + processed_dims_sm[threadId] = *processed_dims(threadId); + } + } + + MGARDX_EXEC void Operation2() { + // __syncthreads(); + + for (DIM d = 0; d < D; d++) + idx[d] = 0; + + nr = shape_sm[curr_dim_r]; + nc = shape_sm[curr_dim_c]; + nf = shape_sm[curr_dim_f]; + nf_c = shape_c_sm[curr_dim_f]; + + zero_other = true; + PADDING = (nf % 2 == 0); + + bidx = FunctorBase::GetBlockIdX(); + if (nf_c % 2 == 1) { + firstD = div_roundup(nf_c, FunctorBase::GetBlockDimX()); + } else { + firstD = div_roundup(nf_c, FunctorBase::GetBlockDimX()); + } + blockId = bidx % firstD; + bidx /= firstD; + + for (DIM d = 0; d < D; d++) { + if (d != curr_dim_r && d != curr_dim_c && d != curr_dim_f) { + SIZE t = shape_sm[d]; + for (DIM k = 0; k < processed_n; k++) { + if (d == *processed_dims(k)) { + t = shape_c_sm[d]; + } + } + idx[d] = bidx % t; + bidx /= t; + if (idx[d] >= shape_c_sm[d]) + zero_other = false; + } + } + + zero_r = shape_c_sm[curr_dim_r]; + zero_c = shape_c_sm[curr_dim_c]; + zero_f = shape_c_sm[curr_dim_f]; + + if (D < 3) { + nr = 1; + zero_r = 1; + } + if (D < 2) { + nc = 1; + zero_c = 1; + } + + v1.offset(idx); + v2.offset(idx); + w.offset(idx); + + // if (debug2) { + // printf("idx: %d %d %d %d\n", idx[3], idx[2], idx[1], idx[0]); + // printf("other_offset_v: %llu\n", other_offset_v); + // printf("other_offset_w: %llu\n", other_offset_w); + // } + r_gl = FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ() + + FunctorBase::GetThreadIdZ(); + c_gl = FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY() + + FunctorBase::GetThreadIdY(); + f_gl = blockId * FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + r_sm = FunctorBase::GetThreadIdZ(); + c_sm = FunctorBase::GetThreadIdY(); + f_sm = FunctorBase::GetThreadIdX(); + + actual_F = F; + if (nf_c - blockId * FunctorBase::GetBlockDimX() < F) { + actual_F = nf_c - blockId * FunctorBase::GetBlockDimX(); + } + + // if (nf_c % 2 == 1){ + // if(nf_c-1 - blockId * FunctorBase::GetBlockDimX() < F) { + // actual_F = nf_c - 1 - blockId * + // FunctorBase::GetBlockDimX(); } + // } else { + // if(nf_c - blockId * FunctorBase::GetBlockDimX() < F) { + // actual_F = nf_c - blockId * FunctorBase::GetBlockDimX(); } + // } + + // if (debug) printf("actual_F %d\n", actual_F); + + if (r_gl < nr && c_gl < nc && f_gl < nf_c) { + if (zero_other && r_gl < zero_r && c_gl < zero_c && f_gl < zero_f) { + // if (debug) printf("load left vsm[%d]: 0.0\n", f_sm * 2 + 2); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 2)] = 0.0; + } else { + // if (debug) printf("load left vsm[%d]<-dv1[%d, %d, %d]: %f\n", f_sm * + // 2 + // + 2, r_gl, c_gl, f_gl, *v1( r_gl, c_gl, f_gl)); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 2)] = + *v1(r_gl, c_gl, f_gl); + } + + if (f_sm == actual_F - 1) { + if (zero_other && r_gl < zero_r && c_gl < zero_c && f_gl < zero_f) { + // if (debug) printf("load left+1 vsm[%d]: 0.0\n", actual_F * 2 + 2); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 2)] = 0.0; + } else { + if (f_gl + 1 < nf_c) { + // if (debug) printf("load left+1 vsm[%d]: %f\n", actual_F * 2 + 2, + // *v1( r_gl, c_gl, f_gl + 1)); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 2)] = + *v1(r_gl, c_gl, f_gl + 1); + } else { + // if (debug) printf("load left+1 vsm[%d]: 0.0\n", actual_F * 2 + + // 2); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 2)] = 0.0; + } + } + } + + if (f_sm == 0) { + // left + if (zero_other && r_gl < zero_r && c_gl < zero_c && f_gl < zero_f) { + // coarse (-1) + // if (debug) printf("load left-1 vsm[0]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = 0.0; + } else { + if (f_gl >= 1) { + // other (-1) + // if (debug) printf("load left-1 vsm[0]: %f\n", dv1[get_idx(lddv11, + // lddv12, r_gl, c_gl, f_gl-1)]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = + *v1(r_gl, c_gl, f_gl - 1); + } else { + // other (-1) + // if (debug) printf("load left-1 vsm[0]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = 0.0; + } + } + } + + // right + if (!PADDING) { // other = nf_c - 1 + if (nf_c % 2 != 0) { + if (f_gl >= 1 && + f_gl < nf_c) { // shift for better memory access pattern + // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * + // 2 + // + 1, *v2( r_gl, c_gl, f_gl - 1), r_gl, + // c_gl, f_gl - 1); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = + *v2(r_gl, c_gl, f_gl - 1); + } else { + // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 1); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = 0.0; + } + } else { // nf_c % 2 == 0, do not shift + if (f_gl < nf_c - 1) { + // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * + // 2 + // + 3, *v2( r_gl, c_gl, f_gl), r_gl, c_gl, + // f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = + *v2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 3); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = 0.0; + } + } + } else { // PADDING other = nf_c - 2 + if (nf_c % 2 != 0) { + if (f_gl >= 1 && + f_gl < nf_c - 1) { // shift for better memory access pattern + // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * + // 2 + // + 1, *v2( r_gl, c_gl, f_gl - 1), r_gl, + // c_gl, f_gl - 1); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = + *v2(r_gl, c_gl, f_gl - 1); + } else { + // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 1); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = 0.0; + } + } else { // nf_c % 2 == 0 + if (f_gl < nf_c - 2) { // do not shift + // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * + // 2 + // + 3, *v2( r_gl, c_gl, f_gl), r_gl, c_gl, + // f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = + *v2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 3); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = 0.0; + } + } + } + + if (f_sm == actual_F - 1) { + // right (+1) + if (!PADDING) { + if (nf_c % 2 != 0) { + if (f_gl < nf_c - 1) { + // if (debug) printf("load right+1 vsm[%d]: %f <- %d %d %d\n", + // actual_F * 2 + 1, *v2( r_gl, c_gl, f_gl), + // r_gl, c_gl, f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = + *v2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load right+1 vsm[%d]: 0.0\n", actual_F * 2 + + // 1); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = 0.0; + } + } else { // nf_c % 2 == 0 + if (f_gl >= actual_F) { + // if (debug) printf("load right-1 vsm[1]: %f <- %d %d %d\n", + // *v2( r_gl, c_gl, f_gl - actual_F), r_gl, + // c_gl, f_gl - actual_F); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = + *v2(r_gl, c_gl, f_gl - actual_F); + } else { + // if (debug) printf("load right-1 vsm[1]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = 0.0; + } + } + } else { + if (nf_c % 2 != 0) { + if (f_gl < nf_c - 2) { + // if (debug) printf("actual_F(%d), load right+1 vsm[%d]: %f <- %d + // %d %d\n", actual_F, actual_F * 2 + 1, *v2( r_gl, c_gl, f_gl), + // r_gl, c_gl, f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = + *v2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load right+1 vsm[%d]: 0.0\n", actual_F * 2 + + // 1); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = 0.0; + } + } else { // nf_c % 2 == 0 + if (f_gl >= actual_F && f_gl - actual_F < nf_c - 2) { + // if (debug) printf("load right-1 vsm[1]: %f <- %d %d %d\n", + // *v2( r_gl, c_gl, f_gl - actual_F), r_gl, + // c_gl, f_gl - actual_F); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = + *v2(r_gl, c_gl, f_gl - actual_F); + } else { + // if (debug) printf("load right-1 vsm[1]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = 0.0; + } + } + } + } + } + + // if (debug) printf("actual_F: %d\n", actual_F); + if (r_sm == 0 && c_sm == 0 && f_sm < actual_F) { + // if (debug) printf("blockId * F * 2 + f_sm = %d\n", blockId * F * 2 + + // f_sm); + if (blockId * F * 2 + f_sm < nf) { // padding: num of dist == nf, + // non-padding: non of dist == nf - 1 + // if (debug) printf("load dist/ratio1[%d]: %f <- %d\n", 2 + f_sm, + // *dist_f(blockId * F * 2 + f_sm), blockId * F * 2 + f_sm); + dist_f_sm[2 + f_sm] = *dist_f(blockId * F * 2 + f_sm); + ratio_f_sm[2 + f_sm] = *ratio_f(blockId * F * 2 + f_sm); + } else { + // if (debug) printf("load dist/ratio1[%d]: 0.0\n", 2 + f_sm); + dist_f_sm[2 + f_sm] = 0.0; + ratio_f_sm[2 + f_sm] = 0.0; + } + + if (blockId * F * 2 + actual_F + f_sm < nf) { + // if (debug) printf("load dist/ratio2[%d]: %f <- %d\n", 2 + actual_F + + // f_sm, *dist_f(blockId * F * 2 + actual_F + f_sm), blockId * F * 2 + + // actual_F + f_sm); + dist_f_sm[2 + actual_F + f_sm] = + *dist_f(blockId * F * 2 + actual_F + f_sm); + ratio_f_sm[2 + actual_F + f_sm] = + *ratio_f(blockId * F * 2 + actual_F + f_sm); + } else { + // if (debug) printf("load dist/ratio2[%d]: 0.0\n", 2 + actual_F + + // f_sm); + dist_f_sm[2 + actual_F + f_sm] = 0.0; + ratio_f_sm[2 + actual_F + f_sm] = 0.0; + } + } + + if (blockId > 0) { + if (f_sm < 2) { + // dist_f_sm[f_sm] = *dist_f(f_gl - 2); + // ratio_f_sm[f_sm] = *ratio_f(f_gl - 2); + // if (debug) printf("load dist/ratio-1[%d]: %f <- %d\n", f_sm, + // *dist_f(blockId * F * 2 + f_sm - 2), blockId * F * 2 + f_sm - 2); + dist_f_sm[f_sm] = *dist_f(blockId * F * 2 + f_sm - 2); + ratio_f_sm[f_sm] = *ratio_f(blockId * F * 2 + f_sm - 2); + } + } else { + if (f_sm < 2) { + // if (debug) printf("load dist/ratio-1[%d]: 0.0 <- %d\n", f_sm); + dist_f_sm[f_sm] = 0.0; + ratio_f_sm[f_sm] = 0.0; + } + } + } + + MGARDX_EXEC void Operation3() { + + // __syncthreads(); + + if (r_gl < nr && c_gl < nc && f_gl < nf_c) { + T h1 = dist_f_sm[f_sm * 2]; + T h2 = dist_f_sm[f_sm * 2 + 1]; + T h3 = dist_f_sm[f_sm * 2 + 2]; + T h4 = dist_f_sm[f_sm * 2 + 3]; + T r1 = ratio_f_sm[f_sm * 2]; + T r2 = ratio_f_sm[f_sm * 2 + 1]; + T r3 = ratio_f_sm[f_sm * 2 + 2]; + T r4 = 1 - r3; + T a = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2)]; + T b = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)]; + T c = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 2)]; + T d = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)]; + T e = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 4)]; + + // bool debug = false; + // if (idx[3] == 0) debug = false; + // if (debug) { + // printf("f_sm(%d) %f %f %f %f %f f_sm_h %f %f %f %f f_sm_r %f %f %f + // %f, out: %f\n",f_sm, a,b,c,d,e, h1,h2,h3,h4,r1,r2,r3,r4, + // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4)); + // } + + // T tb = a * h1/6 + b * (h1+h2)/3 + c * h2/6; + // T tc = b * h2/6 + c * (h2+h3)/3 + d * h3/6; + // T td = c * h3/6 + d * (h3+h4)/3 + e * h4/6; + + // if (debug) printf("f_sm(%d) tb tc td tc: %f %f %f %f\n", f_sm, tb, tc, + // td, tc+tb * r1 + td * r4); + + // tc += tb * r1 + td * r4; + + *w(r_gl, c_gl, f_gl) = + mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4); + + // if (debug) printf("store[%d %d %d] %f \n", r_gl, c_gl, f_gl, + // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4)); + + // printf("test block %d F %d nf %d\n", blockId, F, nf); + // if (f_gl+1 == nf_c-1) { + + // // T te = h4 * d + 2 * h4 * e; + // //printf("f_sm(%d) mm-e: %f\n", f_sm, te); + // // te += td * r3; + // *w( r_gl, c_gl, f_gl+1) = + // mass_trans(c, d, e, (T)0.0, (T)0.0, h1, h2, (T)0.0, (T)0.0, r1, + // r2, (T)0.0, (T)0.0); + // } + } + + v1.reset_offset(); + v2.reset_offset(); + w.reset_offset(); + } + + MGARDX_CONT SIZE shared_memory_size() { + SIZE size = 0; + size = (R * C * (F * 2 + 3) + (F * 2 + 3) * 2) * sizeof(T); + align_byte_offset(size); + size += (D * 4) * sizeof(SIZE); + align_byte_offset(size); + size += (D * 1) * sizeof(DIM); + return size; + } + +private: + // functor parameters + SubArray<1, SIZE, DeviceType> shape; + SubArray<1, SIZE, DeviceType> shape_c; + DIM processed_n; + SubArray<1, DIM, DeviceType> processed_dims; + DIM curr_dim_r, curr_dim_c, curr_dim_f; + SubArray<1, T, DeviceType> dist_f; + SubArray<1, T, DeviceType> ratio_f; + SubArray v1, v2, w; + + // thread local variables + + SIZE threadId; + + T *v_sm; + SIZE ldsm1, ldsm2; + T *dist_f_sm; + T *ratio_f_sm; + + SIZE *shape_sm; + SIZE *shape_c_sm; + DIM *processed_dims_sm; + + SIZE idx[D]; + SIZE nr, nc, nf, nf_c; + bool zero_other; + bool PADDING; + + SIZE bidx; + SIZE firstD; + SIZE blockId; + + SIZE zero_r; + SIZE zero_c; + SIZE zero_f; + + SIZE r_gl; + SIZE c_gl; + SIZE f_gl; + + SIZE r_sm; + SIZE c_sm; + SIZE f_sm; + + SIZE actual_F; +}; + +template +class Lpk1ReoKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "lpk1_nd"; + MGARDX_CONT + Lpk1ReoKernel(SubArray<1, SIZE, DeviceType> shape, + SubArray<1, SIZE, DeviceType> shape_c, DIM processed_n, + SubArray<1, DIM, DeviceType> processed_dims, DIM curr_dim_r, + DIM curr_dim_c, DIM curr_dim_f, + SubArray<1, T, DeviceType> dist_f, + SubArray<1, T, DeviceType> ratio_f, + SubArray v1, SubArray v2, + SubArray w) + : shape(shape), shape_c(shape_c), processed_n(processed_n), + processed_dims(processed_dims), curr_dim_r(curr_dim_r), + curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), dist_f(dist_f), + ratio_f(ratio_f), v1(v1), v2(v2), w(w) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Lpk1ReoFunctor; + FunctorType functor(shape, shape_c, processed_n, processed_dims, curr_dim_r, + curr_dim_c, curr_dim_f, dist_f, ratio_f, v1, v2, w); + + SIZE nr = shape.dataHost()[curr_dim_r]; + SIZE nc = shape.dataHost()[curr_dim_c]; + SIZE nf = shape.dataHost()[curr_dim_f]; + SIZE nf_c = shape_c.dataHost()[curr_dim_f]; + + SIZE total_thread_z = nr; + SIZE total_thread_y = nc; + SIZE total_thread_x = nf_c; + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = R; + tby = C; + tbx = F; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + + for (DIM d = 0; d < D; d++) { + if (d != curr_dim_f && d != curr_dim_c && d != curr_dim_r) { + SIZE t = shape.dataHost()[d]; + for (DIM k = 0; k < processed_n; k++) { + if (d == processed_dims.dataHost()[k]) { + t = shape_c.dataHost()[d]; + } + } + gridx *= t; + } + } + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, SIZE, DeviceType> shape; + SubArray<1, SIZE, DeviceType> shape_c; + DIM processed_n; + SubArray<1, DIM, DeviceType> processed_dims; + DIM curr_dim_r, curr_dim_c, curr_dim_f; + SubArray<1, T, DeviceType> dist_f; + SubArray<1, T, DeviceType> ratio_f; + SubArray v1, v2, w; +}; + +template +class Lpk2ReoFunctor : public Functor { +public: + MGARDX_CONT Lpk2ReoFunctor() {} + MGARDX_CONT Lpk2ReoFunctor( + SubArray<1, SIZE, DeviceType> shape, + SubArray<1, SIZE, DeviceType> shape_c, DIM processed_n, + SubArray<1, DIM, DeviceType> processed_dims, DIM curr_dim_r, + DIM curr_dim_c, DIM curr_dim_f, SubArray<1, T, DeviceType> dist_c, + SubArray<1, T, DeviceType> ratio_c, SubArray v1, + SubArray v2, SubArray w) + : shape(shape), shape_c(shape_c), processed_n(processed_n), + processed_dims(processed_dims), curr_dim_r(curr_dim_r), + curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), dist_c(dist_c), + ratio_c(ratio_c), v1(v1), v2(v2), w(w) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + threadId = (FunctorBase::GetThreadIdZ() * + (FunctorBase::GetBlockDimX() * + FunctorBase::GetBlockDimY())) + + (FunctorBase::GetThreadIdY() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + + Byte *sm = FunctorBase::GetSharedMemory(); + SIZE offset = 0; + ldsm1 = F; + ldsm2 = C * 2 + 3; + v_sm = (T *)&sm[offset]; + advance_with_align(offset, ldsm1 * ldsm2 * R); + + dist_c_sm = (T *)&sm[offset]; + advance_with_align(offset, ldsm2); + ratio_c_sm = (T *)&sm[offset]; + advance_with_align(offset, ldsm2); + + // switching data type + align_byte_offset(offset); + + shape_sm = (SIZE *)&sm[offset]; + ; + advance_with_align(offset, D); + shape_c_sm = (SIZE *)&sm[offset]; + ; + advance_with_align(offset, D); + + // switching data type + align_byte_offset(offset); + + processed_dims_sm = (DIM *)&sm[offset]; + advance_with_align(offset, D); + + if (threadId < D) { + shape_sm[threadId] = *shape(threadId); + shape_c_sm[threadId] = *shape_c(threadId); + } + if (threadId < processed_n) { + processed_dims_sm[threadId] = *processed_dims(threadId); + } + } + + MGARDX_EXEC void Operation2() { + // __syncthreads(); + + for (DIM d = 0; d < D; d++) + idx[d] = 0; + + nr = shape_sm[curr_dim_r]; + nc = shape_sm[curr_dim_c]; + nf_c = shape_c_sm[curr_dim_f]; + nc_c = shape_c_sm[curr_dim_c]; + PADDING = (nc % 2 == 0); + + if (D < 3) { + nr = 1; + } + + bidx = FunctorBase::GetBlockIdX(); + firstD = div_roundup(nf_c, FunctorBase::GetBlockDimX()); + blockId_f = bidx % firstD; + bidx /= firstD; + + for (DIM d = 0; d < D; d++) { + if (d != curr_dim_r && d != curr_dim_c && d != curr_dim_f) { + SIZE t = shape_sm[d]; + for (DIM k = 0; k < processed_n; k++) { + if (d == processed_dims_sm[k]) { + t = shape_c_sm[d]; + } + } + idx[d] = bidx % t; + bidx /= t; + } + } + + v1.offset(idx); + v2.offset(idx); + w.offset(idx); + + // SIZE other_offset_v = get_idx(ldvs_sm, idx); + // SIZE other_offset_w = get_idx(ldws_sm, idx); + + // dv1 = dv1 + other_offset_v; + // dv2 = dv2 + other_offset_v; + // dw = dw + other_offset_w; + + // if (debug2) { + // printf("idx: %d %d %d %d\n", idx[3], idx[2], idx[1], idx[0]); + // printf("other_offset_v: %llu\n", other_offset_v); + // printf("other_offset_w: %llu\n", other_offset_w); + // } + + r_gl = FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ() + + FunctorBase::GetThreadIdZ(); + c_gl = FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY() + + FunctorBase::GetThreadIdY(); + f_gl = blockId_f * FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + blockId = FunctorBase::GetBlockIdY(); + + r_sm = FunctorBase::GetThreadIdZ(); + c_sm = FunctorBase::GetThreadIdY(); + f_sm = FunctorBase::GetThreadIdX(); + + actual_C = C; + if (nc_c - FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY() < + C) { + actual_C = nc_c - FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY(); + } + + // if (nc_c % 2 == 1){ + // if(nc_c-1 - FunctorBase::GetBlockIdY() * + // FunctorBase::GetBlockDimY() < C) { actual_C = nc_c - 1 - + // FunctorBase::GetBlockIdY() * + // FunctorBase::GetBlockDimY(); } + // } else { + // if(nc_c - FunctorBase::GetBlockIdY() * + // FunctorBase::GetBlockDimY() < C) { actual_C = nc_c - + // FunctorBase::GetBlockIdY() * + // FunctorBase::GetBlockDimY(); } + // } + + // bool debug = false; + // if (idx[3] == 0 && r_gl == 0 ) debug = false; + + // if (debug) printf("actual_C %d\n", actual_C); + + if (r_gl < nr && c_gl < nc_c && f_gl < nf_c) { + // if (debug) printf("load up vsm[%d]: %f <- %d %d %d\n", c_sm * 2 + 2, + // *v1( r_gl, c_gl, f_gl), r_gl, c_gl, f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 2, f_sm)] = + *v1(r_gl, c_gl, f_gl); + + if (c_sm == actual_C - 1) { + if (c_gl + 1 < nc_c) { + // if (debug) printf("load up+1 vsm[%d]: %f <- %d %d %d\n", actual_C * + // 2 + // + 2, dv1[get_idx(lddv11, lddv12, r_gl, blockId * C + actual_C, + // f_gl)], r_gl, blockId * C + actual_C, f_gl); + // c_gl+1 == blockId * C + C + v_sm[get_idx(ldsm1, ldsm2, r_sm, actual_C * 2 + 2, f_sm)] = + *v1(r_gl, c_gl + 1, f_gl); + } else { + // if (debug) printf("load up+1 vsm[%d]: 0.0\n", actual_C * 2 + 2); + v_sm[get_idx(ldsm1, ldsm2, r_sm, actual_C * 2 + 2, f_sm)] = 0.0; + } + } + + if (c_sm == 0) { + if (c_gl >= 1) { + // if (debug) printf("load up-1 vsm[0]: %f <- %d %d %d\n", + // *v1( r_gl, c_gl-1, f_gl), r_gl, c_gl-1, + // f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = + *v1(r_gl, c_gl - 1, f_gl); + } else { + // if (debug) printf("load up-1 vsm[0]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = 0.0; + } + } + + if (!PADDING) { + if (c_gl < nc_c - 1) { + // if (debug) printf("load down vsm[%d]: %f <- %d %d %d\n", c_sm * 2 + + // 3, *v2( r_gl, c_gl, f_gl), r_gl, c_gl, f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = + *v2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load down vsm[%d]: 0.0\n", c_sm * 2 + 3); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = 0.0; + } + } else { + if (c_gl < nc_c - 2) { + // if (debug) printf("load down vsm[%d]: %f <- %d %d %d\n", c_sm * 2 + + // 3, *v2( r_gl, c_gl, f_gl), r_gl, c_gl, f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = + *v2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load down vsm[%d]: 0.0\n", c_sm * 2 + 3); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = 0.0; + } + } + + if (c_gl >= 1 && + (PADDING && c_gl - 1 < nc_c - 2 || !PADDING && c_gl - 1 < nc_c - 1)) { + if (c_sm == 0) { + // if (debug) printf("load down-1 vsm[1]: %f <- %d %d %d\n", + // *v2( r_gl, c_gl-1, f_gl), r_gl, c_gl-1, + // f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, 1, f_sm)] = + *v2(r_gl, c_gl - 1, f_gl); + } + } else { + if (c_sm == 0) { + // if (debug) printf("load down-1 vsm[1]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, r_sm, 1, f_sm)] = 0.0; + } + } + } + + // load dist/ratio using f_sm for better performance + // assumption F >= C + if (r_sm == 0 && c_sm == 0 && f_sm < actual_C) { + if (blockId * C * 2 + f_sm < nc) { + dist_c_sm[2 + f_sm] = *dist_c(blockId * C * 2 + f_sm); + ratio_c_sm[2 + f_sm] = *ratio_c(blockId * C * 2 + f_sm); + } else { + dist_c_sm[2 + f_sm] = 0.0; + ratio_c_sm[2 + f_sm] = 0.0; + } + + if (blockId * C * 2 + actual_C + f_sm < nc) { + dist_c_sm[2 + actual_C + f_sm] = + *dist_c(blockId * C * 2 + actual_C + f_sm); + ratio_c_sm[2 + actual_C + f_sm] = + *ratio_c(blockId * C * 2 + actual_C + f_sm); + } else { + dist_c_sm[2 + actual_C + f_sm] = 0.0; + ratio_c_sm[2 + actual_C + f_sm] = 0.0; + } + } + + if (blockId > 0) { + if (f_sm < 2) { + dist_c_sm[f_sm] = *dist_c(blockId * C * 2 - 2 + f_sm); + ratio_c_sm[f_sm] = *ratio_c(blockId * C * 2 - 2 + f_sm); + } + } else { + if (f_sm < 2) { + dist_c_sm[f_sm] = 0.0; + ratio_c_sm[f_sm] = 0.0; + } + } + } + + MGARDX_EXEC void Operation3() { + + // __syncthreads(); + + if (r_gl < nr && c_gl < nc_c && f_gl < nf_c) { + T h1 = dist_c_sm[c_sm * 2]; + T h2 = dist_c_sm[c_sm * 2 + 1]; + T h3 = dist_c_sm[c_sm * 2 + 2]; + T h4 = dist_c_sm[c_sm * 2 + 3]; + T r1 = ratio_c_sm[c_sm * 2]; + T r2 = ratio_c_sm[c_sm * 2 + 1]; + T r3 = ratio_c_sm[c_sm * 2 + 2]; + T r4 = 1 - r3; + T a = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2, f_sm)]; + T b = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 1, f_sm)]; + T c = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 2, f_sm)]; + T d = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)]; + T e = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 4, f_sm)]; + + // if (debug) { + // printf("c_sm(%d) %f %f %f %f %f\n",c_sm, a,b,c,d,e); + // printf("c_sm_h(%d) %f %f %f %f\n",c_sm, h1,h2,h3,h4); + // printf("c_sm_r(%d) %f %f %f %f\n",c_sm, r1,r2,r3,r4); + // } + + // T tb = a * h1 + b * 2 * (h1+h2) + c * h2; + // T tc = b * h2 + c * 2 * (h2+h3) + d * h3; + // T td = c * h3 + d * 2 * (h3+h4) + e * h4; + + // if (debug) printf("c_sm(%d) tb tc td tc: %f %f %f %f\n", f_sm, tb, tc, + // td, tc+tb * r1 + td * r4); + + // tc += tb * r1 + td * r4; + + // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) { + // printf("mr2(%d) mm2: %f -> (%d %d %d)\n", c_sm, tc, r_gl, c_gl, + // f_gl); + // // printf("f_sm(%d) b c d: %f %f %f\n", f_sm, tb, tc, td); + // } + + // if (debug) { + // printf("f_sm(%d) %f %f %f %f %f f_sm_h %f %f %f %f f_sm_r %f %f %f + // %f, out: %f\n",f_sm, a,b,c,d,e, h1,h2,h3,h4,r1,r2,r3,r4, + // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4)); + // } + + *w(r_gl, c_gl, f_gl) = + mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4); + + // if (debug) printf("store[%d %d %d] %f \n", r_gl, c_gl, f_gl, + // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4)); + + // printf("%d %d %d\n", r_gl, c_gl, f_gl); + // if (blockId * C + C == nc-1) { + // if (c_gl + 1 == nc_c - 1) { + // // T te = h4 * d + 2 * h4 * e; + // // te += td * r3; + // *w( r_gl, blockId * C + actual_C, f_gl) = + // mass_trans(c, d, e, (T)0.0, (T)0.0, + // h1, h2, (T)0.0, (T)0.0, r1, r2, (T)0.0, (T)0.0); + // } + // } + } + v1.reset_offset(); + v2.reset_offset(); + w.reset_offset(); + } + + MGARDX_CONT SIZE shared_memory_size() { + SIZE size = 0; + size = (R * (C * 2 + 3) * F + (C * 2 + 3) * 2) * sizeof(T); + align_byte_offset(size); + size += (D * 4) * sizeof(SIZE); + align_byte_offset(size); + size += (D * 1) * sizeof(DIM); + return size; + } + +private: + // functor parameters + SubArray<1, SIZE, DeviceType> shape; + SubArray<1, SIZE, DeviceType> shape_c; + DIM processed_n; + SubArray<1, DIM, DeviceType> processed_dims; + DIM curr_dim_r, curr_dim_c, curr_dim_f; + SubArray<1, T, DeviceType> dist_c; + SubArray<1, T, DeviceType> ratio_c; + SubArray v1, v2, w; + + // thread local variables + + SIZE threadId; + + T *v_sm; + SIZE ldsm1, ldsm2; + T *dist_c_sm; + T *ratio_c_sm; + + SIZE *shape_sm; + SIZE *shape_c_sm; + DIM *processed_dims_sm; + + SIZE idx[D]; + SIZE nr, nc, nf, nf_c, nc_c; + bool zero_other; + bool PADDING; + + SIZE bidx; + SIZE firstD; + SIZE blockId_f; + SIZE blockId; + + SIZE zero_r; + SIZE zero_c; + SIZE zero_f; + + SIZE r_gl; + SIZE c_gl; + SIZE f_gl; + + SIZE r_sm; + SIZE c_sm; + SIZE f_sm; + + SIZE actual_C; +}; + +template +class Lpk2ReoKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "lpk2_nd"; + MGARDX_CONT + Lpk2ReoKernel(SubArray<1, SIZE, DeviceType> shape, + SubArray<1, SIZE, DeviceType> shape_c, DIM processed_n, + SubArray<1, DIM, DeviceType> processed_dims, DIM curr_dim_r, + DIM curr_dim_c, DIM curr_dim_f, + SubArray<1, T, DeviceType> dist_c, + SubArray<1, T, DeviceType> ratio_c, + SubArray v1, SubArray v2, + SubArray w) + : shape(shape), shape_c(shape_c), processed_n(processed_n), + processed_dims(processed_dims), curr_dim_r(curr_dim_r), + curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), dist_c(dist_c), + ratio_c(ratio_c), v1(v1), v2(v2), w(w) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Lpk2ReoFunctor; + FunctorType functor(shape, shape_c, processed_n, processed_dims, curr_dim_r, + curr_dim_c, curr_dim_f, dist_c, ratio_c, v1, v2, w); + + SIZE nr = shape.dataHost()[curr_dim_r]; + SIZE nc = shape.dataHost()[curr_dim_c]; + SIZE nf = shape.dataHost()[curr_dim_f]; + SIZE nc_c = shape_c.dataHost()[curr_dim_c]; + SIZE nf_c = shape_c.dataHost()[curr_dim_f]; + + SIZE total_thread_z = nr; + SIZE total_thread_y = nc_c; + SIZE total_thread_x = nf_c; + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = R; + tby = C; + tbx = F; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + + for (DIM d = 0; d < D; d++) { + if (d != curr_dim_f && d != curr_dim_c && d != curr_dim_r) { + SIZE t = shape.dataHost()[d]; + for (DIM k = 0; k < processed_n; k++) { + if (d == processed_dims.dataHost()[k]) { + t = shape_c.dataHost()[d]; + } + } + gridx *= t; + } + } + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, SIZE, DeviceType> shape; + SubArray<1, SIZE, DeviceType> shape_c; + DIM processed_n; + SubArray<1, DIM, DeviceType> processed_dims; + DIM curr_dim_r, curr_dim_c, curr_dim_f; + SubArray<1, T, DeviceType> dist_c; + SubArray<1, T, DeviceType> ratio_c; + SubArray v1, v2, w; +}; + +template +class Lpk3ReoFunctor : public Functor { +public: + MGARDX_CONT Lpk3ReoFunctor() {} + MGARDX_CONT Lpk3ReoFunctor( + SubArray<1, SIZE, DeviceType> shape, + SubArray<1, SIZE, DeviceType> shape_c, DIM processed_n, + SubArray<1, DIM, DeviceType> processed_dims, DIM curr_dim_r, + DIM curr_dim_c, DIM curr_dim_f, SubArray<1, T, DeviceType> dist_r, + SubArray<1, T, DeviceType> ratio_r, SubArray v1, + SubArray v2, SubArray w) + : shape(shape), shape_c(shape_c), processed_n(processed_n), + processed_dims(processed_dims), curr_dim_r(curr_dim_r), + curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), dist_r(dist_r), + ratio_r(ratio_r), v1(v1), v2(v2), w(w) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + threadId = (FunctorBase::GetThreadIdZ() * + (FunctorBase::GetBlockDimX() * + FunctorBase::GetBlockDimY())) + + (FunctorBase::GetThreadIdY() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + + Byte *sm = FunctorBase::GetSharedMemory(); + SIZE offset = 0; + ldsm1 = F; + ldsm2 = C; + v_sm = (T *)&sm[offset]; + advance_with_align(offset, ldsm1 * ldsm2 * (R * 2 + 3)); + + dist_r_sm = (T *)&sm[offset]; + advance_with_align(offset, R * 2 + 3); + ratio_r_sm = (T *)&sm[offset]; + advance_with_align(offset, R * 2 + 3); + + // switching data type + align_byte_offset(offset); + + shape_sm = (SIZE *)&sm[offset]; + advance_with_align(offset, D); + shape_c_sm = (SIZE *)&sm[offset]; + advance_with_align(offset, D); + + // switching data type + align_byte_offset(offset); + + processed_dims_sm = (DIM *)&sm[offset]; + advance_with_align(offset, D); + + if (threadId < D) { + shape_sm[threadId] = *shape(threadId); + shape_c_sm[threadId] = *shape_c(threadId); + } + if (threadId < processed_n) { + processed_dims_sm[threadId] = *processed_dims(threadId); + } + } + MGARDX_EXEC void Operation2() { + // __syncthreads(); + + for (DIM d = 0; d < D; d++) + idx[d] = 0; + + nr = shape_sm[curr_dim_r]; + nf_c = shape_c_sm[curr_dim_f]; + nc_c = shape_c_sm[curr_dim_c]; + nr_c = shape_c_sm[curr_dim_r]; + PADDING = (nr % 2 == 0); + + bidx = FunctorBase::GetBlockIdX(); + firstD = div_roundup(nf_c, FunctorBase::GetBlockDimX()); + blockId_f = bidx % firstD; + bidx /= firstD; + + for (DIM d = 0; d < D; d++) { + if (d != curr_dim_r && d != curr_dim_c && d != curr_dim_f) { + SIZE t = shape_sm[d]; + for (DIM k = 0; k < processed_n; k++) { + if (d == processed_dims_sm[k]) { + t = shape_c_sm[d]; + } + } + idx[d] = bidx % t; + bidx /= t; + } + } + + v1.offset(idx); + v2.offset(idx); + w.offset(idx); + + // SIZE other_offset_v = get_idx(ldvs_sm, idx); + // SIZE other_offset_w = get_idx(ldws_sm, idx); + + // dv1 = dv1 + other_offset_v; + // dv2 = dv2 + other_offset_v; + // dw = dw + other_offset_w; + + // if (debug2) { + // printf("idx: %d %d %d %d\n", idx[3], idx[2], idx[1], idx[0]); + // printf("other_offset_v: %llu\n", other_offset_v); + // printf("other_offset_w: %llu\n", other_offset_w); + // } + + r_gl = FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ() + + FunctorBase::GetThreadIdZ(); + c_gl = FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY() + + FunctorBase::GetThreadIdY(); + f_gl = blockId_f * FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + blockId = FunctorBase::GetBlockIdZ(); + + r_sm = FunctorBase::GetThreadIdZ(); + c_sm = FunctorBase::GetThreadIdY(); + f_sm = FunctorBase::GetThreadIdX(); + + actual_R = R; + if (nr_c - FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ() < + R) { + actual_R = nr_c - FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ(); + } + // if (nr_c % 2 == 1){ + // if(nr_c-1 - FunctorBase::GetBlockIdZ() * + // FunctorBase::GetBlockDimZ() < R) { actual_R = nr_c - 1 - + // FunctorBase::GetBlockIdZ() * + // FunctorBase::GetBlockDimZ(); } + // } else { + // if(nr_c - FunctorBase::GetBlockIdZ() * + // FunctorBase::GetBlockDimZ() < R) { actual_R = nr_c - + // FunctorBase::GetBlockIdZ() * + // FunctorBase::GetBlockDimZ(); } + // } + + // if (debug) printf("actual_R %d\n", actual_R); + + // bool debug = false; + // if (idx[3] == 0 && idx[2] == 0 && f_gl == 2 && c_gl == 1) debug = false; + + // if (debug) printf("RCF: %d %d %d\n", R, C, F); + if (r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + // if (debug) printf("load front vsm[%d]: %f <- %d %d %d\n", r_sm * 2 + 2, + // *v1( r_gl, c_gl, f_gl), r_gl, c_gl, f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 2, c_sm, f_sm)] = + *v1(r_gl, c_gl, f_gl); + + if (r_sm == actual_R - 1) { + if (r_gl + 1 < nr_c) { + // if (debug) printf("load front+1 vsm[%d]: %f <- %d %d %d\n", + // actual_R + // * 2 + 2, dv1[get_idx(lddv11, lddv12, blockId * R + actual_R, c_gl, + // f_gl)], blockId * R + actual_R, c_gl, f_gl); + v_sm[get_idx(ldsm1, ldsm2, actual_R * 2 + 2, c_sm, f_sm)] = + *v1(r_gl + 1, c_gl, f_gl); + } else { + // if (debug) printf("load front+1 vsm[%d]: 0.0\n", actual_R * 2 + 2); + v_sm[get_idx(ldsm1, ldsm2, actual_R * 2 + 2, c_sm, f_sm)] = 0.0; + } + } + + if (r_sm == 0) { + if (r_gl >= 1) { + // if (debug) printf("load front-1 vsm[0]: %f <- %d %d %d\n", + // *v1( r_gl-1, c_gl, f_gl), r_gl-1, c_gl, + // f_gl); + v_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + *v1(r_gl - 1, c_gl, f_gl); + } else { + // if (debug) printf("load front-1 vsm[0]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = 0.0; + } + } + + if (!PADDING) { + if (r_gl < nr_c - 1) { + // if (debug) printf("load back vsm[%d]: 0.0\n", r_sm * 2 + 3); + v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = + *v2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load back vsm[%d]: %f <- %d %d %d\n", r_sm * 2 + + // 3, *v2(r_gl, c_gl, f_gl), r_gl, c_gl, f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = 0.0; + } + } else { + if (r_gl < nr_c - 2) { + // if (debug) printf("load back vsm[%d]: %f <- %d %d %d\n", r_sm * 2 + + // 3, *v2(r_gl, c_gl, f_gl), r_gl, c_gl, f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = + *v2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load back vsm[%d]: 0.0\n", r_sm * 2 + 3); + v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = 0.0; + } + } + + if (r_gl >= 1 && + (PADDING && r_gl - 1 < nr_c - 2 || !PADDING && r_gl - 1 < nr_c - 1)) { + // if (blockId > 0) { + if (r_sm == 0) { + // if (debug) printf("load back-1 vsm[1]: %f <- %d %d %d\n", + // *v2(r_gl-1, c_gl, f_gl), r_gl-1, c_gl, + // f_gl); + v_sm[get_idx(ldsm1, ldsm2, 1, c_sm, f_sm)] = + *v2(r_gl - 1, c_gl, f_gl); + } + } else { + if (r_sm == 0) { + // if (debug) printf("load back-1 vsm[1]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, 1, c_sm, f_sm)] = 0.0; + } + } + } + + // load dist/ratio using f_sm for better performance + // assumption F >= R + if (r_sm == 0 && c_sm == 0 && f_sm < actual_R) { + if (blockId * R * 2 + f_sm < nr) { + dist_r_sm[2 + f_sm] = *dist_r(blockId * R * 2 + f_sm); + // if (debug2 ) printf("load dist 1 [%d]: %f [%d]\n", 2 + f_sm, + // dist_r_sm[2 + f_sm], blockId * R * 2 + f_sm); + ratio_r_sm[2 + f_sm] = *ratio_r(blockId * R * 2 + f_sm); + // if (debug2 )printf("load ratio 1 [%d]: %f [%d]\n", 2 + f_sm, + // ratio_r_sm[2 + f_sm], blockId * R * 2 + f_sm); + } else { + dist_r_sm[2 + f_sm] = 0.0; + ratio_r_sm[2 + f_sm] = 0.0; + } + if (blockId * R * 2 + actual_R + f_sm < nr) { + dist_r_sm[2 + actual_R + f_sm] = + *dist_r(blockId * R * 2 + actual_R + f_sm); + // if (debug2 )printf("load dist 2 [%d]: %f [%d]\n", 2 + actual_R + + // f_sm, dist_r_sm[2 + actual_R + f_sm], blockId * R * 2 + actual_R + + // f_sm); + ratio_r_sm[2 + actual_R + f_sm] = + *ratio_r(blockId * R * 2 + actual_R + f_sm); + // if (debug2 )printf("load ratio 2 [%d]: %f [%d]\n", 2 + actual_R + + // f_sm, ratio_r_sm[2 + actual_R + f_sm], blockId * R * 2 + actual_R + + // f_sm); + } else { + dist_r_sm[2 + actual_R + f_sm] = 0.0; + ratio_r_sm[2 + actual_R + f_sm] = 0.0; + } + } + + if (blockId > 0) { + if (f_sm < 2) { + dist_r_sm[f_sm] = *dist_r(blockId * R * 2 - 2 + f_sm); + // if (debug2 )printf("load dist -1 [%d]: %f [%d]\n", f_sm, + // dist_r_sm[f_sm], blockId * R * 2 - 2 + f_sm); + ratio_r_sm[f_sm] = *ratio_r(blockId * R * 2 - 2 + f_sm); + // if (debug2 )printf("load ratio -1 [%d]: %f [%d]\n", f_sm, + // ratio_r_sm[f_sm], blockId * R * 2 - 2 + f_sm); + } + } else { + if (f_sm < 2) { + dist_r_sm[f_sm] = 0.0; + ratio_r_sm[f_sm] = 0.0; + } + } + } + + MGARDX_EXEC void Operation3() { + // __syncthreads(); + + // int adjusted_nr_c = nr_c; + if (r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + T h1 = dist_r_sm[r_sm * 2]; + T h2 = dist_r_sm[r_sm * 2 + 1]; + T h3 = dist_r_sm[r_sm * 2 + 2]; + T h4 = dist_r_sm[r_sm * 2 + 3]; + T r1 = ratio_r_sm[r_sm * 2]; + T r2 = ratio_r_sm[r_sm * 2 + 1]; + T r3 = ratio_r_sm[r_sm * 2 + 2]; + T r4 = 1 - r3; + T a = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2, c_sm, f_sm)]; + T b = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 1, c_sm, f_sm)]; + T c = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 2, c_sm, f_sm)]; + T d = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)]; + T e = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 4, c_sm, f_sm)]; + + // __syncthreads(); + // if (debug) { + // printf("r_sm(%d) %f %f %f %f %f\n",r_sm, a,b,c,d,e); + // printf("r_sm_h(%d) %f %f %f %f\n",r_sm, h1,h2,h3,h4); + // printf("r_sm_r(%d) %f %f %f %f\n",r_sm, r1,r2,r3,r4); + // } + // __syncthreads(); + + // T tb = a * h1 + b * 2 * (h1+h2) + c * h2; + // T tc = b * h2 + c * 2 * (h2+h3) + d * h3; + // T td = c * h3 + d * 2 * (h3+h4) + e * h4; + + // if (debug) printf("f_sm(%d) tb tc td tc: %f %f %f %f\n", f_sm, tb, tc, + // td, tc+tb * r1 + td * r4); + + // tc += tb * r1 + td * r4; + + // if (debug) { + // printf("f_sm(%d) %f %f %f %f %f f_sm_h %f %f %f %f f_sm_r %f %f %f + // %f, out: %f\n",f_sm, a,b,c,d,e, h1,h2,h3,h4,r1,r2,r3,r4, + // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4)); + // } + + *w(r_gl, c_gl, f_gl) = + mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4); + + // if (debug) printf("store[%d %d %d] %f (%f)\n", r_gl, c_gl, f_gl, + // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4), + // mass_trans(a, b, c, (T)0.0, (T)0.0, h1, (T)0.0, (T)0.0, + // h4, r1, r2, (T)0.0, (T)0.0)); + // // printf("%d %d %d\n", r_gl, c_gl, f_gl); + // if (blockId * R + R == nr-1) { + // if (r_gl+1 == nr_c - 1) { + // if (r_gl+1 == nr_c - 1) { + // // T te = h4 * d + 2 * h4 * e; + // // te += td * r3; + // *w(blockId * R + actual_R, c_gl, f_gl) = + // mass_trans(c, d, e, (T)0.0, (T)0.0, + // h1, h2, (T)0.0, (T)0.0, r1, r2, (T)0.0, (T)0.0); + + // if (debug) printf("store-last[%d %d %d] %f\n", blockId * R + + // actual_R, c_gl, f_gl, + // mass_trans(c, d, e, (T)0.0, (T)0.0, + // h1, h2, (T)0.0, (T)0.0, r1, r2, (T)0.0, (T)0.0)); + // } + //} + } + v1.reset_offset(); + v2.reset_offset(); + w.reset_offset(); + } + + MGARDX_CONT SIZE shared_memory_size() { + SIZE size = 0; + size = ((R * 2 + 3) * C * F + (R * 2 + 3) * 2) * sizeof(T); + align_byte_offset(size); + size += (D * 4) * sizeof(SIZE); + align_byte_offset(size); + size += (D * 1) * sizeof(DIM); + return size; + } + +private: + // functor parameters + SubArray<1, SIZE, DeviceType> shape; + SubArray<1, SIZE, DeviceType> shape_c; + DIM processed_n; + SubArray<1, DIM, DeviceType> processed_dims; + DIM curr_dim_r, curr_dim_c, curr_dim_f; + SubArray<1, T, DeviceType> dist_r; + SubArray<1, T, DeviceType> ratio_r; + SubArray v1, v2, w; + + // thread local variables + + SIZE threadId; + + T *v_sm; + SIZE ldsm1, ldsm2; + T *dist_r_sm; + T *ratio_r_sm; + + SIZE *shape_sm; + SIZE *shape_c_sm; + DIM *processed_dims_sm; + + SIZE idx[D]; + SIZE nr, nc, nf, nf_c, nc_c, nr_c; + bool zero_other; + bool PADDING; + + SIZE bidx; + SIZE firstD; + SIZE blockId_f; + SIZE blockId; + + SIZE zero_r; + SIZE zero_c; + SIZE zero_f; + + SIZE r_gl; + SIZE c_gl; + SIZE f_gl; + + SIZE r_sm; + SIZE c_sm; + SIZE f_sm; + + SIZE actual_R; +}; + +template +class Lpk3ReoKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "lpk3_nd"; + MGARDX_CONT + Lpk3ReoKernel(SubArray<1, SIZE, DeviceType> shape, + SubArray<1, SIZE, DeviceType> shape_c, DIM processed_n, + SubArray<1, DIM, DeviceType> processed_dims, DIM curr_dim_r, + DIM curr_dim_c, DIM curr_dim_f, + SubArray<1, T, DeviceType> dist_r, + SubArray<1, T, DeviceType> ratio_r, + SubArray v1, SubArray v2, + SubArray w) + : shape(shape), shape_c(shape_c), processed_n(processed_n), + processed_dims(processed_dims), curr_dim_r(curr_dim_r), + curr_dim_c(curr_dim_c), curr_dim_f(curr_dim_f), dist_r(dist_r), + ratio_r(ratio_r), v1(v1), v2(v2), w(w) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Lpk3ReoFunctor; + FunctorType functor(shape, shape_c, processed_n, processed_dims, curr_dim_r, + curr_dim_c, curr_dim_f, dist_r, ratio_r, v1, v2, w); + + SIZE nr = shape.dataHost()[curr_dim_r]; + SIZE nc = shape.dataHost()[curr_dim_c]; + SIZE nf = shape.dataHost()[curr_dim_f]; + SIZE nr_c = shape_c.dataHost()[curr_dim_r]; + SIZE nc_c = shape_c.dataHost()[curr_dim_c]; + SIZE nf_c = shape_c.dataHost()[curr_dim_f]; + + SIZE total_thread_z = nr_c; + SIZE total_thread_y = nc_c; + SIZE total_thread_x = nf_c; + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = R; + tby = C; + tbx = F; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + + for (DIM d = 0; d < D; d++) { + if (d != curr_dim_f && d != curr_dim_c && d != curr_dim_r) { + SIZE t = shape.dataHost()[d]; + for (DIM k = 0; k < processed_n; k++) { + if (d == processed_dims.dataHost()[k]) { + t = shape_c.dataHost()[d]; + } + } + gridx *= t; + } + } + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, SIZE, DeviceType> shape; + SubArray<1, SIZE, DeviceType> shape_c; + DIM processed_n; + SubArray<1, DIM, DeviceType> processed_dims; + DIM curr_dim_r, curr_dim_c, curr_dim_f; + SubArray<1, T, DeviceType> dist_r; + SubArray<1, T, DeviceType> ratio_r; + SubArray v1, v2, w; +}; + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel3D.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel3D.hpp new file mode 100644 index 000000000..bdb8ddc20 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/Correction/LinearProcessingKernel3D.hpp @@ -0,0 +1,1098 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_LINEAR_PROCESSSING_KERNEL_3D_TEMPLATE +#define MGARD_X_160_LINEAR_PROCESSSING_KERNEL_3D_TEMPLATE + +#include "../../../RuntimeX/RuntimeX.h" +#include "LPKFunctor.h" +// #include "LinearProcessingKernel.h" + +// #include "../../Functor.h" +// #include "../../AutoTuners/AutoTuner.h" +// #include "../../Task.h" +// #include "../../DeviceAdapters/DeviceAdapter.h" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +class Lpk1Reo3DFunctor : public Functor { +public: + MGARDX_CONT Lpk1Reo3DFunctor() {} + MGARDX_CONT Lpk1Reo3DFunctor(SIZE nr, SIZE nc, SIZE nf, SIZE nf_c, + SIZE zero_r, SIZE zero_c, SIZE zero_f, + SubArray<1, T, DeviceType> ddist_f, + SubArray<1, T, DeviceType> dratio_f, + SubArray dv1, + SubArray dv2, + SubArray dw) + : nr(nr), nc(nc), nf(nf), nf_c(nf_c), zero_r(zero_r), zero_c(zero_c), + zero_f(zero_f), ddist_f(ddist_f), dratio_f(dratio_f), dv1(dv1), + dv2(dv2), dw(dw) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + // bool debug = false; + // if (blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == 1 && + // threadIdx.y == 0 && threadIdx.z == 0 ) debug = false; + + // bool debug2 = false; + // if (blockIdx.z == gridDim.z-1 && blockIdx.y == 1 && blockIdx.x == 16) + // debug2 = false; + + PADDING = (nf % 2 == 0); + + T *sm = (T *)FunctorBase::GetSharedMemory(); + ldsm1 = F * 2 + 3; + ldsm2 = C; + v_sm = sm; + dist_f_sm = sm + ldsm1 * ldsm2 * R; + ratio_f_sm = dist_f_sm + ldsm1; + + // if (FunctorBase::GetBlockIdZ() == 0 && + // FunctorBase::GetBlockIdY() == 0 && + // FunctorBase::GetBlockIdX() == 0 && + // FunctorBase::GetThreadIdZ() == 1 && + // FunctorBase::GetThreadIdY() == 0 ) debug = true; + + r_gl = FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ() + + FunctorBase::GetThreadIdZ(); + c_gl = FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY() + + FunctorBase::GetThreadIdY(); + f_gl = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + blockId = FunctorBase::GetBlockIdX(); + + r_sm = FunctorBase::GetThreadIdZ(); + c_sm = FunctorBase::GetThreadIdY(); + f_sm = FunctorBase::GetThreadIdX(); + + actual_F = F; + if (nf_c - blockId * FunctorBase::GetBlockDimX() < F) { + actual_F = nf_c - blockId * FunctorBase::GetBlockDimX(); + } + + // if (nf_c % 2 == 1){ + // if(nf_c-1 - blockId * blockDim.x < F) { actual_F = nf_c - 1 - blockId * + // blockDim.x; } + // } else { + // if(nf_c - blockId * blockDim.x < F) { actual_F = nf_c - blockId * + // blockDim.x; } + // } + + // if (debug) printf("actual_F %d\n", actual_F); + + if (r_gl < nr && c_gl < nc && f_gl < nf_c) { + if (r_gl < zero_r && c_gl < zero_c && f_gl < zero_f) { + // if (debug) printf("load left vsm[%d]: 0.0\n", f_sm * 2 + 2); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 2)] = 0.0; + } else { + // if (debug) printf("load left vsm[%d]<-dv1[%d, %d, %d]: %f\n", f_sm * + // 2 + // + 2, r_gl, c_gl, f_gl, dv1[get_idx(lddv11, lddv12, r_gl, c_gl, + // f_gl)]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 2)] = + *dv1(r_gl, c_gl, f_gl); + } + + if (f_sm == actual_F - 1) { + if (r_gl < zero_r && c_gl < zero_c && f_gl < zero_f) { + // if (debug) printf("load left+1 vsm[%d]: 0.0\n", actual_F * 2 + 2); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 2)] = 0.0; + } else { + if (f_gl + 1 < nf_c) { + // if (debug) printf("load left+1 vsm[%d]: %f\n", actual_F * 2 + 2, + // dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl + 1)]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 2)] = + *dv1(r_gl, c_gl, f_gl + 1); + } else { + // if (debug) printf("load left+1 vsm[%d]: 0.0\n", actual_F * 2 + + // 2); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 2)] = 0.0; + } + } + } + + if (f_sm == 0) { + // left + if (r_gl < zero_r && c_gl < zero_c && f_gl < zero_f) { + // coarse (-1) + // if (debug) printf("load left-1 vsm[0]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = 0.0; + } else { + if (f_gl >= 1) { + // other (-1) + // if (debug) printf("load left-1 vsm[0]: %f\n", dv1[get_idx(lddv11, + // lddv12, r_gl, c_gl, f_gl-1)]); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = + *dv1(r_gl, c_gl, f_gl - 1); + } else { + // other (-1) + // if (debug) printf("load left-1 vsm[0]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 0)] = 0.0; + } + } + } + + // right + if (!PADDING) { + if (nf_c % 2 != 0) { + if (f_gl >= 1 && f_gl < nf_c) { + // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * + // 2 + // + 1, dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - 1)], r_gl, + // c_gl, f_gl - 1); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = + *dv2(r_gl, c_gl, f_gl - 1); + } else { + // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 1); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = 0.0; + } + } else { // nf_c % 2 == 0 + if (f_gl < nf_c - 1) { + // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * + // 2 + // + 3, dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)], r_gl, c_gl, + // f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = + *dv2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 3); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = 0.0; + } + } + } else { // PADDING + if (nf_c % 2 != 0) { + if (f_gl >= 1 && f_gl < nf_c - 1) { + // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * + // 2 + // + 1, dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - 1)], r_gl, + // c_gl, f_gl - 1); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = + *dv2(r_gl, c_gl, f_gl - 1); + } else { + // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 1); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)] = 0.0; + } + } else { // nf_c % 2 == 0 + if (f_gl < nf_c - 2) { + // if (debug) printf("load right vsm[%d]: %f <- %d %d %d\n", f_sm * + // 2 + // + 3, dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl)], r_gl, c_gl, + // f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = + *dv2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load right vsm[%d]: 0\n", f_sm * 2 + 3); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)] = 0.0; + } + } + } + + if (f_sm == actual_F - 1) { + // right (+1) + if (!PADDING) { + if (nf_c % 2 != 0) { + if (f_gl < nf_c - 1) { + // if (debug) printf("load right+1 vsm[%d]: %f <- %d %d %d\n", + // actual_F * 2 + 1, dv2[get_idx(lddv21, lddv22, r_gl, c_gl, + // f_gl)], r_gl, c_gl, f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = + *dv2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load right+1 vsm[%d]: 0.0\n", actual_F * 2 + + // 1); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = 0.0; + } + } else { // nf_c % 2 == 0 + if (f_gl >= actual_F) { + // if (debug) printf("load right-1 vsm[1]: %f <- %d %d %d\n", + // dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - actual_F)], + // r_gl, c_gl, f_gl - actual_F); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = + *dv2(r_gl, c_gl, f_gl - actual_F); + } else { + // if (debug) printf("load right-1 vsm[1]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = 0.0; + } + } + } else { + if (nf_c % 2 != 0) { + if (f_gl < nf_c - 2) { + // if (debug) printf("actual_F(%d), load right+1 vsm[%d]: %f <- %d + // %d %d\n", actual_F, actual_F * 2 + 1, dv2[get_idx(lddv21, + // lddv22, r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = + *dv2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load right+1 vsm[%d]: 0.0\n", actual_F * 2 + + // 1); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, actual_F * 2 + 1)] = 0.0; + } + } else { // nf_c % 2 == 0 + if (f_gl >= actual_F && f_gl - actual_F < nf_c - 2) { + // if (debug) printf("load right-1 vsm[1]: %f <- %d %d %d\n", + // dv2[get_idx(lddv21, lddv22, r_gl, c_gl, f_gl - actual_F)], + // r_gl, c_gl, f_gl - actual_F); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = + *dv2(r_gl, c_gl, f_gl - actual_F); + } else { + // if (debug) printf("load right-1 vsm[1]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, 1)] = 0.0; + } + } + } + } + } + + // debug = false; + // if (r_gl == 0 && c_gl == 0) debug = true; + + if (r_sm == 0 && c_sm == 0 && f_sm < actual_F) { + if (blockId * F * 2 + f_sm < nf) { + dist_f_sm[2 + f_sm] = *ddist_f(blockId * F * 2 + f_sm); + ratio_f_sm[2 + f_sm] = *dratio_f(blockId * F * 2 + f_sm); + // if (debug) printf("load dist[%d] -> sm[%d]: %f\n", blockId * F * 2 + + // f_sm, 2 + f_sm, *ddist_f(blockId * F * 2 + f_sm)); + } else { + dist_f_sm[2 + f_sm] = 0.0; + ratio_f_sm[2 + f_sm] = 0.0; + } + + if (blockId * F * 2 + actual_F + f_sm < nf) { + dist_f_sm[2 + actual_F + f_sm] = + *ddist_f(blockId * F * 2 + actual_F + f_sm); + ratio_f_sm[2 + actual_F + f_sm] = + *dratio_f(blockId * F * 2 + actual_F + f_sm); + // if (debug) printf("load dist[%d] -> sm[%d]: %f\n", blockId * F * 2 + + // actual_F + f_sm, 2 + actual_F + f_sm, *ddist_f(blockId * F * 2 + + // actual_F + f_sm)); + } else { + dist_f_sm[2 + actual_F + f_sm] = 0.0; + ratio_f_sm[2 + actual_F + f_sm] = 0.0; + } + // dist_f_sm[2 + f_sm] = ddist_f[f_gl]; + // dist_f_sm[2 + actual_F + f_sm] = ddist_f[actual_F + f_gl]; + // ratio_f_sm[2 + f_sm] = dratio_f[f_gl]; + // ratio_f_sm[2 + actual_F + f_sm] = dratio_f[actual_F + f_gl]; + } + + if (blockId > 0) { + if (f_sm < 2) { + // dist_f_sm[f_sm] = ddist_f[f_gl - 2]; + // ratio_f_sm[f_sm] = dratio_f[f_gl - 2]; + dist_f_sm[f_sm] = *ddist_f(blockId * F * 2 + f_sm - 2); + ratio_f_sm[f_sm] = *dratio_f(blockId * F * 2 + f_sm - 2); + } + } else { + if (f_sm < 2) { + dist_f_sm[f_sm] = 0.0; + ratio_f_sm[f_sm] = 0.0; + } + } + } + + MGARDX_EXEC void Operation2() { + if (r_gl < nr && c_gl < nc && f_gl < nf_c) { + T h1 = dist_f_sm[f_sm * 2]; + T h2 = dist_f_sm[f_sm * 2 + 1]; + T h3 = dist_f_sm[f_sm * 2 + 2]; + T h4 = dist_f_sm[f_sm * 2 + 3]; + T r1 = ratio_f_sm[f_sm * 2]; + T r2 = ratio_f_sm[f_sm * 2 + 1]; + T r3 = ratio_f_sm[f_sm * 2 + 2]; + T r4 = 1 - r3; + T a = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2)]; + T b = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 1)]; + T c = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 2)]; + T d = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 3)]; + T e = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm, f_sm * 2 + 4)]; + + // if (f_gl == 0 && c_gl == 2 && r_gl == 0) { + // printf("f_sm(%d) %f %f %f %f %f\n",f_sm, a,b,c,d,e); + // printf("f_sm_h(%d) %f %f %f %f\n",f_sm, h1,h2,h3,h4); + // printf("f_sm_r(%d) %f %f %f %f\n",f_sm, r1,r2,r3,r4); + // } + + // T tb = a * h1/6 + b * (h1+h2)/3 + c * h2/6; + // T tc = b * h2/6 + c * (h2+h3)/3 + d * h3/6; + // T td = c * h3/6 + d * (h3+h4)/3 + e * h4/6; + + // if (f_gl == 0 && c_gl == 2 && r_gl == 0) { + // printf("tb tc td %f %f %f\n", tb, tc, td); + // } + + // if (debug) printf("f_sm(%d) tb tc td tc: %f %f %f %f\n", f_sm, tb, tc, + // td, tc+tb * r1 + td * r4); + + // tc += tb * r1 + td * r4; + + // if (f_gl == 0 && c_gl == 2 && r_gl == 0) { + // printf("tc %f\n", tc, td); + // } + + *dw(r_gl, c_gl, f_gl) = + mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4); + + // if (f_gl == 0 && c_gl == 2 && r_gl == 0) { + // // if (debug) + // printf("store[%d %d %d] %f \n", r_gl, c_gl, f_gl, + // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4)); + // } + + // printf("test block %d F %d nf %d\n", blockId, F, nf); + // if (f_gl+1 == nf_c-1) { + + // // T te = h4 * d + 2 * h4 * e; + // //printf("f_sm(%d) mm-e: %f\n", f_sm, te); + // // te += td * r3; + // dw[get_idx(lddw1, lddw2, r_gl, c_gl, f_gl+1)] = + // mass_trans(c, d, e, (T)0.0, (T)0.0, h1, h2, (T)0.0, (T)0.0, r1, + // r2, (T)0.0, (T)0.0); + // } + } + } + + MGARDX_EXEC void Operation3() {} + + MGARDX_EXEC void Operation4() {} + + MGARDX_EXEC void Operation5() {} + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + size = (R * C * (F * 2 + 3) + (F * 2 + 3) * 2) * sizeof(T); + return size; + } + +private: + // functor parameters + SIZE nr, nc, nf, nf_c, zero_r, zero_c, zero_f; + SubArray<1, T, DeviceType> ddist_f; + SubArray<1, T, DeviceType> dratio_f; + SubArray dv1, dv2, dw; + + // thread local variables + bool PADDING; + SIZE ldsm1; + SIZE ldsm2; + T *v_sm; + T *dist_f_sm; + T *ratio_f_sm; + SIZE r_gl, c_gl, f_gl; + SIZE blockId; + SIZE r_sm, c_sm, f_sm; + SIZE actual_F; + bool debug = false; +}; + +template +class Lpk1Reo3DKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "lpk1_3d"; + MGARDX_CONT + Lpk1Reo3DKernel(SIZE nr, SIZE nc, SIZE nf, SIZE nf_c, SIZE zero_r, + SIZE zero_c, SIZE zero_f, SubArray<1, T, DeviceType> ddist_f, + SubArray<1, T, DeviceType> dratio_f, + SubArray dv1, + SubArray dv2, SubArray dw) + : nr(nr), nc(nc), nf(nf), nf_c(nf_c), zero_r(zero_r), zero_c(zero_c), + zero_f(zero_f), ddist_f(ddist_f), dratio_f(dratio_f), dv1(dv1), + dv2(dv2), dw(dw) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Lpk1Reo3DFunctor; + FunctorType functor(nr, nc, nf, nf_c, zero_r, zero_c, zero_f, ddist_f, + dratio_f, dv1, dv2, dw); + + SIZE total_thread_z = nr; + SIZE total_thread_y = nc; + SIZE total_thread_x = nf_c; + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = R; + tby = C; + tbx = F; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE nr, nc, nf, nf_c, zero_r, zero_c, zero_f; + SubArray<1, T, DeviceType> ddist_f; + SubArray<1, T, DeviceType> dratio_f; + SubArray dv1, dv2, dw; +}; + +template +class Lpk2Reo3DFunctor : public Functor { +public: + MGARDX_CONT Lpk2Reo3DFunctor() {} + MGARDX_CONT Lpk2Reo3DFunctor(SIZE nr, SIZE nc, SIZE nf_c, SIZE nc_c, + SubArray<1, T, DeviceType> ddist_c, + SubArray<1, T, DeviceType> dratio_c, + SubArray dv1, + SubArray dv2, + SubArray dw) + : nr(nr), nc(nc), nf_c(nf_c), nc_c(nc_c), ddist_c(ddist_c), + dratio_c(dratio_c), dv1(dv1), dv2(dv2), dw(dw) { + Functor(); + } + MGARDX_EXEC void Operation1() { + // bool debug = false; + // if (blockIdx.y == gridDim.y-1 && blockIdx.x == 0 && + // threadIdx.x == 0 ) debug = false; + + // bool debug2 = false; + // if (blockIdx.z == gridDim.z-1 && blockIdx.y == 1 && blockIdx.x == 16) + // debug2 = false; + + PADDING = (nc % 2 == 0); + + T *sm = (T *)FunctorBase::GetSharedMemory(); + ldsm1 = F; + ldsm2 = C * 2 + 3; + v_sm = sm; + dist_c_sm = sm + ldsm1 * ldsm2 * R; + ratio_c_sm = dist_c_sm + ldsm2; + + // bool debug = false; + // if (blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == 0 && + // threadIdx.z == 0 && threadIdx.x == 0 ) debug = false; + + r_gl = FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ() + + FunctorBase::GetThreadIdZ(); + c_gl = FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY() + + FunctorBase::GetThreadIdY(); + f_gl = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + blockId = FunctorBase::GetBlockIdY(); + + r_sm = FunctorBase::GetThreadIdZ(); + c_sm = FunctorBase::GetThreadIdY(); + f_sm = FunctorBase::GetThreadIdX(); + + actual_C = C; + if (nc_c - FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY() < + C) { + actual_C = nc_c - FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY(); + } + + // if (nc_c % 2 == 1){ + // if(nc_c-1 - blockIdx.y * blockDim.y < C) { actual_C = nc_c - 1 - + // blockIdx.y * blockDim.y; } + // } else { + // if(nc_c - blockIdx.y * blockDim.y < C) { actual_C = nc_c - blockIdx.y * + // blockDim.y; } + // } + + // if (debug) printf("actual_C %d\n", actual_C); + + if (r_gl < nr && c_gl < nc_c && f_gl < nf_c) { + // if (debug) printf("load up vsm[%d]: %f <- %d %d %d\n", c_sm * 2 + 2, + // dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 2, f_sm)] = + *dv1(r_gl, c_gl, f_gl); + + if (c_sm == actual_C - 1) { + if (c_gl + 1 < nc_c) { + // if (debug) printf("load up+1 vsm[%d]: %f <- %d %d %d\n", actual_C * + // 2 + // + 2, dv1[get_idx(lddv11, lddv12, r_gl, blockId * C + actual_C, + // f_gl)], r_gl, blockId * C + actual_C, f_gl); + // c_gl+1 == blockId * C + C + v_sm[get_idx(ldsm1, ldsm2, r_sm, actual_C * 2 + 2, f_sm)] = + *dv1(r_gl, c_gl + 1, f_gl); + } else { + // if (debug) printf("load up+1 vsm[%d]: 0.0\n", actual_C * 2 + 2); + v_sm[get_idx(ldsm1, ldsm2, r_sm, actual_C * 2 + 2, f_sm)] = 0.0; + } + } + + if (c_sm == 0) { + if (c_gl >= 1) { + // if (debug) printf("load up-1 vsm[0]: %f <- %d %d %d\n", + // dv1[get_idx(lddv11, lddv12, r_gl, c_gl-1, f_gl)], r_gl, c_gl-1, + // f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = + *dv1(r_gl, c_gl - 1, f_gl); + } else { + // if (debug) printf("load up-1 vsm[0]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, r_sm, 0, f_sm)] = 0.0; + } + } + + if (!PADDING) { + if (c_gl < nc_c - 1) { + // if (debug) printf("load down vsm[%d]: %f <- %d %d %d\n", c_sm * 2 + + // 3, dv2[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, + // f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = + *dv2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load down vsm[%d]: 0.0\n", c_sm * 2 + 3); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = 0.0; + } + } else { + if (c_gl < nc_c - 2) { + // if (debug) printf("load down vsm[%d]: %f <- %d %d %d\n", c_sm * 2 + + // 3, dv2[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, + // f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = + *dv2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load down vsm[%d]: 0.0\n", c_sm * 2 + 3); + v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)] = 0.0; + } + } + + if (c_gl >= 1 && + (PADDING && c_gl - 1 < nc_c - 2 || !PADDING && c_gl - 1 < nc_c - 1)) { + if (c_sm == 0) { + // if (debug) printf("PADDING: %d, c_gl-1: %d nc_c-2: %d\n", PADDING, + // c_gl-1, nc_c - 2); if (debug) printf("load down-1 vsm[1]: %f <- %d + // %d %d\n", dv2[get_idx(lddv11, lddv12, r_gl, c_gl-1, f_gl)], r_gl, + // c_gl-1, f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm, 1, f_sm)] = + *dv2(r_gl, c_gl - 1, f_gl); + } + } else { + if (c_sm == 0) { + // if (debug) printf("load down-1 vsm[1]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, r_sm, 1, f_sm)] = 0.0; + } + } + } + + // load dist/ratio using f_sm for better performance + // assumption F >= C + if (r_sm == 0 && c_sm == 0 && f_sm < actual_C) { + if (blockId * C * 2 + f_sm < nc) { + dist_c_sm[2 + f_sm] = *ddist_c(blockId * C * 2 + f_sm); + ratio_c_sm[2 + f_sm] = *dratio_c(blockId * C * 2 + f_sm); + } else { + dist_c_sm[2 + f_sm] = 0.0; + ratio_c_sm[2 + f_sm] = 0.0; + } + + if (blockId * C * 2 + actual_C + f_sm < nc) { + dist_c_sm[2 + actual_C + f_sm] = + *ddist_c(blockId * C * 2 + actual_C + f_sm); + ratio_c_sm[2 + actual_C + f_sm] = + *dratio_c(blockId * C * 2 + actual_C + f_sm); + } else { + dist_c_sm[2 + actual_C + f_sm] = 0.0; + ratio_c_sm[2 + actual_C + f_sm] = 0.0; + } + } + + if (blockId > 0) { + if (f_sm < 2) { + dist_c_sm[f_sm] = *ddist_c(blockId * C * 2 - 2 + f_sm); + ratio_c_sm[f_sm] = *dratio_c(blockId * C * 2 - 2 + f_sm); + } + } else { + if (f_sm < 2) { + dist_c_sm[f_sm] = 0.0; + ratio_c_sm[f_sm] = 0.0; + } + } + } + + MGARDX_EXEC void Operation2() { + if (r_gl < nr && c_gl < nc_c && f_gl < nf_c) { + T h1 = dist_c_sm[c_sm * 2]; + T h2 = dist_c_sm[c_sm * 2 + 1]; + T h3 = dist_c_sm[c_sm * 2 + 2]; + T h4 = dist_c_sm[c_sm * 2 + 3]; + T r1 = ratio_c_sm[c_sm * 2]; + T r2 = ratio_c_sm[c_sm * 2 + 1]; + T r3 = ratio_c_sm[c_sm * 2 + 2]; + T r4 = 1 - r3; + T a = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2, f_sm)]; + T b = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 1, f_sm)]; + T c = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 2, f_sm)]; + T d = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 3, f_sm)]; + T e = v_sm[get_idx(ldsm1, ldsm2, r_sm, c_sm * 2 + 4, f_sm)]; + + // if (debug) { + // printf("c_sm(%d) %f %f %f %f %f\n",c_sm, a,b,c,d,e); + // printf("c_sm_h(%d) %f %f %f %f\n",c_sm, h1,h2,h3,h4); + // printf("c_sm_r(%d) %f %f %f %f\n",c_sm, r1,r2,r3,r4); + // } + + // T tb = a * h1 + b * 2 * (h1+h2) + c * h2; + // T tc = b * h2 + c * 2 * (h2+h3) + d * h3; + // T td = c * h3 + d * 2 * (h3+h4) + e * h4; + + // if (debug) printf("c_sm(%d) tb tc td tc: %f %f %f %f\n", f_sm, tb, tc, + // td, tc+tb * r1 + td * r4); + + // tc += tb * r1 + td * r4; + + // if (r_gl == 0 && f_gl == 0 && r_sm == 0 && f_sm == 0) { + // printf("mr2(%d) mm2: %f -> (%d %d %d)\n", c_sm, tc, r_gl, c_gl, + // f_gl); + // // printf("f_sm(%d) b c d: %f %f %f\n", f_sm, tb, tc, td); + // } + + *dw(r_gl, c_gl, f_gl) = + mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4); + + // if (debug) printf("store[%d %d %d] %f \n", r_gl, c_gl, f_gl, + // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4)); + + // printf("%d %d %d\n", r_gl, c_gl, f_gl); + // if (blockId * C + C == nc-1) { + // if (c_gl + 1 == nc_c - 1) { + // // T te = h4 * d + 2 * h4 * e; + // // te += td * r3; + // dw[get_idx(lddw1, lddw2, r_gl, blockId * C + actual_C, f_gl)] = + // mass_trans(c, d, e, (T)0.0, (T)0.0, + // h1, h2, (T)0.0, (T)0.0, r1, r2, (T)0.0, (T)0.0); + // } + // } + } + } + + MGARDX_EXEC void Operation3() {} + + MGARDX_EXEC void Operation4() {} + + MGARDX_EXEC void Operation5() {} + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + size = (R * (C * 2 + 3) * F + (C * 2 + 3) * 2) * sizeof(T); + return size; + } + +private: + // functor parameters + SIZE nr, nc, nf_c, nc_c; + SubArray<1, T, DeviceType> ddist_c; + SubArray<1, T, DeviceType> dratio_c; + SubArray dv1, dv2, dw; + + // thread local variables + bool PADDING; + SIZE ldsm1; + SIZE ldsm2; + T *v_sm; + T *dist_c_sm; + T *ratio_c_sm; + SIZE r_gl, c_gl, f_gl; + SIZE blockId; + SIZE r_sm, c_sm, f_sm; + SIZE actual_C; + bool debug; +}; + +template +class Lpk2Reo3DKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "lpk2_3d"; + MGARDX_CONT + Lpk2Reo3DKernel(SIZE nr, SIZE nc, SIZE nf_c, SIZE nc_c, + SubArray<1, T, DeviceType> ddist_c, + SubArray<1, T, DeviceType> dratio_c, + SubArray dv1, + SubArray dv2, SubArray dw) + : nr(nr), nc(nc), nf_c(nf_c), nc_c(nc_c), ddist_c(ddist_c), + dratio_c(dratio_c), dv1(dv1), dv2(dv2), dw(dw) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Lpk2Reo3DFunctor; + FunctorType functor(nr, nc, nf_c, nc_c, ddist_c, dratio_c, dv1, dv2, dw); + + SIZE total_thread_z = nr; + SIZE total_thread_y = nc_c; + SIZE total_thread_x = nf_c; + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = R; + tby = C; + tbx = F; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE nr, nc, nf_c, nc_c; + SubArray<1, T, DeviceType> ddist_c; + SubArray<1, T, DeviceType> dratio_c; + SubArray dv1, dv2, dw; +}; + +template +class Lpk3Reo3DFunctor : public Functor { +public: + MGARDX_CONT Lpk3Reo3DFunctor() {} + MGARDX_CONT Lpk3Reo3DFunctor(SIZE nr, SIZE nc_c, SIZE nf_c, SIZE nr_c, + SubArray<1, T, DeviceType> ddist_r, + SubArray<1, T, DeviceType> dratio_r, + SubArray dv1, + SubArray dv2, + SubArray dw) + : nr(nr), nc_c(nc_c), nf_c(nf_c), nr_c(nr_c), ddist_r(ddist_r), + dratio_r(dratio_r), dv1(dv1), dv2(dv2), dw(dw) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + // bool debug = false; + // if (blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == 0 && + // threadIdx.y == 0 && threadIdx.x == 0 ) debug = true; + + // bool debug2 = false; + // if (blockIdx.z == 0 && blockIdx.y == 0 && blockIdx.x == 0) + // debug2 = true; + + PADDING = (nr % 2 == 0); + T *sm = (T *)FunctorBase::GetSharedMemory(); + ldsm1 = F; + ldsm2 = C; + v_sm = sm; + dist_r_sm = sm + ldsm1 * ldsm2 * (R * 2 + 3); + ratio_r_sm = dist_r_sm + (R * 2 + 3); + + r_gl = FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ() + + FunctorBase::GetThreadIdZ(); + c_gl = FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY() + + FunctorBase::GetThreadIdY(); + f_gl = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + // if (debug) printf("debugging gl: %d %d %d\n", r_gl, c_gl, f_gl); + + blockId = FunctorBase::GetBlockIdZ(); + + r_sm = FunctorBase::GetThreadIdZ(); + c_sm = FunctorBase::GetThreadIdY(); + f_sm = FunctorBase::GetThreadIdX(); + + actual_R = R; + if (nr_c - FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ() < + R) { + actual_R = nr_c - FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ(); + } + // if (nr_c % 2 == 1){ + // if(nr_c-1 - blockIdx.z * blockDim.z < R) { actual_R = nr_c - 1 - + // blockIdx.z * blockDim.z; } + // } else { + // if(nr_c - blockIdx.z * blockDim.z < R) { actual_R = nr_c - blockIdx.z * + // blockDim.z; } + // } + + // if (debug) printf("actual_R %d\n", actual_R); + + // if (debug) printf("RCF: %d %d %d\n", R, C, F); + if (r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + // if (debug) printf("load front vsm[%d]: %f <- %d %d %d\n", r_sm * 2 + 2, + // dv1[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 2, c_sm, f_sm)] = + *dv1(r_gl, c_gl, f_gl); + + if (r_sm == actual_R - 1) { + if (r_gl + 1 < nr_c) { + // if (debug) printf("load front+1 vsm[%d]: %f <- %d %d %d\n", + // actual_R + // * 2 + 2, dv1[get_idx(lddv11, lddv12, blockId * R + actual_R, c_gl, + // f_gl)], blockId * R + actual_R, c_gl, f_gl); + v_sm[get_idx(ldsm1, ldsm2, actual_R * 2 + 2, c_sm, f_sm)] = + *dv1(r_gl + 1, c_gl, f_gl); + } else { + // if (debug) printf("load front+1 vsm[%d]: 0.0\n", actual_R * 2 + 2); + v_sm[get_idx(ldsm1, ldsm2, actual_R * 2 + 2, c_sm, f_sm)] = 0.0; + } + } + + if (r_sm == 0) { + if (r_gl >= 1) { + // if (debug) printf("load front-1 vsm[0]: %f <- %d %d %d\n", + // dv1[get_idx(lddv11, lddv12, r_gl-1, c_gl, f_gl)], r_gl-1, c_gl, + // f_gl); + v_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = + *dv1(r_gl - 1, c_gl, f_gl); + } else { + // if (debug) printf("load front-1 vsm[0]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, 0, c_sm, f_sm)] = 0.0; + } + } + + if (!PADDING) { + if (r_gl < nr_c - 1) { + // if (debug) printf("load back vsm[%d]: %f <- %d %d %d\n", r_sm * 2 + + // 3, dv2[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, + // f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = + *dv2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load back vsm[%d]: 0.0\n", r_sm * 2 + 3); + v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = 0.0; + } + } else { + if (r_gl < nr_c - 2) { + // if (debug) printf("load back vsm[%d]: %f <- %d %d %d\n", r_sm * 2 + + // 3, dv2[get_idx(lddv11, lddv12, r_gl, c_gl, f_gl)], r_gl, c_gl, + // f_gl); + v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = + *dv2(r_gl, c_gl, f_gl); + } else { + // if (debug) printf("load back vsm[%d]: 0.0\n", r_sm * 2 + 3); + v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)] = 0.0; + } + } + + if (r_gl >= 1 && + (PADDING && r_gl - 1 < nr_c - 2 || !PADDING && r_gl < nr_c)) { + // if (blockId > 0) { + if (r_sm == 0) { + // if (debug) printf("load back-1 vsm[1]: %f <- %d %d %d\n", + // dv2[get_idx(lddv11, lddv12, r_gl-1, c_gl, f_gl)], r_gl-1, c_gl, + // f_gl); + v_sm[get_idx(ldsm1, ldsm2, 1, c_sm, f_sm)] = + *dv2(r_gl - 1, c_gl, f_gl); + } + } else { + if (r_sm == 0) { + // if (debug) printf("load back-1 vsm[1]: 0.0\n"); + v_sm[get_idx(ldsm1, ldsm2, 1, c_sm, f_sm)] = 0.0; + } + } + } + + // load dist/ratio using f_sm for better performance + // assumption F >= R + // if (debug2) printf("actual_R: %u\n", actual_R); + if (r_sm == 0 && c_sm == 0 && f_sm < actual_R) { + // if (debug2) printf(" RCF (%u %u %u)blockid(%u) fsm(%u) nr(%u)\n", R, C, + // F, blockId, blockId * R * 2 + f_sm, nr); + if (blockId * R * 2 + f_sm < nr) { + + dist_r_sm[2 + f_sm] = *ddist_r(blockId * R * 2 + f_sm); + // if (debug2 ) printf("load dist 1 [%d]: %f [%d]\n", 2 + f_sm, + // dist_r_sm[2 + f_sm], blockId * R * 2 + f_sm); + ratio_r_sm[2 + f_sm] = *dratio_r(blockId * R * 2 + f_sm); + // if (debug2 )printf("load ratio 1 [%d]: %f [%d]\n", 2 + f_sm, + // ratio_r_sm[2 + f_sm], blockId * R * 2 + f_sm); + } else { + dist_r_sm[2 + f_sm] = 0.0; + ratio_r_sm[2 + f_sm] = 0.0; + } + if (blockId * R * 2 + actual_R + f_sm < nr) { + dist_r_sm[2 + actual_R + f_sm] = + *ddist_r(blockId * R * 2 + actual_R + f_sm); + // if (debug2 )printf("load dist 2 [%d]: %f [%d]\n", 2 + actual_R + + // f_sm, dist_r_sm[2 + actual_R + f_sm], blockId * R * 2 + actual_R + + // f_sm); + ratio_r_sm[2 + actual_R + f_sm] = + *dratio_r(blockId * R * 2 + actual_R + f_sm); + // if (debug2 )printf("load ratio 2 [%d]: %f [%d]\n", 2 + actual_R + + // f_sm, ratio_r_sm[2 + actual_R + f_sm], blockId * R * 2 + actual_R + + // f_sm); + } else { + dist_r_sm[2 + actual_R + f_sm] = 0.0; + ratio_r_sm[2 + actual_R + f_sm] = 0.0; + } + } + + if (blockId > 0) { + if (f_sm < 2) { + dist_r_sm[f_sm] = *ddist_r(blockId * R * 2 - 2 + f_sm); + // if (debug2 )printf("load dist -1 [%d]: %f [%d]\n", f_sm, + // dist_r_sm[f_sm], blockId * R * 2 - 2 + f_sm); + ratio_r_sm[f_sm] = *dratio_r(blockId * R * 2 - 2 + f_sm); + // if (debug2 )printf("load ratio -1 [%d]: %f [%d]\n", f_sm, + // ratio_r_sm[f_sm], blockId * R * 2 - 2 + f_sm); + } + } else { + if (f_sm < 2) { + dist_r_sm[f_sm] = 0.0; + ratio_r_sm[f_sm] = 0.0; + } + } + } + + MGARDX_EXEC void Operation2() { + // int adjusted_nr_c = nr_c; + if (r_gl < nr_c && c_gl < nc_c && f_gl < nf_c) { + T h1 = dist_r_sm[r_sm * 2]; + T h2 = dist_r_sm[r_sm * 2 + 1]; + T h3 = dist_r_sm[r_sm * 2 + 2]; + T h4 = dist_r_sm[r_sm * 2 + 3]; + T r1 = ratio_r_sm[r_sm * 2]; + T r2 = ratio_r_sm[r_sm * 2 + 1]; + T r3 = ratio_r_sm[r_sm * 2 + 2]; + T r4 = 1 - r3; + T a = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2, c_sm, f_sm)]; + T b = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 1, c_sm, f_sm)]; + T c = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 2, c_sm, f_sm)]; + T d = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 3, c_sm, f_sm)]; + T e = v_sm[get_idx(ldsm1, ldsm2, r_sm * 2 + 4, c_sm, f_sm)]; + + // __syncthreads(); + // if (debug) { + // printf("r_sm(%d) %f %f %f %f %f\n",r_sm, a,b,c,d,e); + // printf("r_sm_h(%d) %f %f %f %f\n",r_sm, h1,h2,h3,h4); + // printf("r_sm_r(%d) %f %f %f %f\n",r_sm, r1,r2,r3,r4); + // } + // __syncthreads(); + + // T tb = a * h1/6 + b * 2 * (h1+h2)/6 + c * h2/6; + // T tc = b * h2/6 + c * 2 * (h2+h3)/6 + d * h3/6; + // T td = c * h3/6 + d * 2 * (h3+h4)/6 + e * h4/6; + + // if (debug) printf("f_sm(%d) tb tc td tc: %f %f %f %f\n", f_sm, tb, tc, + // td, tc+tb * r1 + td * r4); + + // tc += tb * r1 + td * r4; + + *dw(r_gl, c_gl, f_gl) = + mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4); + + // if (debug) printf("store[%d %d %d] %f (%f)\n", r_gl, c_gl, f_gl, + // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4), + // mass_trans(a, b, c, (T)0.0, (T)0.0, h1, (T)0.0, (T)0.0, + // h4, r1, r2, (T)0.0, (T)0.0)); + // // printf("%d %d %d\n", r_gl, c_gl, f_gl); + // if (blockId * R + R == nr-1) { + // if (r_gl+1 == nr_c - 1) { + // if (r_gl+1 == nr_c - 1) { + // // T te = h4 * d + 2 * h4 * e; + // // te += td * r3; + // dw[get_idx(lddw1, lddw2, blockId * R + actual_R, c_gl, f_gl)] = + // mass_trans(c, d, e, (T)0.0, (T)0.0, + // h1, h2, (T)0.0, (T)0.0, r1, r2, (T)0.0, (T)0.0); + + // if (debug) printf("store-last[%d %d %d] %f\n", blockId * R + + // actual_R, c_gl, f_gl, + // mass_trans(c, d, e, (T)0.0, (T)0.0, + // h1, h2, (T)0.0, (T)0.0, r1, r2, (T)0.0, (T)0.0)); + // } + //} + } + } + + MGARDX_EXEC void Operation3() {} + + MGARDX_EXEC void Operation4() {} + + MGARDX_EXEC void Operation5() {} + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + size = ((R * 2 + 3) * C * F + (R * 2 + 3) * 2) * sizeof(T); + return size; + } + +private: + // functor parameters + SIZE nr, nc_c, nf_c, nr_c; + SubArray<1, T, DeviceType> ddist_r; + SubArray<1, T, DeviceType> dratio_r; + SubArray dv1, dv2, dw; + + // thread local variables + bool PADDING; + SIZE ldsm1; + SIZE ldsm2; + T *v_sm; + T *dist_r_sm; + T *ratio_r_sm; + SIZE r_gl, c_gl, f_gl; + SIZE blockId; + SIZE r_sm, c_sm, f_sm; + SIZE actual_R; + bool debug; +}; + +template +class Lpk3Reo3DKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "lpk3_3d"; + MGARDX_CONT + Lpk3Reo3DKernel(SIZE nr, SIZE nc_c, SIZE nf_c, SIZE nr_c, + SubArray<1, T, DeviceType> ddist_r, + SubArray<1, T, DeviceType> dratio_r, + SubArray dv1, + SubArray dv2, SubArray dw) + : nr(nr), nc_c(nc_c), nf_c(nf_c), nr_c(nr_c), ddist_r(ddist_r), + dratio_r(dratio_r), dv1(dv1), dv2(dv2), dw(dw) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = Lpk3Reo3DFunctor; + FunctorType functor(nr, nc_c, nf_c, nr_c, ddist_r, dratio_r, dv1, dv2, dw); + + SIZE total_thread_z = nr_c; + SIZE total_thread_y = nc_c; + SIZE total_thread_x = nf_c; + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = R; + tby = C; + tbx = F; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SIZE nr, nc_c, nf_c, nr_c; + SubArray<1, T, DeviceType> ddist_r; + SubArray<1, T, DeviceType> dratio_r; + SubArray dv1, dv2, dw; +}; + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/DataRefactoring.h b/include/mgard-x-160/DataRefactoring/MultiDimension/DataRefactoring.h new file mode 100644 index 000000000..728fd413a --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/DataRefactoring.h @@ -0,0 +1,146 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_DATA_REFACTORING +#define MGARD_X_160_DATA_REFACTORING + +// #include "Common.h" +#include "../../Hierarchy/Hierarchy.h" +#include "../../RuntimeX/RuntimeXPublic.h" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +static bool multidim_refactoring_store = false; +static bool multidim_refactoring_verify = false; +static bool multidim_refactoring_debug_print = false; + +template +void CalcCoefficients3D(Hierarchy &hierarchy, + SubArray dinput, + SubArray &doutput, SIZE l, + int queue_idx); + +template +void CalcCoefficients3DWithErrorCollection( + Hierarchy &hierarchy, SubArray dinput, + SubArray &doutput, SIZE l, + SubArray max_coeffcient, + SubArray max_coeffcient_finer, int queue_idx); + +template +void CoefficientsRestore3D(Hierarchy &hierarchy, + SubArray dinput, + SubArray &doutput, SIZE l, + int queue_idx); + +template +void CalcCorrection3D(Hierarchy &hierarchy, + SubArray dcoeff, + SubArray &dcorrection, SIZE l, + int queue_idx); + +template +void CalcCoefficientsND(Hierarchy &hierarchy, + SubArray dinput1, + SubArray dinput2, + SubArray &doutput, SIZE l, + int queue_idx); + +template +void CoefficientsRestoreND(Hierarchy &hierarchy, + SubArray dinput1, + SubArray dinput2, + SubArray &doutput, SIZE l, + int queue_idx); + +template +void CalcCorrectionND(Hierarchy &hierarchy, + SubArray dcoeff, + SubArray &dcorrection, SIZE l, + int queue_idx); + +template +void CopyND(SubArray dinput, + SubArray doutput, int queue_idx); + +template +void Copy3D(SubArray dinput, + SubArray doutput, bool padding, int queue_idx); + +template +void AddND(SubArray dinput, + SubArray doutput, int queue_idx); + +template +void SubtractND(SubArray dinput, + SubArray doutput, int queue_idx); + +template +void decompose(Hierarchy &hierarchy, + SubArray &v, SubArray w, + SubArray b, int start_level, int stop_level, + bool orthogonal_projection, int queue_idx); + +template +void recompose(Hierarchy &hierarchy, + SubArray &v, SubArray w, + SubArray b, int start_level, int stop_level, + bool orthogonal_projection, int queue_idx); + +template +void decompose_adaptive_resolution( + Hierarchy &hierarchy, SubArray &v, + SIZE l_target, SubArray<1, T, DeviceType> level_max, + SubArray *max_coefficient, int queue_idx); + +template +Array recompose_adaptive_resolution( + Hierarchy &hierarchy, SubArray &v, + SIZE l_target, T iso_value, T error_target, + SubArray<1, T, DeviceType> level_max, + SubArray *max_abs_coefficient, + SubArray *refinement_flag, int queue_idx); + +template +void LevelMax(SIZE l_target, SubArray<1, SIZE, DeviceType> ranges, + SubArray v, + SubArray<1, T, DeviceType> level_max, int queue_idx); + +template +void EarlyFeatureDetector(SubArray v, T current_error, + T iso_value, + SubArray feature_flag_coarser, + SubArray feature_flag, + int queue_idx); + +template +void AccuracyGuard(SIZE total_level, SIZE current_level, + SubArray<1, T, DeviceType> error_impact_budget, + SubArray<1, T, DeviceType> previous_error_impact, + SubArray<1, T, DeviceType> current_error_impact, + SubArray max_abs_coefficient, + SubArray refinement_flag, + int queue_idx); + +template +void CoefficientRetriever(Hierarchy &hierarchy, + SubArray dinput, + SubArray refinement_flag, + SubArray &doutput, SIZE l, + int queue_idx); + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/MultiDimension/DataRefactoring.hpp b/include/mgard-x-160/DataRefactoring/MultiDimension/DataRefactoring.hpp new file mode 100644 index 000000000..2076f06f0 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/MultiDimension/DataRefactoring.hpp @@ -0,0 +1,325 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../Hierarchy/Hierarchy.h" +#include "../../RuntimeX/RuntimeX.h" + +#include "DataRefactoring.h" + +#include + +#ifndef MGARD_X_160_MULTI_DIMENSION_DATA_REFACTORING_HPP +#define MGARD_X_160_MULTI_DIMENSION_DATA_REFACTORING_HPP + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template +void decompose(Hierarchy &hierarchy, + SubArray &v, SubArray w, + SubArray b, int start_level, int stop_level, + bool orthogonal_projection, int queue_idx) { + + if (start_level < 0 || start_level > hierarchy.l_target()) { + throw std::runtime_error("decompose: start_level out of bound."); + } + + if (stop_level < 0 || stop_level > hierarchy.l_target()) { + throw std::runtime_error("decompose: stop_level out of bound."); + } + + std::string prefix = "decomp_"; + if (sizeof(T) == sizeof(double)) + prefix += "d_"; + if (sizeof(T) == sizeof(float)) + prefix += "f_"; + for (int d = 0; d < D; d++) + prefix += + std::to_string(hierarchy.level_shape(hierarchy.l_target(), d)) + "_"; + + Array workspace; + bool shape_pass = true; + if (w.data() != nullptr) { + for (DIM d = 0; d < D; d++) { + if (w.shape(d) < hierarchy.level_shape(hierarchy.l_target())[d] + 2) { + shape_pass = false; + } + } + } + if (w.data() == nullptr || !shape_pass) { + log::info("decompose: allocating workspace as it is not pre-allocated."); + std::vector workspace_shape = + hierarchy.level_shape(hierarchy.l_target()); + for (DIM d = 0; d < D; d++) + workspace_shape[d] += 2; + workspace = Array(workspace_shape); + w = SubArray(workspace); + } + + // Array workspace(workspace_shape); + // workspace.memset(0); can cause large overhead in HIP + // SubArray w(workspace); + + SubArray v_fine = v; + SubArray w_fine = w; + SubArray v_coeff = v; + SubArray w_correction = w; + SubArray v_coarse = v; + + Timer timer; + if constexpr (D <= 3) { + for (int l = start_level; l > stop_level; l--) { + if (multidim_refactoring_debug_print) { + PrintSubarray("input v", v); + } + // if (log::level & log::TIME) + // timer.start(); + + v_fine.resize(hierarchy.level_shape(l)); + w_fine.resize(hierarchy.level_shape(l)); + CopyND(v_fine, w_fine, queue_idx); + + v_coeff.resize(hierarchy.level_shape(l)); + v_coarse.resize(hierarchy.level_shape(l - 1)); + CalcCoefficients3D(hierarchy, w_fine, v_coeff, l, queue_idx); + + if (orthogonal_projection) { + w_correction.resize(hierarchy.level_shape(l)); + CalcCorrection3D(hierarchy, v_coeff, w_correction, l, queue_idx); + + w_correction.resize(hierarchy.level_shape(l - 1)); + AddND(w_correction, v_coarse, queue_idx); + if (multidim_refactoring_debug_print) { + PrintSubarray("after add", v); + } + } + // if (log::level & log::TIME) { + // DeviceRuntime::SyncQueue(queue_idx); + // timer.end(); + // timer.print("Decomposition level " + std::to_string(l)); + // timer.clear(); + // } + } // end of loop + + if (multidim_refactoring_debug_print) { + PrintSubarray("output of decomposition", v); + } + } + + if constexpr (D > 3) { + Array workspace2; + bool shape_pass = true; + if (b.data() != nullptr) { + for (DIM d = 0; d < D; d++) { + if (b.shape(d) < hierarchy.level_shape(hierarchy.l_target())[d] + 2) { + shape_pass = false; + } + } + } + if (b.data() == nullptr || !shape_pass) { + log::info("decompose: allocating workspace as it is not pre-allocated."); + std::vector workspace_shape = + hierarchy.level_shape(hierarchy.l_target()); + for (DIM d = 0; d < D; d++) + workspace_shape[d] += 2; + workspace2 = Array(workspace_shape); + b = SubArray(workspace2); + } + // Array workspace2(workspace_shape); + // SubArray b(workspace2); + SubArray b_fine = b; + for (int l = start_level; l > stop_level; l--) { + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D("before coeff", v); + } + // if (log::level & log::TIME) + // timer.start(); + v_fine.resize(hierarchy.level_shape(l)); + w_fine.resize(hierarchy.level_shape(l)); + CopyND(v_fine, w_fine, queue_idx); + + v_fine.resize(hierarchy.level_shape(l)); + b_fine.resize(hierarchy.level_shape(l)); + CopyND(v_fine, b_fine, queue_idx); + + v_coeff.resize(hierarchy.level_shape(l)); + CalcCoefficientsND(hierarchy, w_fine, b_fine, v_coeff, l, queue_idx); + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D(format("after coeff[%d]", l), v_coeff); + } // debug + + w_correction.resize(hierarchy.level_shape(l)); + CalcCorrectionND(hierarchy, v_coeff, w_correction, l, queue_idx); + + w_correction.resize(hierarchy.level_shape(l - 1)); + v_coarse.resize(hierarchy.level_shape(l - 1)); + AddND(w_correction, v_coarse, queue_idx); + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D(format("after apply correction[%d]", l), v); + } // debug + // if (log::level & log::TIME) { + // DeviceRuntime::SyncQueue(queue_idx); + // timer.end(); + // timer.print("Decomposition level " + std::to_string(l)); + // timer.clear(); + // } + } + } + // DeviceRuntime::SyncDevice(); +} + +template +void recompose(Hierarchy &hierarchy, + SubArray &v, SubArray w, + SubArray b, int start_level, int stop_level, + bool orthogonal_projection, int queue_idx) { + + if (stop_level < 0 || stop_level > hierarchy.l_target()) { + throw std::runtime_error("recompose: stop_level out of bound."); + } + + if (start_level < 0 || start_level > hierarchy.l_target()) { + throw std::runtime_error("recompose: start_level out of bound."); + } + + Array workspace; + bool shape_pass = true; + if (w.data() != nullptr) { + for (DIM d = 0; d < D; d++) { + if (w.shape(d) < hierarchy.level_shape(hierarchy.l_target())[d] + 2) { + shape_pass = false; + } + } + } + if (w.data() == nullptr || !shape_pass) { + log::info("recompose: allocating workspace as it is not pre-allocated."); + std::vector workspace_shape = + hierarchy.level_shape(hierarchy.l_target()); + for (DIM d = 0; d < D; d++) + workspace_shape[d] += 2; + workspace = Array(workspace_shape); + w = SubArray(workspace); + } + + SubArray v_fine = v; + SubArray w_fine = w; + SubArray v_coeff = v; + SubArray w_correction = w; + SubArray v_coarse = v; + + if constexpr (D <= 3) { + if (multidim_refactoring_debug_print) { + PrintSubarray("input of recomposition", v); + } + std::string prefix = "recomp_"; + if (sizeof(T) == sizeof(double)) + prefix += "d_"; + if (sizeof(T) == sizeof(float)) + prefix += "f_"; + for (int d = 0; d < D; d++) + prefix += + std::to_string(hierarchy.level_shape(hierarchy.l_target(), d)) + "_"; + + for (int l = start_level; l < stop_level; l++) { + + if (orthogonal_projection) { + v_coeff.resize(hierarchy.level_shape(l + 1)); + w_correction.resize(hierarchy.level_shape(l + 1)); + CalcCorrection3D(hierarchy, v_coeff, w_correction, l + 1, queue_idx); + + w_correction.resize(hierarchy.level_shape(l)); + v_coarse.resize(hierarchy.level_shape(l)); + SubtractND(w_correction, v_coarse, queue_idx); + } + + v_coeff.resize(hierarchy.level_shape(l + 1)); + w_fine.resize(hierarchy.level_shape(l + 1)); + CoefficientsRestore3D(hierarchy, v_coeff, w_fine, l + 1, queue_idx); + + v_fine.resize(hierarchy.level_shape(l)); + CopyND(w_fine, v_fine, queue_idx); + if (multidim_refactoring_debug_print) { + PrintSubarray("output of recomposition", v); + } + } + } + if constexpr (D > 3) { + Array workspace2; + bool shape_pass = true; + if (b.data() != nullptr) { + for (DIM d = 0; d < D; d++) { + if (b.shape(d) < hierarchy.level_shape(hierarchy.l_target())[d] + 2) { + shape_pass = false; + } + } + } + if (b.data() == nullptr || !shape_pass) { + log::info("decompose: allocating workspace as it is not pre-allocated."); + std::vector workspace_shape = + hierarchy.level_shape(hierarchy.l_target()); + for (DIM d = 0; d < D; d++) + workspace_shape[d] += 2; + workspace2 = Array(workspace_shape); + b = SubArray(workspace2); + } + SubArray b_fine = b; + for (int l = start_level; l < stop_level; l++) { + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D(format("before corection[%d]", l + 1), v); + } + + int curr_dim_r, curr_dim_c, curr_dim_f; + int lddv1, lddv2; + int lddw1, lddw2; + int lddb1, lddb2; + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D(format("before subtract correction[%d]", l + 1), v); + } // deb + + v_coeff.resize(hierarchy.level_shape(l + 1)); + w_correction.resize(hierarchy.level_shape(l + 1)); + CalcCorrectionND(hierarchy, v_coeff, w_correction, l + 1, queue_idx); + + w_correction.resize(hierarchy.level_shape(l)); + v_coarse.resize(hierarchy.level_shape(l)); + SubtractND(w_correction, v_coarse, queue_idx); + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D(format("after subtract correction[%d]", l + 1), v); + } // deb + + v_coeff.resize(hierarchy.level_shape(l + 1)); + w_fine.resize(hierarchy.level_shape(l + 1)); + b_fine.resize(hierarchy.level_shape(l + 1)); + CopyND(v_coeff, b_fine, queue_idx); + CopyND(v_coeff, w_fine, queue_idx); + + v_fine.resize(hierarchy.level_shape(l + 1)); + CoefficientsRestoreND(hierarchy, w_fine, b_fine, v_fine, l + 1, + queue_idx); + } // loop levels + + if (multidim_refactoring_debug_print) { // debug + PrintSubarray4D(format("final output"), v); + } // deb + } // D > 3 + // DeviceRuntime::SyncDevice(); +} + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CalcCoefficients.hpp b/include/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CalcCoefficients.hpp new file mode 100644 index 000000000..d70cfc58e --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CalcCoefficients.hpp @@ -0,0 +1,42 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../../Hierarchy/Hierarchy.h" +#include "../../../RuntimeX/RuntimeX.h" + +#include "../DataRefactoring.h" + +#include "CoefficientKernel.hpp" + +#ifndef MGARD_X_160_DATA_REFACTORING_CALC_COEFFICIENTS +#define MGARD_X_160_DATA_REFACTORING_CALC_COEFFICIENTS + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace single_dimension { + +template +void CalcCoefficients(DIM current_dim, SubArray<1, T, DeviceType> ratio, + SubArray v, + SubArray coarse, + SubArray coeff, int queue_idx) { + + DeviceLauncher::Execute( + SingleDimensionCoefficientKernel( + current_dim, ratio, v, coarse, coeff), + queue_idx); +} + +} // namespace single_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CoefficientKernel.hpp b/include/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CoefficientKernel.hpp new file mode 100644 index 000000000..c054f5567 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CoefficientKernel.hpp @@ -0,0 +1,219 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_SINGLE_DIMENSION_COEFFICIENT_KERNEL_TEMPLATE +#define MGARD_X_160_SINGLE_DIMENSION_COEFFICIENT_KERNEL_TEMPLATE + +#include "../../../RuntimeX/RuntimeX.h" + +#include "../../MultiDimension/Coefficient/GPKFunctor.h" + +#define DECOMPOSE 0 +#define RECOMPOSE 1 + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace single_dimension { + +template +class SingleDimensionCoefficientFunctor : public Functor { +public: + MGARDX_CONT SingleDimensionCoefficientFunctor() {} + MGARDX_CONT SingleDimensionCoefficientFunctor( + DIM current_dim, SubArray<1, T, DeviceType> ratio, + SubArray v, SubArray coarse, + SubArray coeff) + : current_dim(current_dim), ratio(ratio), v(v), coarse(coarse), + coeff(coeff) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE v_left_idx[D]; + SIZE v_middle_idx[D]; + SIZE v_right_idx[D]; + SIZE coeff_idx[D]; + SIZE corase_idx[D]; + + SIZE firstD = div_roundup(coeff.shape(D - 1), F); + + SIZE bidx = FunctorBase::GetBlockIdX(); + coeff_idx[D - 1] = + (bidx % firstD) * F + FunctorBase::GetThreadIdX(); + + bidx /= firstD; + if (D >= 2) + coeff_idx[D - 2] = FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY() + + FunctorBase::GetThreadIdY(); + if (D >= 3) + coeff_idx[D - 3] = FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ() + + FunctorBase::GetThreadIdZ(); + + for (int d = D - 4; d >= 0; d--) { + coeff_idx[d] = bidx % coeff.shape(d); + bidx /= coeff.shape(d); + } + + bool in_range = true; + for (int d = D - 1; d >= 0; d--) { + if (coeff_idx[d] >= coeff.shape(d)) + in_range = false; + } + + if (in_range) { + for (int d = D - 1; d >= 0; d--) { + if (d != current_dim) { + v_left_idx[d] = coeff_idx[d]; + v_middle_idx[d] = coeff_idx[d]; + v_right_idx[d] = coeff_idx[d]; + corase_idx[d] = coeff_idx[d]; + } else { + v_left_idx[d] = coeff_idx[d] * 2; + v_middle_idx[d] = coeff_idx[d] * 2 + 1; + v_right_idx[d] = coeff_idx[d] * 2 + 2; + corase_idx[d] = coeff_idx[d]; + } + } + + if (OP == DECOMPOSE) { + coeff[coeff_idx] = + v[v_middle_idx] - lerp(v[v_left_idx], v[v_right_idx], + *ratio(v_left_idx[current_dim])); + // if (coeff_idx[current_dim] == 1) { + // printf("left: %f, right: %f, middle: %f, ratio: %f, coeff: %f\n", + // *v(v_left_idx), *v(v_right_idx), *v(v_middle_idx), + // *ratio(v_left_idx[current_dim]), *coeff(coeff_idx)); + // } + coarse[corase_idx] = v[v_left_idx]; + if (coeff_idx[current_dim] == coeff.shape(current_dim) - 1) { + corase_idx[current_dim]++; + coarse[corase_idx] = v[v_right_idx]; + if (v.shape(current_dim) % 2 == 0) { + v_right_idx[current_dim]++; + corase_idx[current_dim]++; + coarse[corase_idx] = v[v_right_idx]; + } + } + } else if (OP == RECOMPOSE) { + T left = coarse[corase_idx]; + corase_idx[current_dim]++; + T right = coarse[corase_idx]; + corase_idx[current_dim]--; + + v[v_left_idx] = left; + if (coeff_idx[current_dim] == coeff.shape(current_dim) - 1) { + corase_idx[current_dim]++; + v[v_right_idx] = right; + if (v.shape(current_dim) % 2 == 0) { + v_right_idx[current_dim]++; + corase_idx[current_dim]++; + v[v_right_idx] = coarse[corase_idx]; + v_right_idx[current_dim]--; + corase_idx[current_dim]--; + } + corase_idx[current_dim]--; + } + + v[v_middle_idx] = coeff[coeff_idx] + + lerp(left, right, *ratio(v_left_idx[current_dim])); + // if (coeff_idx[current_dim] == 1) { + // printf("left: %f, right: %f, middle: %f (%f), ratio: %f, coeff: + // %f\n", + // *v(v_left_idx), *v(v_right_idx), *v(v_middle_idx), + // *coeff(coeff_idx) + lerp(*v(v_left_idx), *v(v_right_idx), + // *ratio(v_left_idx[current_dim])), + // *ratio(v_left_idx[current_dim]), *coeff(coeff_idx)); + // } + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + // functor parameters + DIM current_dim; + SubArray<1, T, DeviceType> ratio; + SubArray v; + SubArray coarse; + SubArray coeff; +}; + +template +class SingleDimensionCoefficientKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "sdck"; + MGARDX_CONT + SingleDimensionCoefficientKernel(DIM current_dim, + SubArray<1, T, DeviceType> ratio, + SubArray v, + SubArray coarse, + SubArray coeff) + : current_dim(current_dim), ratio(ratio), v(v), coarse(coarse), + coeff(coeff) {} + + template + MGARDX_CONT + Task> + GenTask(int queue_idx) { + + using FunctorType = + SingleDimensionCoefficientFunctor; + FunctorType functor(current_dim, ratio, v, coarse, coeff); + + SIZE nr = 1, nc = 1, nf = 1; + if (D >= 3) + nr = coeff.shape(D - 3); + if (D >= 2) + nc = coeff.shape(D - 2); + nf = coeff.shape(D - 1); + + SIZE total_thread_z = nr; + SIZE total_thread_y = nc; + SIZE total_thread_x = nf; + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + + tbz = R; + tby = C; + tbx = F; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + + for (DIM d = 3; d < D; d++) { + gridx *= coeff.shape(D - (d + 1)); + } + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + // functor parameters + DIM current_dim; + SubArray<1, T, DeviceType> ratio; + SubArray v; + SubArray coarse; + SubArray coeff; +}; + +} // namespace single_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CoefficientsRestore.hpp b/include/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CoefficientsRestore.hpp new file mode 100644 index 000000000..0c176cef3 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CoefficientsRestore.hpp @@ -0,0 +1,42 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../../Hierarchy/Hierarchy.h" +#include "../../../RuntimeX/RuntimeX.h" + +#include "../DataRefactoring.h" + +#include "CoefficientKernel.hpp" + +#ifndef MGARD_X_160_DATA_REFACTORING_COEFFICIENTS_RESTORE +#define MGARD_X_160_DATA_REFACTORING_COEFFICIENTS_RESTORE + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace single_dimension { + +template +void CoefficientsRestore(DIM current_dim, SubArray<1, T, DeviceType> ratio, + SubArray v, + SubArray coarse, + SubArray coeff, int queue_idx) { + + DeviceLauncher::Execute( + SingleDimensionCoefficientKernel( + current_dim, ratio, v, coarse, coeff), + queue_idx); +} + +} // namespace single_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/SingleDimension/Correction/CalcCorrection.hpp b/include/mgard-x-160/DataRefactoring/SingleDimension/Correction/CalcCorrection.hpp new file mode 100644 index 000000000..0f57b5f63 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/SingleDimension/Correction/CalcCorrection.hpp @@ -0,0 +1,102 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../../Hierarchy/Hierarchy.h" +#include "../../../RuntimeX/RuntimeX.h" + +#include "../DataRefactoring.h" + +#include "../../MultiDimension/CopyND/LevelwiseProcessingKernel.hpp" +#include "../../MultiDimension/Correction/IterativeProcessingKernel.hpp" + +#include "MassTransKernel.hpp" + +#ifndef MGARD_X_160_DATA_REFACTORING_CALC_CORRECTION +#define MGARD_X_160_DATA_REFACTORING_CALC_CORRECTION + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace single_dimension { + +template +void CalcCorrection(Hierarchy &hierarchy, + SubArray &coeff, + SubArray &correction, SIZE curr_dim, + SIZE l, int queue_idx) { + + DeviceLauncher::Execute( + SingleDimensionMassTransKernel( + curr_dim, SubArray(hierarchy.dist(l, curr_dim)), + SubArray(hierarchy.ratio(l, curr_dim)), coeff, correction), + queue_idx); + + if (singledim_refactoring_debug_print) { + PrintSubarray("SingleDimensionMassTrans", correction); + } + + DIM curr_dim_r, curr_dim_c, curr_dim_f; + if (curr_dim == D - 1) { + curr_dim_r = D - 3, curr_dim_c = D - 2, curr_dim_f = D - 1; + correction.project(curr_dim_r, curr_dim_c, curr_dim_f); + DeviceLauncher::Execute( + multi_dimension::Ipk1ReoKernel( + curr_dim_r, curr_dim_c, curr_dim_f, + SubArray(hierarchy.am(l - 1, D - 1)), + SubArray(hierarchy.bm(l - 1, D - 1)), correction), + queue_idx); + if (singledim_refactoring_debug_print) { + PrintSubarray("Ipk1Reo", correction); + } + + } else if (curr_dim == D - 2) { + curr_dim_r = D - 3, curr_dim_c = D - 2, curr_dim_f = D - 1; + correction.project(curr_dim_r, curr_dim_c, curr_dim_f); + DeviceLauncher::Execute( + multi_dimension::Ipk2ReoKernel( + curr_dim_r, curr_dim_c, curr_dim_f, + SubArray(hierarchy.am(l - 1, D - 2)), + SubArray(hierarchy.bm(l - 1, D - 2)), correction), + queue_idx); + if (singledim_refactoring_debug_print) { + PrintSubarray("Ipk2Reo", correction); + } + } else if (curr_dim == D - 3) { + curr_dim_r = D - 3, curr_dim_c = D - 2, curr_dim_f = D - 1; + correction.project(curr_dim_r, curr_dim_c, curr_dim_f); + DeviceLauncher::Execute( + multi_dimension::Ipk3ReoKernel( + curr_dim_r, curr_dim_c, curr_dim_f, + SubArray(hierarchy.am(l - 1, D - 3)), + SubArray(hierarchy.bm(l - 1, D - 3)), correction), + queue_idx); + if (singledim_refactoring_debug_print) { + PrintSubarray("Ipk3Reo", correction); + } + } else { + curr_dim_r = curr_dim, curr_dim_c = D - 2, curr_dim_f = D - 1; + correction.project(curr_dim_r, curr_dim_c, curr_dim_f); + DeviceLauncher::Execute( + multi_dimension::Ipk3ReoKernel( + curr_dim_r, curr_dim_c, curr_dim_f, + SubArray(hierarchy.am(l - 1, curr_dim)), + SubArray(hierarchy.bm(l - 1, curr_dim)), correction), + queue_idx); + if (singledim_refactoring_debug_print) { + PrintSubarray("Ipk3Reo", correction); + } + } +} + +} // namespace single_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/SingleDimension/Correction/MassTransKernel.hpp b/include/mgard-x-160/DataRefactoring/SingleDimension/Correction/MassTransKernel.hpp new file mode 100644 index 000000000..b970ea186 --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/SingleDimension/Correction/MassTransKernel.hpp @@ -0,0 +1,191 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_SINGLE_DIMENSION_MASSTRANS_KERNEL_TEMPLATE +#define MGARD_X_160_SINGLE_DIMENSION_MASSTRANS_KERNEL_TEMPLATE + +#include "../../../RuntimeX/RuntimeX.h" + +#include "../../MultiDimension/Correction/LPKFunctor.h" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace single_dimension { + +template +class SingleDimensionMassTransFunctor : public Functor { +public: + MGARDX_CONT SingleDimensionMassTransFunctor() {} + MGARDX_CONT SingleDimensionMassTransFunctor(DIM current_dim, + SubArray<1, T, DeviceType> dist, + SubArray<1, T, DeviceType> ratio, + SubArray coeff, + SubArray v) + : current_dim(current_dim), dist(dist), ratio(ratio), coeff(coeff), v(v) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE v_idx[D]; + + SIZE firstD = div_roundup(v.shape(D - 1), F); + + SIZE bidx = FunctorBase::GetBlockIdX(); + v_idx[D - 1] = + (bidx % firstD) * F + FunctorBase::GetThreadIdX(); + + bidx /= firstD; + if (D >= 2) + v_idx[D - 2] = FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY() + + FunctorBase::GetThreadIdY(); + if (D >= 3) + v_idx[D - 3] = FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ() + + FunctorBase::GetThreadIdZ(); + + for (int d = D - 4; d >= 0; d--) { + v_idx[d] = bidx % v.shape(d); + bidx /= v.shape(d); + } + + bool in_range = true; + for (int d = D - 1; d >= 0; d--) { + if (v_idx[d] >= v.shape(d)) + in_range = false; + } + + if (in_range) { + + const T a = 0.0; + const T c = 0.0; + const T e = 0.0; + T b = 0.0; + T d = 0.0; + + if (v_idx[current_dim] > 0 && + v_idx[current_dim] < coeff.shape(current_dim)) { + v_idx[current_dim]--; + b = coeff[v_idx]; + v_idx[current_dim]++; + } + + if (v_idx[current_dim] < coeff.shape(current_dim)) { + // v_idx[current_dim] ++; + d = coeff[v_idx]; + // v_idx[current_dim] --; + } + + T h1 = 0, h2 = 0, h3 = 0, h4 = 0; + T r1 = 0, r2 = 0, r3 = 0, r4 = 0; + + if (v_idx[current_dim] > 0 && + v_idx[current_dim] * 2 < + coeff.shape(current_dim) + v.shape(current_dim) - 1) { + h1 = *dist(v_idx[current_dim] * 2 - 2); + h2 = *dist(v_idx[current_dim] * 2 - 1); + r1 = *ratio(v_idx[current_dim] * 2 - 2); + r2 = *ratio(v_idx[current_dim] * 2 - 1); + } + if (v_idx[current_dim] * 2 < + coeff.shape(current_dim) + v.shape(current_dim) - 1) { + h3 = *dist(v_idx[current_dim] * 2); + h4 = *dist(v_idx[current_dim] * 2 + 1); + r3 = *ratio(v_idx[current_dim] * 2); + r4 = 1 - r3; + } + + // printf("v_idx = [%d %d] %f %f %f %f %f f_sm_h %f %f %f %f f_sm_r %f %f + // %f %f, out: %f\n", + // v_idx[1], v_idx[0], a,b,c,d,e, h1,h2,h3,h4, r1,r2,r3,r4, + // mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4)); + + v[v_idx] = mass_trans(a, b, c, d, e, h1, h2, h3, h4, r1, r2, r3, r4); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + // functor parameters + DIM current_dim; + SubArray<1, T, DeviceType> dist; + SubArray<1, T, DeviceType> ratio; + SubArray coeff; + SubArray v; +}; + +template +class SingleDimensionMassTransKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "sdmtk"; + MGARDX_CONT + SingleDimensionMassTransKernel(DIM current_dim, + SubArray<1, T, DeviceType> dist, + SubArray<1, T, DeviceType> ratio, + SubArray coeff, + SubArray v) + : current_dim(current_dim), dist(dist), ratio(ratio), coeff(coeff), v(v) { + } + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + + using FunctorType = + SingleDimensionMassTransFunctor; + FunctorType functor(current_dim, dist, ratio, coeff, v); + + SIZE nr = 1, nc = 1, nf = 1; + if (D >= 3) + nr = v.shape(D - 3); + if (D >= 2) + nc = v.shape(D - 2); + nf = v.shape(D - 1); + + SIZE total_thread_z = nr; + SIZE total_thread_y = nc; + SIZE total_thread_x = nf; + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + + tbz = R; + tby = C; + tbx = F; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + + for (int d = D - 4; d >= 0; d--) { + gridx *= coeff.shape(d); + } + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + // functor parameters + DIM current_dim; + SubArray<1, T, DeviceType> dist; + SubArray<1, T, DeviceType> ratio; + SubArray coeff; + SubArray v; +}; + +} // namespace single_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/SingleDimension/DataRefactoring.h b/include/mgard-x-160/DataRefactoring/SingleDimension/DataRefactoring.h new file mode 100644 index 000000000..a7047d7ea --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/SingleDimension/DataRefactoring.h @@ -0,0 +1,74 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_SINGLE_DIMENSION_DATA_REFACTORING +#define MGARD_X_160_SINGLE_DIMENSION_DATA_REFACTORING + +#include "../../Hierarchy/Hierarchy.h" +#include "../../RuntimeX/RuntimeXPublic.h" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace single_dimension { + +static bool singledim_refactoring_store = false; +static bool singledim_refactoring_verify = false; +static bool singledim_refactoring_debug_print = false; + +template +void CalcCoefficients(DIM current_dim, SubArray<1, T, DeviceType> ratio, + SubArray v, + SubArray coarse, + SubArray coeff, int queue_idx); + +template +void CoefficientsRestore(DIM current_dim, SubArray<1, T, DeviceType> ratio, + SubArray v, + SubArray coarse, + SubArray coeff, int queue_idx); + +template +void CalcCorrection(Hierarchy &hierarchy, + SubArray &coeff, + SubArray &correction, SIZE curr_dim, + SIZE l, int queue_idx); + +template +void decompose(Hierarchy &hierarchy, + SubArray &v, int start_level, int stop_level, + int queue_idx); + +template +void recompose(Hierarchy &hierarchy, + SubArray &v, int start_level, int stop_level, + int queue_idx); + +} // namespace single_dimension + +namespace multi_dimension { + +template +void CopyND(SubArray dinput, + SubArray doutput, int queue_idx); + +template +void AddND(SubArray dinput, + SubArray doutput, int queue_idx); + +template +void SubtractND(SubArray dinput, + SubArray doutput, int queue_idx); + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/DataRefactoring/SingleDimension/DataRefactoring.hpp b/include/mgard-x-160/DataRefactoring/SingleDimension/DataRefactoring.hpp new file mode 100644 index 000000000..145319e7e --- /dev/null +++ b/include/mgard-x-160/DataRefactoring/SingleDimension/DataRefactoring.hpp @@ -0,0 +1,197 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../Hierarchy/Hierarchy.h" +#include "../../RuntimeX/RuntimeX.h" + +#include "DataRefactoring.h" + +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace single_dimension { + +static bool store = false; +static bool verify = false; +static bool debug_print = false; + +template +void decompose(Hierarchy &hierarchy, + SubArray &v, int start_level, int stop_level, + int queue_idx) { + + if (stop_level < 0) { + throw std::runtime_error("decompose: stop_level out of bound."); + } + + std::vector workspace_shape = + hierarchy.level_shape(hierarchy.l_target()); + for (DIM d = 0; d < D; d++) + workspace_shape[d] += 2; + Array workspace(workspace_shape); + SubArray w(workspace); + + if (singledim_refactoring_debug_print) { + PrintSubarray("Input", v); + } + + for (int l = start_level; l > stop_level; l--) { + for (int curr_dim = D - 1; curr_dim >= 0; curr_dim--) { + if (singledim_refactoring_debug_print) { + std::cout << "l: " << l << " curr_dim: " << curr_dim << "\n"; + } + std::vector fine_shape(D); + std::vector coarse_shape(D); + std::vector coeff_shape(D); + for (int d = D - 1; d >= 0; d--) { + if (d > curr_dim) { + fine_shape[d] = hierarchy.level_shape(l - 1, d); + } else { + fine_shape[d] = hierarchy.level_shape(l, d); + } + } + + for (int d = D - 1; d >= 0; d--) { + if (d == curr_dim) { + coarse_shape[d] = hierarchy.level_shape(l - 1, d); + coeff_shape[d] = fine_shape[d] - hierarchy.level_shape(l - 1, d); + } else { + coarse_shape[d] = fine_shape[d]; + coeff_shape[d] = fine_shape[d]; + } + } + + SubArray v_fine = v; + v_fine.resize(fine_shape); + SubArray w_fine = w; + w_fine.resize(fine_shape); + SubArray coarse = v; + coarse.resize(coarse_shape); + SubArray coeff = v; + coeff.offset_dim(curr_dim, hierarchy.level_shape(l - 1, curr_dim)); + coeff.resize(coeff_shape); + SubArray correction = w; + correction.resize(coarse_shape); + + multi_dimension::CopyND(v_fine, w_fine, queue_idx); + + if (singledim_refactoring_debug_print) { + PrintSubarray("COPY", w_fine); + } + + CalcCoefficients(curr_dim, SubArray(hierarchy.ratio(l, curr_dim)), w_fine, + coarse, coeff, queue_idx); + + if (singledim_refactoring_debug_print) { + PrintSubarray("SingleDimensionCoefficient - fine", w_fine); + PrintSubarray("SingleDimensionCoefficient - corase", coarse); + PrintSubarray("SingleDimensionCoefficient - coeff", coeff); + } + + CalcCorrection(hierarchy, coeff, correction, curr_dim, l, queue_idx); + + multi_dimension::AddND(correction, coarse, queue_idx); + + if (singledim_refactoring_debug_print) { + PrintSubarray("ADD", coarse); + } + + } // loop dimensions + } // loop levels +} + +template +void recompose(Hierarchy &hierarchy, + SubArray &v, int start_level, int stop_level, + int queue_idx) { + + if (stop_level < 0 || stop_level > hierarchy.l_target()) { + throw std::runtime_error("recompose: stop_level out of bound."); + } + + std::vector workspace_shape = + hierarchy.level_shape(hierarchy.l_target()); + for (DIM d = 0; d < D; d++) + workspace_shape[d] += 2; + Array workspace(workspace_shape); + SubArray w(workspace); + + if (singledim_refactoring_debug_print) { + PrintSubarray("Input", v); + } + + for (int l = start_level; l < stop_level; l++) { + for (int curr_dim = 0; curr_dim < D; curr_dim++) { + if (singledim_refactoring_debug_print) { + std::cout << "l: " << l << " curr_dim: " << curr_dim << "\n"; + } + std::vector fine_shape(D); + std::vector coarse_shape(D); + std::vector coeff_shape(D); + for (int d = D - 1; d >= 0; d--) { + if (d > curr_dim) { + fine_shape[d] = hierarchy.level_shape(l, d); + } else { + fine_shape[d] = hierarchy.level_shape(l + 1, d); + } + } + + for (int d = D - 1; d >= 0; d--) { + if (d == curr_dim) { + coarse_shape[d] = hierarchy.level_shape(l, d); + coeff_shape[d] = fine_shape[d] - hierarchy.level_shape(l, d); + } else { + coarse_shape[d] = fine_shape[d]; + coeff_shape[d] = fine_shape[d]; + } + } + + SubArray v_fine = v; + v_fine.resize(fine_shape); + SubArray w_fine = w; + w_fine.resize(fine_shape); + SubArray coarse = v; + coarse.resize(coarse_shape); + SubArray coeff = v; + coeff.offset_dim(curr_dim, hierarchy.level_shape(l, curr_dim)); + coeff.resize(coeff_shape); + SubArray correction = w; + correction.resize(coarse_shape); + + CalcCorrection(hierarchy, coeff, correction, curr_dim, l + 1, queue_idx); + + multi_dimension::SubtractND(correction, coarse, queue_idx); + + if (singledim_refactoring_debug_print) { + PrintSubarray("SUBTRACT", coarse); + } + + CoefficientsRestore(curr_dim, SubArray(hierarchy.ratio(l + 1, curr_dim)), + w_fine, coarse, coeff, queue_idx); + + if (singledim_refactoring_debug_print) { + PrintSubarray("SingleDimensionCoefficient - fine", w_fine); + PrintSubarray("SingleDimensionCoefficient - corase", coarse); + PrintSubarray("SingleDimensionCoefficient - coeff", coeff); + } + + multi_dimension::CopyND(w_fine, v_fine, queue_idx); + + if (singledim_refactoring_debug_print) { + PrintSubarray("COPY", v_fine); + } + + } // loop dimensions + } // loop levels +} + +} // namespace single_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 diff --git a/include/mgard-x-160/DomainDecomposer/DomainDecomposer.hpp b/include/mgard-x-160/DomainDecomposer/DomainDecomposer.hpp new file mode 100644 index 000000000..79655a337 --- /dev/null +++ b/include/mgard-x-160/DomainDecomposer/DomainDecomposer.hpp @@ -0,0 +1,855 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_DOMAIN_DECOMPOSER_HPP +#define MGARD_X_160_DOMAIN_DECOMPOSER_HPP + +#include "../Config/Config.h" +#include "../Hierarchy/Hierarchy.hpp" + +namespace mgard_x_160 { + +enum class subdomain_copy_direction : uint8_t { + OriginalToSubdomain, + SubdomainToOriginal +}; + +template +class DomainDecomposer { +public: + size_t EstimateMemoryFootprint(std::vector shape, + bool enable_prefetch) { + size_t estimate_memory_usgae = 0; + + Array<1, T, DeviceType> array_with_pitch({1}); + size_t pitch_size = array_with_pitch.ld(0) * sizeof(T); + + size_t input_space = roundup((size_t)shape[D - 1] * sizeof(T), pitch_size); + for (DIM d = 0; d < D - 1; d++) { + input_space *= shape[d]; + } + + SIZE num_elements = 1; + for (int i = 0; i < shape.size(); i++) + num_elements *= shape[i]; + size_t output_space = 0; + output_space += num_elements * sizeof(HUFFMAN_CODE); + output_space += config.estimate_outlier_ratio * sizeof(QUANTIZED_INT); + + estimate_memory_usgae = input_space + output_space; + + log::dbg("Input output space: " + + std::to_string((double)(input_space + output_space) / 1e9) + + " GB"); + + using HierarchyType = typename OperatorType::HierarchyType; + size_t hierarchy_space = + HierarchyType::EstimateMemoryFootprint(shape, config); + estimate_memory_usgae += hierarchy_space; + log::dbg("Hierarchy space: " + + std::to_string((double)hierarchy_space / 1e9) + " GB"); + + // For prefetching + if (enable_prefetch) { + estimate_memory_usgae *= 2; + } + estimate_memory_usgae += + OperatorType::EstimateMemoryFootprint(shape, config); + log::dbg("Operation space: " + + std::to_string( + (double)OperatorType::EstimateMemoryFootprint(shape, config) / + 1e9) + + " GB"); + + return estimate_memory_usgae; + } + + bool need_domain_decomposition(std::vector shape, + bool enable_prefetch) { + // using Cache = CompressorCache; + size_t estm = EstimateMemoryFootprint(shape, enable_prefetch); + size_t aval = + std::min((SIZE)DeviceRuntime::GetAvailableMemory(), + config.max_memory_footprint); + log::dbg("Estimated memory usage: " + std::to_string((double)estm / 1e9) + + "GB, Available: " + std::to_string((double)aval / 1e9) + "GB"); + bool need = estm >= aval; + return need; + } + + std::vector dim_num_subdomain() { + std::vector _dim_num_subdomain(D, 1); + if (config.domain_decomposition == domain_decomposition_type::MaxDim || + config.domain_decomposition == domain_decomposition_type::Variable) { + _dim_num_subdomain[_domain_decomposed_dim] = this->_num_subdomains; + } else if (config.domain_decomposition == + domain_decomposition_type::Block) { + for (int d = D - 1; d >= 0; d--) { + _dim_num_subdomain[d] = (shape[d] - 1) / _domain_decomposed_size + 1; + } + } + return _dim_num_subdomain; + } + + std::vector dim_subdomain_id(int subdomain_id) { + std::vector _dim_num_subdomain = dim_num_subdomain(); + std::vector _dim_subdomain_id(D); + for (int d = D - 1; d >= 0; d--) { + _dim_subdomain_id[d] = subdomain_id % _dim_num_subdomain[d]; + subdomain_id /= _dim_num_subdomain[d]; + } + return _dim_subdomain_id; + } + + std::vector dim_subdomain_offset(int subdomain_id) { + std::vector _dim_subdomain_id = dim_subdomain_id(subdomain_id); + std::vector _dim_subdomain_offset(D); + for (int d = D - 1; d >= 0; d--) { + _dim_subdomain_offset[d] = _dim_subdomain_id[d] * _domain_decomposed_size; + } + return _dim_subdomain_offset; + } + + std::vector subdomain_shape(int subdomain_id) { + if (subdomain_id >= _num_subdomains) { + throw std::runtime_error( + "DomainDecomposer: subdomain_shape wrong subdomain_id."); + } + if (!_domain_decomposed) { + return shape; + } else { + if (config.domain_decomposition == domain_decomposition_type::MaxDim) { + if (subdomain_id < + shape[_domain_decomposed_dim] / _domain_decomposed_size) { + std::vector chunk_shape = shape; + chunk_shape[_domain_decomposed_dim] = _domain_decomposed_size; + return chunk_shape; + } else { + SIZE leftover_dim_size = + shape[_domain_decomposed_dim] % _domain_decomposed_size; + std::vector leftover_shape = shape; + leftover_shape[_domain_decomposed_dim] = leftover_dim_size; + return leftover_shape; + } + } else if (config.domain_decomposition == + domain_decomposition_type::Block) { + std::vector chunk_shape(D); + std::vector _dim_subdomain_id = dim_subdomain_id(subdomain_id); + for (int d = D - 1; d >= 0; d--) { + if (_dim_subdomain_id[d] < shape[d] / _domain_decomposed_size) { + chunk_shape[d] = _domain_decomposed_size; + } else { + chunk_shape[d] = shape[d] % _domain_decomposed_size; + } + } + return chunk_shape; + } else if (config.domain_decomposition == + domain_decomposition_type::Variable) { + std::vector chunk_shape = shape; + chunk_shape[_domain_decomposed_dim] = + _domain_decomposed_sizes[subdomain_id]; + return chunk_shape; + } else { + throw std::runtime_error("Wrong domain decomposition type."); + return shape; + } + } + } + + int largest_subdomain() { + SIZE max_count = 0; + int max_subdomain = 0; + for (int id = 0; id < this->_num_subdomains; id++) { + SIZE count = 1; + std::vector chunk_shape = subdomain_shape(id); + for (int i = 0; i < chunk_shape.size(); i++) { + count *= chunk_shape[i]; + } + if (count > max_count) { + max_count = count; + max_subdomain = id; + } + } + return max_subdomain; + } + + SIZE subdomain_compressed_buffer_size(int subdomain_id) { + std::vector shape = subdomain_shape(subdomain_id); + SIZE num_elements = 1; + for (int i = 0; i < shape.size(); i++) + num_elements *= shape[i]; + SIZE size = 0; + size += num_elements * sizeof(HUFFMAN_CODE); + size += config.estimate_outlier_ratio * sizeof(QUANTIZED_INT); + return size; + } + + bool generate_max_dim_domain_decomposition_strategy( + std::vector shape, DIM &_domain_decomposed_dim, + SIZE &_domain_decomposed_size) { + // determine max dimension + SIZE max_dim = 0; + for (DIM d = 0; d < D; d++) { + if (shape[d] > max_dim) { + max_dim = shape[d]; + _domain_decomposed_dim = d; + } + } + + // domain decomposition strategy + std::vector chunk_shape = shape; + + SIZE curr_num_subdomains = 1; + + bool need_prefetch = false; + // Then check if each chunk can fit into device memory + while (need_domain_decomposition(chunk_shape, need_prefetch)) { + // Divide by 2 and round up + chunk_shape[_domain_decomposed_dim] = + (chunk_shape[_domain_decomposed_dim] - 1) / 2 + 1; + + curr_num_subdomains = (shape[_domain_decomposed_dim] - 1) / + chunk_shape[_domain_decomposed_dim] + + 1; + need_prefetch = curr_num_subdomains > 1; + } + _domain_decomposed_size = chunk_shape[_domain_decomposed_dim]; + return true; + } + + bool + generate_block_domain_decomposition_strategy(std::vector shape, + SIZE &_domain_decomposed_size) { + std::vector chunk_shape(D, _domain_decomposed_size); + + int curr_num_subdomains = 1; + for (DIM d = 0; d < D; d++) { + curr_num_subdomains *= (shape[d] - 1) / _domain_decomposed_size + 1; + } + + // Need prefetch if there are more subdomains than devices + bool need_prefetch = curr_num_subdomains > 1; + // Then check if each chunk can fit into device memory + while (need_domain_decomposition(chunk_shape, need_prefetch)) { + // Divide by 2 and round up + _domain_decomposed_size = (_domain_decomposed_size - 1) / 2 + 1; + chunk_shape = std::vector(D, _domain_decomposed_size); + + curr_num_subdomains = 1; + for (DIM d = 0; d < D; d++) { + curr_num_subdomains *= (shape[d] - 1) / _domain_decomposed_size + 1; + } + + need_prefetch = curr_num_subdomains > 1; + } + return true; + } + + typename OperatorType::HierarchyType subdomain_hierarchy(int subdomain_id) { + using HierarchyType = typename OperatorType::HierarchyType; + if (uniform) { + return HierarchyType(subdomain_shape(subdomain_id), config); + } else { + if (config.domain_decomposition == domain_decomposition_type::MaxDim || + config.domain_decomposition == domain_decomposition_type::Variable) { + std::vector chunk_coords = coords; + std::vector chunk_shape = subdomain_shape(subdomain_id); + T *decompose_dim_coord = new T[chunk_shape[_domain_decomposed_dim]]; + MemoryManager::Copy1D( + decompose_dim_coord, + coords[_domain_decomposed_dim] + + subdomain_id * _domain_decomposed_size, + chunk_shape[_domain_decomposed_dim], 0); + DeviceRuntime::SyncQueue(0); + chunk_coords[_domain_decomposed_dim] = decompose_dim_coord; + HierarchyType hierarchy(chunk_shape, chunk_coords, config); + delete[] decompose_dim_coord; + return hierarchy; + } else if (config.domain_decomposition == + domain_decomposition_type::Block) { + std::vector chunk_coords(D); + std::vector chunk_shape = subdomain_shape(subdomain_id); + std::vector _dim_subdomain_id = dim_subdomain_id(subdomain_id); + for (int d = D - 1; d >= 0; d--) { + chunk_coords[d] = new T[chunk_shape[d]]; + MemoryManager::Copy1D( + chunk_coords[d], + coords[d] + _dim_subdomain_id[d] * _domain_decomposed_size, + chunk_shape[d], 0); + DeviceRuntime::SyncQueue(0); + } + HierarchyType hierarchy(chunk_shape, chunk_coords, config); + for (int d = D - 1; d >= 0; d--) + delete[] chunk_coords[d]; + return hierarchy; + } else { + throw std::runtime_error("Wrong domain decomposition type."); + } + } + } + + DomainDecomposer() : _domain_decomposed(false) {} + + // Find domain decomposion method + DomainDecomposer(std::vector shape, Config config) + : original_data(nullptr), shape(shape), config(config), + keep_original_data_decomposed(false) { + if (!need_domain_decomposition(shape, false) && + config.domain_decomposition != domain_decomposition_type::Block && + config.domain_decomposition != domain_decomposition_type::Variable) { + this->_domain_decomposed = false; + this->_domain_decomposed_dim = 0; + this->_domain_decomposed_size = this->shape[0]; + this->_num_subdomains = 1; + log::info("DomainDecomposer: no decomposition used"); + } else { + this->_domain_decomposed = true; + if (config.domain_decomposition == domain_decomposition_type::MaxDim) { + generate_max_dim_domain_decomposition_strategy( + shape, this->_domain_decomposed_dim, this->_domain_decomposed_size); + this->_num_subdomains = (shape[this->_domain_decomposed_dim] - 1) / + this->_domain_decomposed_size + + 1; + log::info( + "Domain decomposed dim: " + std::to_string(_domain_decomposed_dim) + + ", Domain decomposed size: " + + std::to_string(_domain_decomposed_size)); + log::info("DomainDecomposer: decomposed into " + + std::to_string(this->_num_subdomains) + + " subdomains using MaxDim method"); + } else if (config.domain_decomposition == + domain_decomposition_type::Block) { + this->_domain_decomposed_size = config.block_size; + generate_block_domain_decomposition_strategy( + shape, this->_domain_decomposed_size); + this->_num_subdomains = 1; + for (DIM d = 0; d < D; d++) { + this->_num_subdomains *= + (shape[d] - 1) / this->_domain_decomposed_size + 1; + } + log::info("Domain decomposed size: " + + std::to_string(_domain_decomposed_size)); + log::info("DomainDecomposer: decomposed into " + + std::to_string(this->_num_subdomains) + + " subdomains using Block method"); + } else if (config.domain_decomposition == + domain_decomposition_type::Variable) { + this->_domain_decomposed_dim = config.domain_decomposition_dim; + this->_domain_decomposed_sizes = config.domain_decomposition_sizes; + this->_num_subdomains = this->_domain_decomposed_sizes.size(); + log::info("DomainDecomposer: decomposed into " + + std::to_string(this->_num_subdomains) + + " subdomains using Variable method"); + } else { + throw std::runtime_error("Wrong domain decomposition type."); + } + } + + uniform = true; + } + + // Find domain decomposion method + DomainDecomposer(std::vector shape, Config config, + std::vector coords) + : original_data(nullptr), shape(shape), config(config), coords(coords), + keep_original_data_decomposed(false) { + if (!need_domain_decomposition(shape, false) && + config.domain_decomposition != domain_decomposition_type::Block && + config.domain_decomposition != domain_decomposition_type::Variable) { + this->_domain_decomposed = false; + this->_domain_decomposed_dim = 0; + this->_domain_decomposed_size = this->shape[0]; + this->_num_subdomains = 1; + log::info("DomainDecomposer: no decomposition used"); + } else { + this->_domain_decomposed = true; + if (config.domain_decomposition == domain_decomposition_type::MaxDim) { + generate_max_dim_domain_decomposition_strategy( + shape, this->_domain_decomposed_dim, this->_domain_decomposed_size); + this->_num_subdomains = (shape[this->_domain_decomposed_dim] - 1) / + this->_domain_decomposed_size + + 1; + log::info( + "Domain decomposed dim: " + std::to_string(_domain_decomposed_dim) + + ", Domain decomposed size: " + + std::to_string(_domain_decomposed_size)); + log::info("DomainDecomposer: decomposed into " + + std::to_string(this->_num_subdomains) + + " subdomains using MaxDim method"); + } else if (config.domain_decomposition == + domain_decomposition_type::Block) { + this->_domain_decomposed_size = config.block_size; + generate_block_domain_decomposition_strategy( + shape, this->_domain_decomposed_size); + this->_num_subdomains = 1; + for (DIM d = 0; d < D; d++) { + this->_num_subdomains *= + (shape[d] - 1) / this->_domain_decomposed_size + 1; + } + log::info("Domain decomposed size: " + + std::to_string(_domain_decomposed_size)); + log::info("DomainDecomposer: decomposed into " + + std::to_string(this->_num_subdomains) + + " subdomains using Block method"); + } else if (config.domain_decomposition == + domain_decomposition_type::Variable) { + this->_domain_decomposed_dim = config.domain_decomposition_dim; + this->_domain_decomposed_sizes = config.domain_decomposition_sizes; + this->_num_subdomains = this->_domain_decomposed_sizes.size(); + log::info("DomainDecomposer: decomposed into " + + std::to_string(this->_num_subdomains) + + " subdomains using Variable method"); + } else { + throw std::runtime_error("Wrong domain decomposition type."); + } + } + + uniform = false; + } + + // Force to use this domain decomposion method + DomainDecomposer(std::vector shape, bool _domain_decomposed, + DIM _domain_decomposed_dim, SIZE _domain_decomposed_size, + Config config) + : original_data(nullptr), shape(shape), + _domain_decomposed_dim(_domain_decomposed_dim), + _domain_decomposed_size(_domain_decomposed_size), + _domain_decomposed(_domain_decomposed), config(config), + keep_original_data_decomposed(false) { + if (!this->_domain_decomposed) { + this->_domain_decomposed_dim = 0; + this->_domain_decomposed_size = this->shape[0]; + this->_num_subdomains = 1; + log::info("DomainDecomposer: no decomposition used"); + } else { + if (config.domain_decomposition == domain_decomposition_type::MaxDim) { + this->_num_subdomains = (shape[this->_domain_decomposed_dim] - 1) / + this->_domain_decomposed_size + + 1; + log::info( + "Domain decomposed dim: " + std::to_string(_domain_decomposed_dim) + + ", Domain decomposed size: " + + std::to_string(_domain_decomposed_size)); + log::info("DomainDecomposer: decomposed into " + + std::to_string(this->_num_subdomains) + + " subdomains using MaxDim method"); + } else if (config.domain_decomposition == + domain_decomposition_type::Block) { + this->_num_subdomains = 1; + for (DIM d = 0; d < D; d++) { + this->_num_subdomains *= + (shape[d] - 1) / this->_domain_decomposed_size + 1; + } + log::info("Domain decomposed size: " + + std::to_string(_domain_decomposed_size)); + log::info("DomainDecomposer: decomposed into " + + std::to_string(this->_num_subdomains) + + " subdomains using Block method"); + } else if (config.domain_decomposition == + domain_decomposition_type::Variable) { + this->_domain_decomposed_dim = config.domain_decomposition_dim; + this->_domain_decomposed_sizes = config.domain_decomposition_sizes; + this->_num_subdomains = this->_domain_decomposed_sizes.size(); + log::info("DomainDecomposer: decomposed into " + + std::to_string(this->_num_subdomains) + + " subdomains using Variable method"); + } else { + throw std::runtime_error("Wrong domain decomposition type."); + } + } + + uniform = true; + } + + // Force to use this domain decomposion method + DomainDecomposer(std::vector shape, bool _domain_decomposed, + DIM _domain_decomposed_dim, SIZE _domain_decomposed_size, + Config config, std::vector coords) + : original_data(nullptr), shape(shape), + _domain_decomposed_dim(_domain_decomposed_dim), + _domain_decomposed_size(_domain_decomposed_size), + _domain_decomposed(_domain_decomposed), config(config), coords(coords), + keep_original_data_decomposed(false) { + if (!this->_domain_decomposed) { + this->_domain_decomposed_dim = 0; + this->_domain_decomposed_size = this->shape[0]; + this->_num_subdomains = 1; + log::info("DomainDecomposer: no decomposition used"); + } else { + if (config.domain_decomposition == domain_decomposition_type::MaxDim) { + this->_num_subdomains = (shape[this->_domain_decomposed_dim] - 1) / + this->_domain_decomposed_size + + 1; + log::info( + "Domain decomposed dim: " + std::to_string(_domain_decomposed_dim) + + ", Domain decomposed size: " + + std::to_string(_domain_decomposed_size)); + log::info("DomainDecomposer: decomposed into " + + std::to_string(this->_num_subdomains) + + " subdomains using MaxDim method"); + } else if (config.domain_decomposition == + domain_decomposition_type::Block) { + this->_num_subdomains = 1; + for (DIM d = 0; d < D; d++) { + this->_num_subdomains *= + (shape[d] - 1) / this->_domain_decomposed_size + 1; + } + log::info("Domain decomposed size: " + + std::to_string(_domain_decomposed_size)); + log::info("DomainDecomposer: decomposed into " + + std::to_string(this->_num_subdomains) + + " subdomains using Block method"); + } else if (config.domain_decomposition == + domain_decomposition_type::Variable) { + this->_domain_decomposed_dim = config.domain_decomposition_dim; + this->_domain_decomposed_sizes = config.domain_decomposition_sizes; + this->_num_subdomains = this->_domain_decomposed_sizes.size(); + log::info("DomainDecomposer: decomposed into " + + std::to_string(this->_num_subdomains) + + " subdomains using Variable method"); + } else { + throw std::runtime_error("Wrong domain decomposition type."); + } + } + + uniform = false; + } + + // void calc_domain_decompose_parameter(SIZE subdomain_id, + // std::vector subdomain_shape, + // SIZE &subdomain_ld, SIZE &original_ld, + // SIZE &n1, SIZE &n2) { + // SIZE decomposed_size; + // if (subdomain_id < + // shape[_domain_decomposed_dim] / _domain_decomposed_size) { + // decomposed_size = _domain_decomposed_size; + // } else { + // decomposed_size = shape[_domain_decomposed_dim] % + // _domain_decomposed_size; + // } + + // std::vector original_shape = original_data_shape(subdomain_shape); + + // subdomain_ld = subdomain_shape[_domain_decomposed_dim]; // + // decomposed_size; for (int d = D - 1; d > (int)_domain_decomposed_dim; + // d--) { + // subdomain_ld *= subdomain_shape[d]; + // } + // // std::cout << "dst_ld: " << dst_ld << "\n"; + // original_ld = 1; + // for (int d = D - 1; d >= (int)_domain_decomposed_dim; d--) { + // original_ld *= original_shape[d]; + // } + // // std::cout << "src_ld: " << src_ld << "\n"; + // n1 = subdomain_shape[_domain_decomposed_dim]; + // for (int d = D - 1; d > (int)_domain_decomposed_dim; d--) { + // n1 *= subdomain_shape[d]; + // } + // // std::cout << "n1: " << n1 << "\n"; + // n2 = 1; + // for (int d = 0; d < (int)_domain_decomposed_dim; d++) { + // n2 *= subdomain_shape[d]; + // } + // // std::cout << "n2: " << n2 << "\n"; + // } + + SIZE calc_offset(std::vector shape, + std::vector dim_subdomain_offset) { + SIZE curr_stride = 1; + SIZE ret_idx = 0; + for (int d = D - 1; d >= 0; d--) { + ret_idx += dim_subdomain_offset[d] * curr_stride; + curr_stride *= shape[d]; + } + return ret_idx; + } + + bool check_shape(Array &subdomain_data, + std::vector shape) { + if (subdomain_data.data() == nullptr) { + return false; + } + for (DIM d = 0; d < D; d++) { + if (subdomain_data.shape(d) != shape[d]) { + return false; + } + } + return true; + } + + T *original_data_ptr(int subdomain_id) { + if (config.domain_decomposition == domain_decomposition_type::MaxDim || + config.domain_decomposition == domain_decomposition_type::Variable) { + if (!keep_original_data_decomposed) { + // Subdomains are slabs along _domain_decomposed_dim: subdomain i starts + // at the previous slabs' total extent along that dimension, times that + // dimension's stride. (Summing the previous subdomains' element counts, + // as before, is only right when it is the slowest dimension; otherwise + // copies ran past the end of the array.) + SIZE start = 0; + for (int i = 0; i < subdomain_id; i++) { + start += subdomain_shape(i)[_domain_decomposed_dim]; + } + SIZE stride = 1; + for (int d = D - 1; d > (int)_domain_decomposed_dim; d--) { + stride *= shape[d]; + } + return original_data + start * stride; + } else { + return decomposed_original_data[subdomain_id]; + } + } else if (config.domain_decomposition == + domain_decomposition_type::Block) { + if (!keep_original_data_decomposed) { + return original_data + + calc_offset(shape, dim_subdomain_offset(subdomain_id)); + } else { + return decomposed_original_data[subdomain_id]; + } + } else { + throw std::runtime_error("Wrong domain decomposition type."); + } + } + + std::vector original_data_shape(std::vector subdomain_shape) { + if (!keep_original_data_decomposed) { + return shape; + } else { + return subdomain_shape; + } + } + + void copy_subdomain(Array &subdomain_data, int subdomain_id, + enum subdomain_copy_direction direction, int queue_idx) { + if (subdomain_id >= _num_subdomains) { + throw std::runtime_error( + "DomainDecomposer::copy_subdomain wrong subdomain_id."); + } + + if (!_domain_decomposed) { + // if (keep_original_data_decomposed) { + // throw std::runtime_error("Do not support restoring to decomposed data + // when no domain decomposition was used."); + // } + + if (direction == subdomain_copy_direction::OriginalToSubdomain) { + subdomain_data.resize(shape); + } + SIZE linearized_width = 1; + for (DIM d = 0; d < D - 1; d++) + linearized_width *= subdomain_data.shape(d); + if (direction == subdomain_copy_direction::OriginalToSubdomain) { + MemoryManager::CopyND( + subdomain_data.data(), subdomain_data.ld(D - 1), original_data, + subdomain_data.shape(D - 1), subdomain_data.shape(D - 1), + linearized_width, queue_idx); + } else { + MemoryManager::CopyND( + original_data, subdomain_data.shape(D - 1), subdomain_data.data(), + subdomain_data.ld(D - 1), subdomain_data.shape(D - 1), + linearized_width, queue_idx); + } + } else { + if (config.domain_decomposition == domain_decomposition_type::MaxDim || + config.domain_decomposition == domain_decomposition_type::Variable) { + if (keep_original_data_decomposed) { + throw std::runtime_error( + "Do not support restoring to decomposed data when using " + "MaxDim or Variable"); + } + T *data = original_data_ptr(subdomain_id); + if (direction == subdomain_copy_direction::OriginalToSubdomain) { + subdomain_data.resize(subdomain_shape(subdomain_id), queue_idx); + } + SIZE subdomain_ld, original_ld, n1, n2; + // calc_domain_decompose_parameter(subdomain_id, subdomain_data.shape(), + // subdomain_ld, original_ld, n1, n2); SIZE decomposed_size; if + // (subdomain_id < + // shape[_domain_decomposed_dim] / _domain_decomposed_size) { + // decomposed_size = _domain_decomposed_size; + // } else { + // decomposed_size = shape[_domain_decomposed_dim] % + // _domain_decomposed_size; + // } + + std::vector original_shape = + original_data_shape(subdomain_data.shape()); + + subdomain_ld = subdomain_data.shape(_domain_decomposed_dim); + for (int d = D - 1; d > (int)_domain_decomposed_dim; d--) { + subdomain_ld *= subdomain_data.shape(d); + } + // std::cout << "subdomain_ld: " << subdomain_ld << "\n"; + original_ld = 1; + for (int d = D - 1; d >= (int)_domain_decomposed_dim; d--) { + original_ld *= original_shape[d]; + } + // std::cout << "original_ld: " << original_ld << "\n"; + n1 = subdomain_data.shape(_domain_decomposed_dim); + for (int d = D - 1; d > (int)_domain_decomposed_dim; d--) { + n1 *= subdomain_data.shape(d); + } + // std::cout << "n1: " << n1 << "\n"; + n2 = 1; + for (int d = 0; d < (int)_domain_decomposed_dim; d++) { + n2 *= subdomain_data.shape(d); + } + // std::cout << "n2: " << n2 << "\n"; + + if (direction == subdomain_copy_direction::OriginalToSubdomain) { + MemoryManager::CopyND(subdomain_data.data(), subdomain_ld, + data, original_ld, n1, n2, + queue_idx); + } else { + MemoryManager::CopyND(data, original_ld, + subdomain_data.data(), subdomain_ld, + n1, n2, queue_idx); + } + } else if (config.domain_decomposition == + domain_decomposition_type::Block) { + std::vector chunk_shape = subdomain_shape(subdomain_id); + if (direction == subdomain_copy_direction::OriginalToSubdomain) { + subdomain_data.resize(chunk_shape, queue_idx); + } + + T *data = original_data_ptr(subdomain_id); + std::vector shape = original_data_shape(subdomain_data.shape()); + + if (D == 1) { + if (direction == subdomain_copy_direction::OriginalToSubdomain) { + MemoryManager::CopyND( + subdomain_data.data(), subdomain_data.ld(D - 1), data, + shape[D - 1], subdomain_data.shape(D - 1), 1, queue_idx); + } else { + MemoryManager::CopyND( + data, shape[D - 1], subdomain_data.data(), + subdomain_data.ld(D - 1), subdomain_data.shape(D - 1), 1, + queue_idx); + } + } else if (D == 2) { + if (direction == subdomain_copy_direction::OriginalToSubdomain) { + MemoryManager::CopyND( + subdomain_data.data(), subdomain_data.ld(D - 1), data, + shape[D - 1], subdomain_data.shape(D - 1), + subdomain_data.shape(D - 2), queue_idx); + } else { + MemoryManager::CopyND( + data, shape[D - 1], subdomain_data.data(), + subdomain_data.ld(D - 1), subdomain_data.shape(D - 1), + subdomain_data.shape(D - 2), queue_idx); + } + } else if (D == 3) { + for (SIZE i = 0; i < subdomain_data.shape(D - 3); i++) { + if (direction == subdomain_copy_direction::OriginalToSubdomain) { + MemoryManager::CopyND( + subdomain_data.data() + + calc_offset(subdomain_data.shape(), {i, 0, 0}), + subdomain_data.ld(D - 1), + data + calc_offset(shape, {i, 0, 0}), shape[D - 1], + subdomain_data.shape(D - 1), subdomain_data.shape(D - 2), + queue_idx); + } else { + MemoryManager::CopyND( + data + calc_offset(shape, {i, 0, 0}), shape[D - 1], + subdomain_data.data() + + calc_offset(subdomain_data.shape(), {i, 0, 0}), + subdomain_data.ld(D - 1), subdomain_data.shape(D - 1), + subdomain_data.shape(D - 2), queue_idx); + } + } + } else if (D == 4) { + for (SIZE j = 0; j < subdomain_data.shape(D - 4); j++) { + for (SIZE i = 0; i < subdomain_data.shape(D - 3); i++) { + if (direction == subdomain_copy_direction::OriginalToSubdomain) { + MemoryManager::CopyND( + subdomain_data.data() + + calc_offset(subdomain_data.shape(), {j, i, 0, 0}), + subdomain_data.ld(D - 1), + data + calc_offset(shape, {j, i, 0, 0}), shape[D - 1], + subdomain_data.shape(D - 1), subdomain_data.shape(D - 2), + queue_idx); + } else { + MemoryManager::CopyND( + data + calc_offset(shape, {j, i, 0, 0}), shape[D - 1], + subdomain_data.data() + + calc_offset(subdomain_data.shape(), {j, i, 0, 0}), + subdomain_data.ld(D - 1), subdomain_data.shape(D - 1), + subdomain_data.shape(D - 2), queue_idx); + } + } + } + } else if (D == 5) { + for (SIZE k = 0; k < subdomain_data.shape(D - 5); k++) { + for (SIZE j = 0; j < subdomain_data.shape(D - 4); j++) { + for (SIZE i = 0; i < subdomain_data.shape(D - 3); i++) { + if (direction == + subdomain_copy_direction::OriginalToSubdomain) { + MemoryManager::CopyND( + subdomain_data.data() + + calc_offset(subdomain_data.shape(), {k, j, i, 0, 0}), + subdomain_data.ld(D - 1), + data + calc_offset(shape, {k, j, i, 0, 0}), shape[D - 1], + subdomain_data.shape(D - 1), subdomain_data.shape(D - 2), + queue_idx); + } else { + MemoryManager::CopyND( + data + calc_offset(shape, {k, j, i, 0, 0}), shape[D - 1], + subdomain_data.data() + + calc_offset(subdomain_data.shape(), {k, j, i, 0, 0}), + subdomain_data.ld(D - 1), subdomain_data.shape(D - 1), + subdomain_data.shape(D - 2), queue_idx); + } + } + } + } + } else { + throw std::runtime_error( + "Copy subdomain does not support higher than 5D data."); + } + } else { + throw std::runtime_error("Wrong domain decomposition type."); + } + } + } + + bool domain_decomposed() { return _domain_decomposed; } + + DIM domain_decomposed_dim() { return _domain_decomposed_dim; } + + SIZE domain_decomposed_size() { return _domain_decomposed_size; } + + SIZE num_subdomains() { return _num_subdomains; } + + void set_original_data(T *original_data) { + this->original_data = original_data; + keep_original_data_decomposed = false; + } + + void set_decomposed_original_data(std::vector decomposed_original_data) { + this->decomposed_original_data = decomposed_original_data; + original_data = decomposed_original_data[0]; + keep_original_data_decomposed = true; + } + + std::vector shape; + bool _domain_decomposed; + DIM _domain_decomposed_dim; + SIZE _domain_decomposed_size; + std::vector _domain_decomposed_sizes; + SIZE _num_subdomains; + std::vector> subdomain_hierarchies; + T *original_data; + std::vector decomposed_original_data; + bool keep_original_data_decomposed; + Config config; + bool uniform; + std::vector coords; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Hierarchy/Hierarchy.h b/include/mgard-x-160/Hierarchy/Hierarchy.h new file mode 100644 index 000000000..af8dfea61 --- /dev/null +++ b/include/mgard-x-160/Hierarchy/Hierarchy.h @@ -0,0 +1,129 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../Config/Config.h" +#include "../RuntimeX/RuntimeX.h" +#include "../Utilities/Types.h" + +#ifndef MGARD_X_160_HIERARCHY_H +#define MGARD_X_160_HIERARCHY_H + +namespace mgard_x_160 { + +template struct Hierarchy { + + /* for general users */ + Hierarchy(std::vector shape, Config config); + Hierarchy(std::vector shape, std::vector coords, Config config); + + /* for Internal use only */ + Hierarchy(); + Hierarchy(const Hierarchy &hierarchy); + // Moves hand over the device arrays (a moved-from Array is left empty), so + // e.g. inserting into a hierarchy cache does not deep-copy every array. + Hierarchy(Hierarchy &&hierarchy) = default; + Hierarchy &operator=(const Hierarchy &hierarchy) = default; + Hierarchy &operator=(Hierarchy &&hierarchy) = default; + + SIZE total_num_elems(); + SIZE level_num_elems(SIZE level); + std::vector level_num_elems(); + SIZE linearized_width(); + SIZE l_target(); + std::vector level_shape(SIZE level); + SIZE level_shape(SIZE level, DIM dim); + Array<1, SIZE, DeviceType> &level_shape_array(SIZE level); + Array<1, T, DeviceType> &dist(SIZE level, DIM dim); + Array<1, T, DeviceType> &ratio(SIZE level, DIM dim); + Array<1, T, DeviceType> &am(SIZE level, DIM dim); + Array<1, T, DeviceType> &bm(SIZE level, DIM dim); + Array<1, DIM, DeviceType> &processed(SIZE idx, DIM &processed_n); + Array<1, DIM, DeviceType> &unprocessed(SIZE idx, DIM &processed_n); + Array<2, SIZE, DeviceType> &level_ranges(); + Array<2, int, DeviceType> &level_marks(); + Array<1, T, DeviceType> &level_volumes(bool reciprocal); + Array<2, SIZE, DeviceType> &level_volume_offsets(); + data_structure_type data_structure(); + bool is_initialized(); + bool can_reuse(std::vector shape); + // Whether a cached hierarchy is still valid for `config`. A plain hierarchy + // does not depend on it; HybridHierarchy hides this to compare its level + // counts. + bool matches_config(Config config) { return true; } + size_t EstimateMemoryFootprint(std::vector shape); + static size_t EstimateMemoryFootprint(std::vector shape, + Config config); + + ~Hierarchy(); + +protected: + // Describe only the input grid: one level equal to `shape`, so l_target(), + // level_shape() and total_num_elems() work, but no device arrays (coords, + // dist/ratio, am/bm, volumes, ranges/marks) are built. For hierarchies such + // as HybridHierarchy whose decomposition does not use them. + void init_shape_only(std::vector shape, data_structure_type dstype); + +private: + // Shape of the finest grid + std::vector shape; + // Pre-computed varaibles + SIZE _total_num_elems; + std::vector _level_num_elems; + SIZE _linearized_width; + // For out-of-bound returns + Array<1, T, DeviceType> dummy_array; + // Target level: number of times of decomposition + // Total number of levels = l_target+1 + SIZE _l_target; + // Shape of grid of each level + std::vector> _level_shape; + std::vector> _level_shape_array; + // Coordinates + std::vector> _coords_org; + // Pre-computed cell distance array + std::vector>> _dist_array; + // Pre-computed neighbor cell distance ratio array + std::vector>> _ratio_array; + // Pre-computed coefficients for solving tri-diag system + std::vector>> _am_array; + std::vector>> _bm_array; + // Pre-computed markers for processing high dimensional data + DIM _processed_n[D]; + DIM _unprocessed_n[D]; + Array<1, DIM, DeviceType> _processed_dims[D]; + Array<1, DIM, DeviceType> _unprocessed_dims[D]; + // Pre-computed range array for fast quantization + Array<2, SIZE, DeviceType> _level_ranges; + Array<2, int, DeviceType> _level_marks; + // Pre-computed volume array for fast quantization. Stored per level: the + // level_shape[l][d] volumes of each (level l, dim d) are packed back to + // back starting at _level_volume_offsets(l, d), so the total is + // sum_l sum_d level_shape[l][d] rather than (l_target + 1) * D * max_width + // (which in 1D is ~log2(N) * N). + Array<1, T, DeviceType> _level_volumes; + Array<1, T, DeviceType> _level_volumes_reciprocal; + Array<2, SIZE, DeviceType> _level_volume_offsets; + // Indicating if it is uniform or non-uniform grid + enum data_structure_type dstype; + + std::vector create_uniform_coords(std::vector shape, + bool normalize_coordinates); + void coord_to_dist(SIZE dof, T *coord, T *dist); + void dist_to_ratio(SIZE dof, T *dist, T *ratio); + void reduce_dist(SIZE dof, T *dist, T *dist2); + void calc_am_bm(SIZE dof, T *dist, T *am, T *bm); + void calc_volume(SIZE dof, T *dist, T *volume, bool reciprocal); + void init(std::vector shape, std::vector coords, + SIZE target_level = 0); + void destroy(); + bool uniform_coords_created = false; + bool initialized = false; +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Hierarchy/Hierarchy.hpp b/include/mgard-x-160/Hierarchy/Hierarchy.hpp new file mode 100644 index 000000000..dfbab7b58 --- /dev/null +++ b/include/mgard-x-160/Hierarchy/Hierarchy.hpp @@ -0,0 +1,874 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../RuntimeX/RuntimeX.h" +#include "Hierarchy.h" + +#include +#include +#include +#include +#include + +#ifndef MGARD_X_160_HIERARCHY_HPP +#define MGARD_X_160_HIERARCHY_HPP + +namespace mgard_x_160 { + +// MGARD 1.6.0 version, unchanged: the decomposition is rebuilt with exactly +// the geometry 1.6.0 compressed with. +template +void Hierarchy::coord_to_dist(SIZE dof, T *coord, T *dist) { + if (dof <= 1) + return; + // printf("coord_to_dist\n"); + T *h_coord = new T[dof]; + T *h_dist = new T[dof]; + for (int i = 0; i < dof; i++) + h_dist[i] = 0.0; + // cudaMemcpyAsyncHelper(*this, h_coord, coord, dof * sizeof(T), AUTO, 0); + MemoryManager::Copy1D(h_coord, coord, dof, 0); + DeviceRuntime::SyncQueue(0); + // this->sync(0); + for (int i = 0; i < dof - 1; i++) { + h_dist[i] = h_coord[i + 1] - h_coord[i]; + } + if (dof != 2 && dof % 2 == 0) { + T last_dist = h_dist[dof - 2]; + h_dist[dof - 2] = last_dist / 2.0; + h_dist[dof - 1] = last_dist / 2.0; + } + // cudaMemcpyAsyncHelper(*this, dist, h_dist, dof * sizeof(T), AUTO, 0); + // this->sync(0); + MemoryManager::Copy1D(dist, h_dist, dof, 0); + DeviceRuntime::SyncQueue(0); + + delete[] h_coord; + delete[] h_dist; +} + +// MGARD 1.6.0 version, unchanged: the decomposition is rebuilt with exactly +// the geometry 1.6.0 compressed with. +template +void Hierarchy::dist_to_ratio(SIZE dof, T *dist, T *ratio) { + if (dof <= 1) + return; + // printf("dist_to_ratio %llu\n", dof); + T *h_dist = new T[dof]; + T *h_ratio = new T[dof]; + for (int i = 0; i < dof; i++) + h_ratio[i] = 0.0; + // cudaMemcpyAsyncHelper(*this, h_dist, dist, dof * sizeof(T), AUTO, 0); + // this->sync(0); + MemoryManager::Copy1D(h_dist, dist, dof, 0); + DeviceRuntime::SyncQueue(0); + for (int i = 0; i < dof - 2; i++) { + h_ratio[i] = h_dist[i] / (h_dist[i + 1] + h_dist[i]); + // printf("dof: %llu ratio: %f\n", dof, h_ratio[i]); + } + if (dof % 2 == 0) { + h_ratio[dof - 2] = h_dist[dof - 2] / (h_dist[dof - 1] + h_dist[dof - 2]); + // printf("dof: %llu ratio: %f\n", dof, h_ratio[dof - 2]); + } + // cudaMemcpyAsyncHelper(*this, ratio, h_ratio, dof * sizeof(T), AUTO, 0); + // this->sync(0); + MemoryManager::Copy1D(ratio, h_ratio, dof, 0); + DeviceRuntime::SyncQueue(0); + delete[] h_dist; + delete[] h_ratio; +} + +// MGARD 1.6.0 version, unchanged: the decomposition is rebuilt with exactly +// the geometry 1.6.0 compressed with. +template +void Hierarchy::reduce_dist(SIZE dof, T *dist, T *dist2) { + if (dof <= 1) + return; + // printf("reduce_dist\n"); + SIZE dof2 = dof / 2 + 1; + T *h_dist = new T[dof]; + T *h_dist2 = new T[dof2]; + for (int i = 0; i < dof2; i++) + h_dist2[i] = 0.0; + // cudaMemcpyAsyncHelper(*this, h_dist, dist, dof * sizeof(T), AUTO, 0); + // this->sync(0); + MemoryManager::Copy1D(h_dist, dist, dof, 0); + DeviceRuntime::SyncQueue(0); + for (int i = 0; i < dof2 - 1; i++) { + h_dist2[i] = h_dist[i * 2] + h_dist[i * 2 + 1]; + } + if (dof2 != 2 && dof2 % 2 == 0) { + T last_dist = h_dist2[dof2 - 2]; + h_dist2[dof2 - 2] = last_dist / 2.0; + h_dist2[dof2 - 1] = last_dist / 2.0; + } + // cudaMemcpyAsyncHelper(*this, dist2, h_dist2, dof2 * sizeof(T), AUTO, 0); + // this->sync(0); + MemoryManager::Copy1D(dist2, h_dist2, dof2, 0); + DeviceRuntime::SyncQueue(0); + delete[] h_dist; + delete[] h_dist2; +} + +template +void Hierarchy::calc_am_bm(SIZE dof, T *dist, T *am, T *bm) { + T *h_dist = new T[dof]; + T *h_am = new T[dof + 1]; + T *h_bm = new T[dof + 1]; + for (int i = 0; i < dof + 1; i++) { + h_am[i] = 0.0; + h_bm[i] = 0.0; + } + // cudaMemcpyAsyncHelper(*this, h_dist, dist, dof * sizeof(T), AUTO, 0); + // this->sync(0); + MemoryManager::Copy1D(h_dist, dist, dof, 0); + DeviceRuntime::SyncQueue(0); + h_bm[0] = 2 * h_dist[0] / 6; + h_am[0] = 0.0; + + for (int i = 1; i < dof - 1; i++) { + T a_j = h_dist[i - 1] / 6; + T w = a_j / h_bm[i - 1]; + h_bm[i] = 2 * (h_dist[i - 1] + h_dist[i]) / 6 - w * a_j; + h_am[i] = a_j; + } + T a_j = h_dist[dof - 2] / 6; + T w = a_j / h_bm[dof - 2]; + h_bm[dof - 1] = 2 * h_dist[dof - 2] / 6 - w * a_j; + h_am[dof - 1] = a_j; +#ifdef MGARD_X_160_FMA + for (int i = 0; i < dof + 1; i++) { + h_am[i] = -1 * h_am[i]; + h_bm[i] = 1 / h_bm[i]; + } +#endif + // cudaMemcpyAsyncHelper(*this, am, h_am, dof * sizeof(T), AUTO, 0); + // cudaMemcpyAsyncHelper(*this, bm+1, h_bm, dof * sizeof(T), AUTO, 0); //add + // offset + + T one = 1; + // cudaMemcpyAsyncHelper(*this, bm, &one, sizeof(T), AUTO, 0); //add offset + T zero = 0; + // cudaMemcpyAsyncHelper(*this, am+dof, &zero, sizeof(T), AUTO, 0); + + MemoryManager::Copy1D(am, h_am, dof, 0); + MemoryManager::Copy1D(bm + 1, h_bm, dof, 0); + MemoryManager::Copy1D(bm, &one, 1, 0); + MemoryManager::Copy1D(am + dof, &zero, 1, 0); + DeviceRuntime::SyncQueue(0); + + // this->sync(0); + delete[] h_dist; + delete[] h_am; + delete[] h_bm; +} + +// MGARD 1.6.0 version, unchanged: the decomposition is rebuilt with exactly +// the geometry 1.6.0 compressed with. +template +void Hierarchy::calc_volume(SIZE dof, T *dist, T *volume, + bool reciprocal) { + T *h_dist = new T[dof]; + T *h_volume = new T[dof]; + for (int i = 0; i < dof; i++) { + h_volume[i] = 0.0; + } + MemoryManager::Copy1D(h_dist, dist, dof, 0); + DeviceRuntime::SyncQueue(0); + // level-wise uniform quantizer + if (dof > 1) { + for (int i = 0; i < dof; i++) { + h_volume[i] = 1.0 / (T)(dof - 1); + } + } + + if (reciprocal) { + for (int i = 0; i < dof; i++) { + h_volume[i] = 1.0 / h_volume[i]; + } + } + MemoryManager::Copy1D(volume, h_volume, dof, 0); + DeviceRuntime::SyncQueue(0); + delete[] h_dist; + delete[] h_volume; +} + +template +void Hierarchy::init(std::vector shape, + std::vector coords, + SIZE max_level) { + + this->shape = shape; + + std::vector> shape_level; + for (int d = 0; d < D; d++) { + std::vector curr_shape_level; + SIZE n = shape[d]; + while (n > 2) { + curr_shape_level.push_back(n); + n = n / 2 + 1; + } + curr_shape_level.push_back(2); + shape_level.push_back(curr_shape_level); + } + + SIZE nlevel = shape_level[0].size(); + for (DIM d = 1; d < D; d++) { + nlevel = std::min(nlevel, (SIZE)shape_level[d].size()); + } + + _l_target = nlevel - 1; + _l_target = std::min(_l_target, max_level); + + for (int l = 0; l < _l_target + 1; l++) { + std::vector curr_level_shape(D); + Array<1, SIZE, DeviceType> curr_level_shape_array({D}); + assert(shape_level.size() == D); + for (int d = 0; d < D; d++) { + curr_level_shape[d] = shape_level[d][_l_target - l]; + } + curr_level_shape_array.load(curr_level_shape.data()); + curr_level_shape_array.hostCopy(); + _level_shape.push_back(curr_level_shape); + _level_shape_array.push_back(curr_level_shape_array); + } + + { + _total_num_elems = 1; + for (int d = D - 1; d >= 0; d--) { + _total_num_elems *= _level_shape[_l_target][d]; + } + _linearized_width = 1; + for (int d = D - 2; d >= 0; d--) { + _linearized_width *= _level_shape[_l_target][d]; + } + } + + { // Ranges + SIZE *ranges_h_org = new SIZE[(_l_target + 2) * D]; + // beginning of the range + for (int d = 0; d < D; d++) { + ranges_h_org[d] = 0; + } + for (int l = 0; l < _l_target + 1; l++) { + for (int d = 0; d < D; d++) { + // Use l+1 bacause the first is reserved for 0 (beginning of the range) + ranges_h_org[(l + 1) * D + d] = _level_shape[l][d]; + } + } + bool pitched = false; + _level_ranges = Array<2, SIZE, DeviceType>({_l_target + 2, D}, pitched); + _level_ranges.load(ranges_h_org); + _level_ranges.hostCopy(); // keeping a copy on the host + delete[] ranges_h_org; + } + + { + SIZE max_width = 0; + for (DIM d = 0; d < D; d++) { + max_width = std::max(max_width, _level_shape[_l_target][d]); + } + int *_level_marks_h = new int[D * max_width]; + for (DIM d = 0; d < D; d++) { + int curr_level = 0; + int i = 0; + for (SIZE l = 0; l < _l_target + 1; l++) { + for (; i < _level_shape[l][d]; i++) { + _level_marks_h[d * max_width + i] = l; + } + } + } + bool pitched = false; + _level_marks = Array<2, int, DeviceType>({D, max_width}, pitched); + _level_marks.load(_level_marks_h); + // PrintSubarray("_level_marks", SubArray(_level_marks)); + delete[] _level_marks_h; + } + + { // Coords + for (int d = 0; d < D; d++) { + Array<1, T, DeviceType> coord({shape[d]}); + coord.load(coords[d]); + _coords_org.push_back(coord); + } + } + + { // calculate dist and ratio + _dist_array = + std::vector>>(_l_target + 1); + _ratio_array = + std::vector>>(_l_target + 1); + for (SIZE l = 0; l < _l_target + 1; l++) { + _dist_array[l] = std::vector>(D); + _ratio_array[l] = std::vector>(D); + } + // For the finest level: l = _l_target; + for (DIM d = 0; d < D; d++) { + _dist_array[_l_target][d] = + Array<1, T, DeviceType>({_level_shape[_l_target][d]}); + _ratio_array[_l_target][d] = + Array<1, T, DeviceType>({_level_shape[_l_target][d]}); + coord_to_dist(_level_shape[_l_target][d], this->_coords_org[d].data(), + _dist_array[_l_target][d].data()); + dist_to_ratio(_level_shape[_l_target][d], + _dist_array[_l_target][d].data(), + _ratio_array[_l_target][d].data()); + } + + // for l = 1 ... _l_target + for (int l = _l_target - 1; l >= 0; l--) { + for (DIM d = 0; d < D; d++) { + _dist_array[l][d] = Array<1, T, DeviceType>({_level_shape[l][d]}); + _ratio_array[l][d] = Array<1, T, DeviceType>({_level_shape[l][d]}); + + reduce_dist(_level_shape[l + 1][d], _dist_array[l + 1][d].data(), + _dist_array[l][d].data()); + dist_to_ratio(_level_shape[l][d], _dist_array[l][d].data(), + _ratio_array[l][d].data()); + } + } + } + + { // volume for quantization, packed per level (see Hierarchy.h) + std::vector offsets_h((_l_target + 1) * D); + SIZE total_volumes = 0; + for (SIZE l = 0; l < _l_target + 1; l++) { + for (DIM d = 0; d < D; d++) { + offsets_h[l * D + d] = total_volumes; + total_volumes += _level_shape[l][d]; + } + } + + bool pitched = false; + _level_volume_offsets = + Array<2, SIZE, DeviceType>({_l_target + 1, D}, pitched); + _level_volume_offsets.load(offsets_h.data()); + _level_volumes = Array<1, T, DeviceType>({total_volumes}, pitched); + _level_volumes_reciprocal = + Array<1, T, DeviceType>({total_volumes}, pitched); + for (SIZE l = 0; l < _l_target + 1; l++) { + for (DIM d = 0; d < D; d++) { + SIZE offset = offsets_h[l * D + d]; + calc_volume(_level_shape[l][d], _dist_array[l][d].data(), + _level_volumes.data() + offset, false); + calc_volume(_level_shape[l][d], _dist_array[l][d].data(), + _level_volumes_reciprocal.data() + offset, true); + } + } + } + + { // am and bm + _am_array = + std::vector>>(_l_target + 1); + _bm_array = + std::vector>>(_l_target + 1); + for (SIZE l = 0; l < _l_target + 1; l++) { + _am_array[l] = std::vector>(D); + _bm_array[l] = std::vector>(D); + } + for (SIZE l = 0; l < _l_target + 1; l++) { + for (DIM d = 0; d < D; d++) { + _am_array[l][d] = Array<1, T, DeviceType>({_level_shape[l][d] + 1}); + _bm_array[l][d] = Array<1, T, DeviceType>({_level_shape[l][d] + 1}); + _am_array[l][d].memset(0); + _bm_array[l][d].memset(0); + calc_am_bm(_level_shape[l][d], _dist_array[l][d].data(), + _am_array[l][d].data(), _bm_array[l][d].data()); + } + } + } + + if (D >= 4) { + std::vector tmp(0); + for (int d = D - 1; d >= 0; d--) { + _processed_n[D - 1 - d] = tmp.size(); + _processed_dims[D - 1 - d] = + Array<1, DIM, DeviceType>({(SIZE)tmp.size()}); + _processed_dims[D - 1 - d].load(tmp.data()); + _processed_dims[D - 1 - d].hostCopy(); // keep a host copy + tmp.push_back(d); + } + } + + if (D >= 4) { + std::vector tmp(0); + for (int d = 0; d < D - 3; d++) { + tmp.push_back(d); + } + // Extra padding needed in for loop below. + tmp.push_back(0); + + //+1 is used for storing empty status + for (int d = 0; d < (int)D - 3 + 1; d++) { + tmp.pop_back(); + _unprocessed_n[d] = tmp.size(); + _unprocessed_dims[d] = Array<1, DIM, DeviceType>({(SIZE)tmp.size()}); + _unprocessed_dims[d].load(tmp.data()); + } + } + + { + _level_num_elems.resize(_l_target + 1); + SIZE prev_num_elems = 0; + for (int level_idx = 0; level_idx < _l_target + 1; level_idx++) { + SIZE curr_num_elems = 1; + for (DIM d = 0; d < D; d++) { + curr_num_elems *= level_shape(level_idx, d); + } + _level_num_elems[level_idx] = curr_num_elems - prev_num_elems; + prev_num_elems = curr_num_elems; + } + } + + dummy_array = Array<1, T, DeviceType>({1}); + + initialized = true; +} + +template +size_t +Hierarchy::EstimateMemoryFootprint(std::vector shape) { + + size_t estimate_memory_usgae = 0; + Array<1, T, DeviceType> array_with_pitch({1}); + SIZE pitch_size = array_with_pitch.ld(0) * sizeof(T); + + this->shape = shape; + std::vector> shape_level; + for (int d = 0; d < D; d++) { + std::vector curr_shape_level; + SIZE n = shape[d]; + while (n > 2) { + curr_shape_level.push_back(n); + n = n / 2 + 1; + } + curr_shape_level.push_back(2); + shape_level.push_back(curr_shape_level); + } + + SIZE nlevel = shape_level[0].size(); + for (DIM d = 1; d < D; d++) { + nlevel = std::min(nlevel, (SIZE)shape_level[d].size()); + } + + _l_target = nlevel - 1; + + for (int l = 0; l < _l_target + 1; l++) { + std::vector curr_level_shape(D); + estimate_memory_usgae += roundup((SIZE)(D * sizeof(SIZE)), pitch_size); + assert(shape_level.size() == D); + for (int d = 0; d < D; d++) { + curr_level_shape[d] = shape_level[d][_l_target - l]; + } + _level_shape.push_back(curr_level_shape); + } + + { + _total_num_elems = 1; + for (int d = D - 1; d >= 0; d--) { + _total_num_elems *= _level_shape[_l_target][d]; + } + _linearized_width = 1; + for (int d = D - 2; d >= 0; d--) { + _linearized_width *= _level_shape[_l_target][d]; + } + } + + { // Ranges + estimate_memory_usgae += (_l_target + 2) * D * sizeof(SIZE); + } + + { // Coords + for (int d = 0; d < D; d++) { + estimate_memory_usgae += + roundup((SIZE)(shape[d] * sizeof(T)), pitch_size); + } + } + + { // calculate dist and ratio + for (DIM d = 0; d < D; d++) { + estimate_memory_usgae += + roundup((SIZE)(_level_shape[_l_target][d] * sizeof(T)), pitch_size); + estimate_memory_usgae += + roundup((SIZE)(_level_shape[_l_target][d] * sizeof(T)), pitch_size); + } + + // for l = 1 ... _l_target + for (int l = _l_target - 1; l >= 0; l--) { + for (DIM d = 0; d < D; d++) { + estimate_memory_usgae += + roundup((SIZE)(_level_shape[l][d] * sizeof(T)), pitch_size); + estimate_memory_usgae += + roundup((SIZE)(_level_shape[l][d] * sizeof(T)), pitch_size); + } + } + } + + { // volume for quantization, packed per level + size_t total_volumes = 0; + for (SIZE l = 0; l < _l_target + 1; l++) { + for (DIM d = 0; d < D; d++) { + total_volumes += _level_shape[l][d]; + } + } + estimate_memory_usgae += 2 * total_volumes * sizeof(T); + estimate_memory_usgae += (_l_target + 1) * D * sizeof(SIZE); + } + + { // am and bm + for (SIZE l = 0; l < _l_target + 1; l++) { + for (DIM d = 0; d < D; d++) { + estimate_memory_usgae += + roundup((SIZE)(_level_shape[l][d] * sizeof(T)), pitch_size); + estimate_memory_usgae += + roundup((SIZE)(_level_shape[l][d] * sizeof(T)), pitch_size); + } + } + } + + { + _level_num_elems.resize(_l_target + 1); + SIZE prev_num_elems = 0; + for (int level_idx = 0; level_idx < _l_target + 1; level_idx++) { + SIZE curr_num_elems = 1; + for (DIM d = 0; d < D; d++) { + curr_num_elems *= level_shape(level_idx, d); + } + _level_num_elems[level_idx] = curr_num_elems - prev_num_elems; + prev_num_elems = curr_num_elems; + } + } + + return estimate_memory_usgae; +} + +template +SIZE Hierarchy::total_num_elems() { + return _total_num_elems; +} + +template +SIZE Hierarchy::level_num_elems(SIZE level) { + if (level > _l_target + 1) { + throw std::runtime_error("Hierarchy::level_num_elems level out of bound."); + } + return _level_num_elems[level]; +} + +template +std::vector Hierarchy::level_num_elems() { + return _level_num_elems; +} + +template +SIZE Hierarchy::linearized_width() { + return _linearized_width; +} + +template +SIZE Hierarchy::l_target() { + return _l_target; +} + +template +std::vector Hierarchy::level_shape(SIZE level) { + if (level > _l_target + 1) { + throw std::runtime_error("Hierarchy::level_shape level out of bound."); + } + return _level_shape[level]; +} + +template +SIZE Hierarchy::level_shape(SIZE level, DIM dim) { + if (level > _l_target + 1) { + throw std::runtime_error("Hierarchy::level_shape level out of bound."); + } + if (dim >= D) + return 1; + return _level_shape[level][dim]; +} + +template +Array<1, SIZE, DeviceType> & +Hierarchy::level_shape_array(SIZE level) { + if (level > _l_target + 1) { + throw std::runtime_error( + "Hierarchy::level_shape_array level out of bound."); + } + return _level_shape_array[level]; +} + +template +Array<1, T, DeviceType> &Hierarchy::dist(SIZE level, + DIM dim) { + if (level > _l_target + 1) { + throw std::runtime_error("Hierarchy::dist level out of bound."); + } + if (dim >= D) + return dummy_array; + return _dist_array[level][dim]; +} + +template +Array<1, T, DeviceType> &Hierarchy::ratio(SIZE level, + DIM dim) { + if (level > _l_target + 1) { + throw std::runtime_error("Hierarchy::ratio level out of bound."); + } + if (dim >= D) + return dummy_array; + return _ratio_array[level][dim]; +} + +template +Array<1, T, DeviceType> &Hierarchy::am(SIZE level, DIM dim) { + if (level > _l_target + 1) { + throw std::runtime_error("Hierarchy::am level out of bound."); + } + if (dim >= D) + return dummy_array; + return _am_array[level][dim]; +} + +template +Array<1, T, DeviceType> &Hierarchy::bm(SIZE level, DIM dim) { + if (level > _l_target + 1) { + throw std::runtime_error("Hierarchy::bm level out of bound."); + } + if (dim >= D) + return dummy_array; + return _bm_array[level][dim]; +} + +template +Array<1, DIM, DeviceType> & +Hierarchy::processed(SIZE idx, DIM &processed_n) { + if (idx >= D) { + throw std::runtime_error("Hierarchy::processed idx out of bound."); + } + processed_n = _processed_n[idx]; + return _processed_dims[idx]; +} + +template +Array<1, DIM, DeviceType> & +Hierarchy::unprocessed(SIZE idx, DIM &processed_n) { + if (idx >= D) { + throw std::runtime_error("Hierarchy::unprocessed idx out of bound."); + } + processed_n = _unprocessed_n[idx]; + return _unprocessed_dims[idx]; +} + +template +Array<2, SIZE, DeviceType> &Hierarchy::level_ranges() { + return _level_ranges; +} + +template +Array<2, int, DeviceType> &Hierarchy::level_marks() { + return _level_marks; +} + +template +Array<1, T, DeviceType> & +Hierarchy::level_volumes(bool reciprocal) { + if (!reciprocal) { + return _level_volumes; + } else { + return _level_volumes_reciprocal; + } +} + +template +Array<2, SIZE, DeviceType> & +Hierarchy::level_volume_offsets() { + return _level_volume_offsets; +} + +template +data_structure_type Hierarchy::data_structure() { + return dstype; +} + +template +void Hierarchy::destroy() { + initialized = false; +} + +template +std::vector +Hierarchy::create_uniform_coords(std::vector shape, + bool normalize_coordinates) { + + std::vector coords(D); + for (int d = 0; d < D; d++) { + T *curr_coords = new T[shape[d]]; + for (int i = 0; i < shape[d]; i++) { + // 0...n-1 + if (!normalize_coordinates) { + curr_coords[i] = (T)i; + } else { + // 0...1 + curr_coords[i] = (T)i / (shape[d] - 1); + } + } + coords[d] = curr_coords; + } + uniform_coords_created = true; + return coords; +} + +template void printShape(std::string name, std::vector shape) { + std::cout << log::log_info << name << ": "; + for (DIM d = 0; d < shape.size(); d++) { + std::cout << shape[d] << " "; + } + std::cout << std::endl; +} + +template +bool Hierarchy::is_initialized() { + return initialized; +} + +template +bool Hierarchy::can_reuse(std::vector shape) { + if (data_structure() == data_structure_type::Cartesian_Grid_Non_Uniform) { + return false; + } + for (DIM d = 0; d < D; d++) { + if (level_shape(l_target(), d) != shape[d]) { + return false; + } + } + return true; +} + +// This constructor is for internal use only +template +Hierarchy::Hierarchy() {} + +template +size_t +Hierarchy::EstimateMemoryFootprint(std::vector shape, + Config config) { + Hierarchy hierarchy; + return hierarchy.EstimateMemoryFootprint(shape); +} + +template +void Hierarchy::init_shape_only(std::vector shape, + data_structure_type dstype) { + int ret = check_shape(shape); + if (ret == -1) { + throw std::runtime_error( + "Number of dimensions mismatch. mgard_x_160::Hierarchy not initialized!"); + } + if (ret == -2) { + throw std::runtime_error("Size of any dimension cannot be smaller than 3. " + "mgard_x_160::Hierarchy not initialized!"); + } + this->shape = shape; + this->dstype = dstype; + _l_target = 0; + _level_shape = {shape}; + _total_num_elems = 1; + for (DIM d = 0; d < D; d++) { + _total_num_elems *= shape[d]; + } + _linearized_width = 1; + for (int d = D - 2; d >= 0; d--) { + _linearized_width *= shape[d]; + } + _level_num_elems = {_total_num_elems}; + initialized = true; +} + +template +Hierarchy::Hierarchy(std::vector shape, Config config) { + int ret = check_shape(shape); + if (ret == -1) { + throw std::runtime_error( + "Number of dimensions mismatch. mgard_x_160::Hierarchy not initialized!"); + } + if (ret == -2) { + throw std::runtime_error("Size of any dimension cannot be smaller than 3. " + "mgard_x_160::Hierarchy not initialized!"); + std::stringstream ss; + for (DIM d = 0; d < D; d++) + ss << shape[d] << " "; + throw std::runtime_error("Input shape: " + ss.str()); + } + dstype = data_structure_type::Cartesian_Grid_Uniform; + std::vector coords = + create_uniform_coords(shape, config.normalize_coordinates); + init(shape, coords, config.num_global_refactoring_level); + assert(uniform_coords_created); + assert(coords.size() == D); + for (int d = 0; d < D; d++) + delete[] coords[d]; +} + +template +Hierarchy::Hierarchy(std::vector shape, + std::vector coords, Config config) { + int ret = check_shape(shape); + if (ret == -1) { + throw std::runtime_error( + "Number of dimensions mismatch. mgard_x_160::Hierarchy not initialized!"); + } + if (ret == -2) { + throw std::runtime_error("Size of any dimension cannot be smaller than 3. " + "mgard_x_160::Hierarchy not initialized!"); + } + + dstype = data_structure_type::Cartesian_Grid_Non_Uniform; + init(shape, coords, config.num_global_refactoring_level); +} + +template +Hierarchy::Hierarchy(const Hierarchy &hierarchy) { + _l_target = hierarchy._l_target; + shape = hierarchy.shape; + _total_num_elems = hierarchy._total_num_elems; + _level_num_elems = hierarchy._level_num_elems; + _linearized_width = hierarchy._linearized_width; + _coords_org = hierarchy._coords_org; + _dist_array = hierarchy._dist_array; + _ratio_array = hierarchy._ratio_array; + _level_volumes = hierarchy._level_volumes; + _level_volumes_reciprocal = hierarchy._level_volumes_reciprocal; + _level_volume_offsets = hierarchy._level_volume_offsets; + _am_array = hierarchy._am_array; + _bm_array = hierarchy._bm_array; + _level_shape = hierarchy._level_shape; + _level_shape_array = hierarchy._level_shape_array; + _level_ranges = hierarchy._level_ranges; + _level_marks = hierarchy._level_marks; + dstype = hierarchy.dstype; + if (D >= 4) { + for (DIM d = 0; d < D; d++) { + _processed_n[d] = hierarchy._processed_n[d]; + _unprocessed_n[d] = hierarchy._unprocessed_n[d]; + _processed_dims[d] = hierarchy._processed_dims[d]; + _unprocessed_dims[d] = hierarchy._unprocessed_dims[d]; + } + } + dummy_array = hierarchy.dummy_array; +} + +template +Hierarchy::~Hierarchy() { + if (initialized) { + destroy(); + } +} + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Hierarchy/HybridHierarchy.h b/include/mgard-x-160/Hierarchy/HybridHierarchy.h new file mode 100644 index 000000000..1495b2f14 --- /dev/null +++ b/include/mgard-x-160/Hierarchy/HybridHierarchy.h @@ -0,0 +1,169 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: October 1, 2026 + */ + +#ifndef MGARD_X_160_HYBRID_HIERARCHY_H +#define MGARD_X_160_HYBRID_HIERARCHY_H + +#include "../RuntimeX/Utilities/Exceptions.h" +#include "Hierarchy.h" + +namespace mgard_x_160 { + +// Hierarchy of the hybrid (BlockMGARD) decomposition: L block-local levels on +// the input grid, then M global multilevel levels on the coarsest local grid +// (or on the input grid when L == 0). +// +// The block-local stage needs nothing from a Hierarchy beyond the input shape +// and the padded per-level block grids, so the base Hierarchy only describes +// the input grid (one level, no device arrays -- see init_shape_only). The +// only full Hierarchy is the global one, built here once, on the grid the +// global stage actually decomposes, and only when M > 0. +template +class HybridHierarchy : public Hierarchy { +public: + HybridHierarchy() {} + + HybridHierarchy(std::vector shape, Config config) { + init(shape, config, data_structure_type::Cartesian_Grid_Uniform); + } + + // Coordinates are accepted for parity with Hierarchy (DomainDecomposer + // builds hierarchies either way) but not used: the block-local transform + // assumes uniform spacing and the global stage is built on a uniform grid. + // The grid is still marked non-uniform, so the compressor cache treats it + // the same as a non-uniform Hierarchy. + HybridHierarchy(std::vector shape, std::vector coords, + Config config) { + init(shape, config, data_structure_type::Cartesian_Grid_Non_Uniform); + } + + // Shape of the input data. + std::vector input_shape() { return this->level_shape(0); } + + SIZE num_local_levels() { return L; } + SIZE num_global_levels() { return M; } + + // Block-local level l (0 = finest) operates on local_fine_shape(l): the + // previous grid padded up to whole blocks. It produces local_coarse_shape(l), + // MGARDX_HYBRID_LOCAL_COARSE_SIZE nodes per block per dimension. + std::vector local_fine_shape(SIZE l) { return fine_shapes[l]; } + std::vector local_coarse_shape(SIZE l) { return coarse_shapes[l]; } + + // Grid the global stage decomposes: the coarsest local grid, or the input + // grid when there is no local stage. + std::vector global_shape() { + return L > 0 ? coarse_shapes[L - 1] : input_shape(); + } + + // Hierarchy of the global stage. Only exists when M > 0. + Hierarchy &global_hierarchy() { + if (M == 0) { + throw ProcessingException( + "HybridHierarchy: no global hierarchy without global levels."); + } + return global; + } + + // Config the global stage is built and run with: defaults, limited to M + // levels. + static Config global_config(SIZE M) { + Config config; + config.num_global_refactoring_level = M; + return config; + } + + bool matches_config(Config config) { + return L == config.num_local_refactoring_level && + M == config.num_global_refactoring_level; + } + + // The hybrid components take their level counts from the config and their + // grids from this hierarchy, so the two have to agree. + void check_config(Config config) { + if (!matches_config(config)) { + throw ProcessingException( + "HybridHierarchy was built for " + std::to_string(L) + + " local / " + std::to_string(M) + " global levels, but config has " + + std::to_string(config.num_local_refactoring_level) + " / " + + std::to_string(config.num_global_refactoring_level) + "."); + } + } + + // Padded block grids of the L local levels of an input of `shape`. + static void compute_local_shapes(std::vector shape, SIZE L, + std::vector> &fine, + std::vector> &coarse) { + fine.clear(); + coarse.clear(); + std::vector curr = shape; + for (SIZE l = 0; l < L; l++) { + std::vector f(D), c(D); + for (DIM d = 0; d < D; d++) { + SIZE num_blocks = (curr[d] - 1) / MGARDX_HYBRID_LOCAL_BLOCK_SIZE + 1; + f[d] = num_blocks * MGARDX_HYBRID_LOCAL_BLOCK_SIZE; + c[d] = num_blocks * MGARDX_HYBRID_LOCAL_COARSE_SIZE; + } + fine.push_back(f); + coarse.push_back(c); + curr = c; + } + } + + // Device memory of a HybridHierarchy for `shape`: only the global + // hierarchy, if any (the base part allocates nothing). Uses the Hierarchy + // estimate, which assumes all levels, so it is an upper bound for M levels. + static size_t EstimateMemoryFootprint(std::vector shape, + Config config) { + SIZE L = config.num_local_refactoring_level; + SIZE M = config.num_global_refactoring_level; + if (M == 0) { + return 0; + } + std::vector global_input = shape; + if (L > 0) { + std::vector> fine, coarse; + compute_local_shapes(shape, L, fine, coarse); + global_input = coarse[L - 1]; + } + return Hierarchy::EstimateMemoryFootprint( + global_input, global_config(M)); + } + + // Same, for this hierarchy's own level counts. + size_t EstimateMemoryFootprint(std::vector shape) { + Config config; + config.num_local_refactoring_level = L; + config.num_global_refactoring_level = M; + return EstimateMemoryFootprint(shape, config); + } + +private: + void init(std::vector shape, Config config, + data_structure_type dstype) { + L = config.num_local_refactoring_level; + M = config.num_global_refactoring_level; + if (L == 0 && M == 0) { + throw ProcessingException( + "HybridHierarchy: local and global levels cannot both be zero."); + } + this->init_shape_only(shape, dstype); + compute_local_shapes(shape, L, fine_shapes, coarse_shapes); + if (M > 0) { + global = Hierarchy(global_shape(), global_config(M)); + } + } + + SIZE L = 0; + SIZE M = 0; + std::vector> fine_shapes; + std::vector> coarse_shapes; + Hierarchy global; +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Lossless/BlockDelta/BlockDelta.hpp b/include/mgard-x-160/Lossless/BlockDelta/BlockDelta.hpp new file mode 100644 index 000000000..fb4d27a1f --- /dev/null +++ b/include/mgard-x-160/Lossless/BlockDelta/BlockDelta.hpp @@ -0,0 +1,494 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_BLOCK_DELTA_HPP +#define MGARD_X_160_BLOCK_DELTA_HPP + +#include "../../RuntimeX/Utilities/Serializer.hpp" +#include "../../Utilities/Types.h" // block_delta_mode_type +#include "../LosslessCompressorInterface.hpp" +#include "BlockDeltaFused.hpp" // CUDA/HIP single-kernel decoupled look-back +#include "BlockDeltaKernels.hpp" // pulls in RuntimeX (types + macros) first + +namespace mgard_x_160 { + +// True for the GPU backends that get the fused single-kernel (decoupled +// look-back) implementation. Everything else uses the portable multi-kernel +// path. The branch is resolved at compile time, so only one path is +// instantiated per backend. +template struct is_gpu_device { + static constexpr bool value = std::is_same::value || + std::is_same::value; +}; + +// BlockDelta lossless backend: a non-entropy alternative to Huffman that +// operates on the same signed quantized-integer stream. See BlockDeltaKernels +// for the encoding scheme. Self-contained: Compress writes a complete buffer +// (header + per-block bit-widths + packed stream); Serialize is therefore a +// no-op and Deserialize parses the header back. +template class BlockDeltaLossless { +public: + BlockDeltaLossless() : initialized(false) {} + + BlockDeltaLossless( + SIZE max_size, int block_size, + block_delta_mode_type mode = block_delta_mode_type::Delta) { + Resize(max_size, block_size, mode, 0); + DeviceRuntime::SyncQueue(0); + } + + void Resize(SIZE max_size, int block_size, block_delta_mode_type mode, + int queue_idx) { + this->initialized = true; + this->max_size = max_size; + this->block_size = block_size; + this->mode = mode; + SIZE max_nblocks = (max_size - 1) / block_size + 1; + bitwidth_array = Array<1, Byte, DeviceType>({max_nblocks}); + bytecount_array = Array<1, size_t, DeviceType>({max_nblocks}); + byte_offset_array = Array<1, size_t, DeviceType>({max_nblocks + 1}); + // Per-block outlier counts (Outlier mode only; tiny, always allocated). + oc_array = Array<1, uint16_t, DeviceType>({max_nblocks}); + // Scan workspace for the multi-kernel path (extended exclusive scan over + // nblocks). Always sized: the GPU backend still uses the portable path for + // Outlier mode, which scans. + DeviceCollective::ScanSumExtended( + max_nblocks, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), scan_workspace, false, queue_idx); + if constexpr (is_gpu_device::value) { + // Decoupled-look-back state for the fused path: one status word per block + // plus a global tile counter and a total-size slot. + status_array = Array<1, unsigned long long, DeviceType>({max_nblocks}); + counter_array = Array<1, unsigned int, DeviceType>({1}); + total_array = Array<1, unsigned long long, DeviceType>({1}); + } + } + + static size_t EstimateMemoryFootprint(SIZE primary_count, int block_size) { + SIZE max_nblocks = (primary_count - 1) / block_size + 1; + size_t size = max_nblocks * sizeof(Byte); // bitwidth + size += max_nblocks * sizeof(size_t); // bytecount + size += (max_nblocks + 1) * sizeof(size_t); // byte_offset + size += max_nblocks * sizeof(uint16_t); // outlier counts + return size; + } + + // ---- public LosslessCompressorInterface-style entry points -------------- + + void Compress(Array<1, T, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + if constexpr (is_gpu_device::value) { + CompressFused(original_data, compressed_data, queue_idx); + } else { + CompressPortable(original_data, compressed_data, queue_idx); + } + } + + void Decompress(Array<1, Byte, DeviceType> &compressed_data, + Array<1, T, DeviceType> &decompressed_data, int queue_idx) { + if constexpr (is_gpu_device::value) { + DecompressFused(compressed_data, decompressed_data, queue_idx); + } else { + DecompressPortable(compressed_data, decompressed_data, queue_idx); + } + } + + // Memory-movement stage (no computation): copy the bit-width array out of the + // workspace and write the scalar header into the compressed buffer. Compress + // already wrote the packed bitstream in place; this only fills the metadata + // sections around it. Kept separate from Compress so it can overlap with the + // next subdomain's kernels in the pipeline (same split as Huffman). + void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + SubArray<1, Byte, DeviceType> cd(compressed_data); + size_t n_v = n, nblocks_v = nblocks, bitwidth_bytes_v = nblocks, + oc_bytes_v = nblocks * sizeof(uint16_t), + packed_bytes_v = packed_bytes; + int block_size_v = block_size; + Byte mode_v = (Byte)mode; + SIZE byte_offset = 0; + SerializeArray(cd, signature, kSignatureLen, byte_offset, queue_idx); + SerializeArray(cd, &n_v, 1, byte_offset, queue_idx); + SerializeArray(cd, &block_size_v, 1, byte_offset, queue_idx); + SerializeArray(cd, &mode_v, 1, byte_offset, queue_idx); + SerializeArray(cd, &nblocks_v, 1, byte_offset, queue_idx); + SerializeArray(cd, &bitwidth_bytes_v, 1, byte_offset, queue_idx); + SerializeArray(cd, bitwidth_array.data(), nblocks, byte_offset, + queue_idx); + if (mode == block_delta_mode_type::Outlier) { + SerializeArray(cd, &oc_bytes_v, 1, byte_offset, queue_idx); + SerializeArray(cd, oc_array.data(), nblocks, byte_offset, + queue_idx); + } + SerializeArray(cd, &packed_bytes_v, 1, byte_offset, queue_idx); + // packed[] is already in place (written by Compress); nothing to copy. + DeviceRuntime::SyncQueue(queue_idx); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("BlockDelta serialize", compressed_data.shape(0)); + timer.clear(); + } + } + + void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + ParseHeader(compressed_data, queue_idx); + } + +private: + // Layout walker; mirrors the section order written by Serialize. Returns the + // total compressed size and, via the out-params, where the bit-width array, + // the outlier-count array (Outlier mode only), and the packed bitstream land. + // Layout: signature | n | block_size | mode | nblocks | bitwidth_bytes | + // bitwidth[] | [Outlier: oc_bytes | oc[]] | packed_bytes | packed[] + SIZE ComputeLayout(Byte mode_v, size_t nblocks, size_t packed_bytes, + SIZE &bitwidth_byte_offset, SIZE &oc_byte_offset, + SIZE &packed_byte_offset) { + SIZE off = 0; + advance_with_align(off, kSignatureLen); // signature + advance_with_align(off, 1); // n + advance_with_align(off, 1); // block_size + advance_with_align(off, 1); // mode + advance_with_align(off, 1); // nblocks + advance_with_align(off, 1); // bitwidth_bytes + align_byte_offset(off); + bitwidth_byte_offset = off; + advance_with_align(off, nblocks); // bitwidth[] + oc_byte_offset = 0; + if (mode_v == (Byte)block_delta_mode_type::Outlier) { + advance_with_align(off, 1); // oc_bytes + align_byte_offset(off); + oc_byte_offset = off; + advance_with_align(off, nblocks); // oc[] + } + advance_with_align(off, 1); // packed_bytes + align_byte_offset(off); + packed_byte_offset = off; + advance_with_align(off, packed_bytes); + return off; + } + + void CompressPortable(Array<1, T, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, + int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + n = original_data.shape(0); + nblocks = (SIZE)((n - 1) / block_size + 1); + SubArray<1, T, DeviceType> data_subarray(original_data); + SubArray<1, Byte, DeviceType> bitwidth_subarray({(SIZE)nblocks}, + bitwidth_array.data()); + SubArray<1, size_t, DeviceType> bytecount_subarray({(SIZE)nblocks}, + bytecount_array.data()); + SubArray<1, size_t, DeviceType> byte_offset_subarray( + {(SIZE)nblocks + 1}, byte_offset_array.data()); + SubArray<1, uint16_t, DeviceType> oc_subarray({(SIZE)nblocks}, + oc_array.data()); + Byte mode_v = (Byte)mode; + + // 1) per-block bit-width + byte-count (+ outlier count) + DeviceLauncher::Execute( + BlockBitwidthKernel( + data_subarray, (SIZE)n, (SIZE)block_size, (SIZE)nblocks, mode_v, + bitwidth_subarray, bytecount_subarray, oc_subarray), + queue_idx); + + // 2) exclusive scan of byte-counts -> per-block byte offsets (+ total) + DeviceCollective::ScanSumExtended( + (SIZE)nblocks, bytecount_subarray, byte_offset_subarray, scan_workspace, + true, queue_idx); + MemoryManager::Copy1D( + &packed_bytes, byte_offset_subarray.data() + nblocks, 1, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + // 3) size the output buffer. Computation only: the metadata (signature, + // scalar header, bit-width array, outlier counts) is *not* written here -- + // that memory movement is deferred to Serialize so it can overlap with the + // next subdomain's computation in the pipeline. + SIZE bitwidth_byte_offset, oc_byte_offset, packed_byte_offset; + SIZE compressed_size = + ComputeLayout(mode_v, nblocks, packed_bytes, bitwidth_byte_offset, + oc_byte_offset, packed_byte_offset); + compressed_data.resize({compressed_size}, queue_idx); + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + + // 4) pack into the packed region (a kernel writing to its final location, + // mirroring how Huffman's deflate writes its bitstream in place). + SubArray<1, Byte, DeviceType> packed_subarray( + {(SIZE)packed_bytes}, (Byte *)compressed_subarray(packed_byte_offset)); + DeviceLauncher::Execute( + BlockPackKernel(data_subarray, (SIZE)n, (SIZE)block_size, + (SIZE)nblocks, mode_v, bitwidth_subarray, + byte_offset_subarray, packed_subarray), + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + log::info("BlockDelta compress ratio: " + std::to_string(n * sizeof(T)) + + "/" + std::to_string(compressed_size) + " (" + + std::to_string((double)n * sizeof(T) / compressed_size) + ")"); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("BlockDelta compress", n * sizeof(T)); + timer.clear(); + } + } + + void ParseHeader(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + SIZE byte_offset = 0; + + Byte *sig = signature_verify; + DeserializeArray(compressed_subarray, sig, kSignatureLen, byte_offset, + false, queue_idx); + size_t *n_ptr = &n, *nblocks_ptr = &nblocks, + *bitwidth_bytes_ptr = &bitwidth_bytes, + *packed_bytes_ptr = &packed_bytes; + int *block_size_ptr = &block_size; + Byte mode_v = 0, *mode_ptr = &mode_v; + DeserializeArray(compressed_subarray, n_ptr, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, block_size_ptr, 1, byte_offset, + false, queue_idx); + DeserializeArray(compressed_subarray, mode_ptr, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, nblocks_ptr, 1, byte_offset, + false, queue_idx); + DeserializeArray(compressed_subarray, bitwidth_bytes_ptr, 1, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + this->mode = (block_delta_mode_type)mode_v; + for (int i = 0; i < kSignatureLen; i++) { + if (signature_verify[i] != signature[i]) { + throw std::runtime_error("BlockDelta signature mismatch."); + } + } + // zero-copy device pointers into the compressed buffer + DeserializeArray(compressed_subarray, bitwidth_ptr, bitwidth_bytes, + byte_offset, true, queue_idx); + if (this->mode == block_delta_mode_type::Outlier) { + size_t oc_bytes = 0, *oc_bytes_ptr = &oc_bytes; + DeserializeArray(compressed_subarray, oc_bytes_ptr, 1, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + DeserializeArray(compressed_subarray, oc_ptr, + oc_bytes / sizeof(uint16_t), byte_offset, true, + queue_idx); + } + DeserializeArray(compressed_subarray, packed_bytes_ptr, 1, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + DeserializeArray(compressed_subarray, packed_ptr, packed_bytes, + byte_offset, true, queue_idx); + } + + void DecompressPortable(Array<1, Byte, DeviceType> &compressed_data, + Array<1, T, DeviceType> &decompressed_data, + int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + decompressed_data.resize({(SIZE)n}, queue_idx); + SubArray<1, T, DeviceType> data_subarray(decompressed_data); + SubArray<1, Byte, DeviceType> bitwidth_subarray({(SIZE)nblocks}, + bitwidth_ptr); + SubArray<1, Byte, DeviceType> packed_subarray({(SIZE)packed_bytes}, + packed_ptr); + SubArray<1, size_t, DeviceType> bytecount_subarray({(SIZE)nblocks}, + bytecount_array.data()); + SubArray<1, size_t, DeviceType> byte_offset_subarray( + {(SIZE)nblocks + 1}, byte_offset_array.data()); + // Outlier mode reads its per-block counts from the (zero-copy) stream + // pointer; other modes don't touch it. + SubArray<1, uint16_t, DeviceType> oc_subarray( + {(SIZE)nblocks}, + mode == block_delta_mode_type::Outlier ? oc_ptr : oc_array.data()); + Byte mode_v = (Byte)mode; + + // Rebuild per-block byte offsets from the stored bit-widths (+ counts). + DeviceLauncher::Execute( + BlockBytecountKernel( + (SIZE)n, (SIZE)block_size, (SIZE)nblocks, mode_v, bitwidth_subarray, + oc_subarray, bytecount_subarray), + queue_idx); + DeviceCollective::ScanSumExtended( + (SIZE)nblocks, bytecount_subarray, byte_offset_subarray, scan_workspace, + true, queue_idx); + + DeviceLauncher::Execute( + BlockUnpackKernel( + packed_subarray, (SIZE)n, (SIZE)block_size, (SIZE)nblocks, mode_v, + bitwidth_subarray, byte_offset_subarray, data_subarray), + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("BlockDelta decompress", n * sizeof(T)); + timer.clear(); + } + } + + // -------- Fused (CUDA/HIP) path ----------------------------------------- + // Single kernel: the cross-block byte-offset scan is resolved in-kernel via + // decoupled look-back (see BlockDeltaFused.hpp). The byte layout is identical + // to the portable path, so streams are interchangeable across backends. + void CompressFused(Array<1, T, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, + int queue_idx) { +#if defined(MGARDX_COMPILE_CUDA) || defined(MGARDX_COMPILE_HIP) + // Outlier mode has variable per-block side records that don't fit the + // single-pass look-back cleanly; use the portable multi-kernel path (which + // also runs on the GPU). Fixed and Delta use the fused single kernel. + if (mode == block_delta_mode_type::Outlier) { + CompressPortable(original_data, compressed_data, queue_idx); + return; + } + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + n = original_data.shape(0); + nblocks = (SIZE)((n - 1) / block_size + 1); + + // Worst case: a block keeps full width -> packed <= n * sizeof(T). The + // bit-width and packed offsets depend only on nblocks, so they are valid + // for the real (trimmed) layout too. + size_t worst_packed = (size_t)n * sizeof(T); + SIZE bitwidth_off, oc_off, packed_off; + SIZE worst_size = ComputeLayout((Byte)mode, nblocks, worst_packed, + bitwidth_off, oc_off, packed_off); + compressed_data.resize({worst_size}, queue_idx); + Byte *base = SubArray<1, Byte, DeviceType>(compressed_data).data(); + + // Reset decoupled-look-back state. + MemoryManager::Memset1D(status_array.data(), (SIZE)nblocks, 0, + queue_idx); + MemoryManager::Memset1D(counter_array.data(), 1, 0, queue_idx); + + // Computation only: the kernel writes the bit-widths into the workspace and + // the packed bitstream into its final location. The metadata memory + // movement (bit-widths + scalar header) is deferred to Serialize. + bool use_delta = (mode != block_delta_mode_type::Fixed); + auto stream = DeviceRuntime::GetQueue(queue_idx); + block_delta_fused::launch_encode( + original_data.data(), (SIZE)n, (SIZE)block_size, (SIZE)nblocks, + use_delta, bitwidth_array.data(), base + packed_off, + status_array.data(), counter_array.data(), total_array.data(), stream); + + unsigned long long total = 0; + MemoryManager::Copy1D(&total, total_array.data(), 1, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + packed_bytes = (size_t)total; + + // Trim to the real size (in-place shrink preserves the kernel's packed + // writes). bitwidth_off is unused here -- Serialize recomputes the layout. + (void)bitwidth_off; + SIZE compressed_size = packed_off + (SIZE)packed_bytes; + compressed_data.resize({compressed_size}, queue_idx); + + log::info( + "BlockDelta(fused) compress ratio: " + std::to_string(n * sizeof(T)) + + "/" + std::to_string(compressed_size) + " (" + + std::to_string((double)n * sizeof(T) / compressed_size) + ")"); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("BlockDelta(fused) compress", n * sizeof(T)); + timer.clear(); + } +#else + CompressPortable(original_data, compressed_data, queue_idx); +#endif + } + + void DecompressFused(Array<1, Byte, DeviceType> &compressed_data, + Array<1, T, DeviceType> &decompressed_data, + int queue_idx) { +#if defined(MGARDX_COMPILE_CUDA) || defined(MGARDX_COMPILE_HIP) + // ParseHeader (via Deserialize) has set n, nblocks, block_size, mode, + // bitwidth_ptr and packed_ptr. Outlier mode uses the portable path. + if (mode == block_delta_mode_type::Outlier) { + DecompressPortable(compressed_data, decompressed_data, queue_idx); + return; + } + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + decompressed_data.resize({(SIZE)n}, queue_idx); + MemoryManager::Memset1D(status_array.data(), (SIZE)nblocks, 0, + queue_idx); + MemoryManager::Memset1D(counter_array.data(), 1, 0, queue_idx); + + bool use_delta = (mode != block_delta_mode_type::Fixed); + auto stream = DeviceRuntime::GetQueue(queue_idx); + block_delta_fused::launch_decode( + packed_ptr, (SIZE)n, (SIZE)block_size, (SIZE)nblocks, use_delta, + bitwidth_ptr, decompressed_data.data(), status_array.data(), + counter_array.data(), stream); + DeviceRuntime::SyncQueue(queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("BlockDelta(fused) decompress", n * sizeof(T)); + timer.clear(); + } +#else + DecompressPortable(compressed_data, decompressed_data, queue_idx); +#endif + } + +public: + bool initialized; + SIZE max_size; + int block_size; + block_delta_mode_type mode; + size_t n; + size_t nblocks; + size_t bitwidth_bytes; + size_t packed_bytes; + +private: + static constexpr int kSignatureLen = 8; + Byte signature[8] = {'M', 'G', 'X', 'B', 'L', 'K', 'D', '\0'}; + Byte signature_verify[8] = {0}; + // zero-copy pointers into the compressed buffer, set by ParseHeader + Byte *bitwidth_ptr = nullptr; + Byte *packed_ptr = nullptr; + uint16_t *oc_ptr = nullptr; + + Array<1, Byte, DeviceType> bitwidth_array; + Array<1, size_t, DeviceType> bytecount_array; + Array<1, size_t, DeviceType> byte_offset_array; + Array<1, uint16_t, DeviceType> oc_array; + Array<1, Byte, DeviceType> scan_workspace; + // Fused (GPU) decoupled-look-back state. + Array<1, unsigned long long, DeviceType> status_array; + Array<1, unsigned int, DeviceType> counter_array; + Array<1, unsigned long long, DeviceType> total_array; +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Lossless/BlockDelta/BlockDeltaFused.hpp b/include/mgard-x-160/Lossless/BlockDelta/BlockDeltaFused.hpp new file mode 100644 index 000000000..ab95e4187 --- /dev/null +++ b/include/mgard-x-160/Lossless/BlockDelta/BlockDeltaFused.hpp @@ -0,0 +1,204 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_BLOCK_DELTA_FUSED_HPP +#define MGARD_X_160_BLOCK_DELTA_FUSED_HPP + +#include "BlockDeltaKernels.hpp" // block_delta:: helpers + RuntimeX types/macros + +// Single-kernel (decoupled look-back) implementation of BlockDelta for the GPU +// backends. The cross-block byte-offset prefix sum -- the one global dependency +// in the scheme -- is resolved inside one kernel launch via a Merrill/Garland +// style decoupled look-back, instead of a separate ScanSumExtended pass. +// +// Mapping: one thread per data block. Each thread acquires a *dynamic* tile id +// via atomicAdd so logical order matches dispatch order (this is what keeps the +// look-back deadlock-free under the GPU's non-preemptive scheduling -- see the +// design discussion). It then computes its block's byte count, publishes it, +// looks back to obtain its exclusive prefix (= its byte offset), and packs. +// +// Intra-block work is serial per thread (same as the portable path); the point +// here is the fused cross-block scan, not intra-block parallelism. The byte +// layout is identical to the portable path, so streams are cross-compatible. + +#if defined(MGARDX_COMPILE_CUDA) || defined(MGARDX_COMPILE_HIP) + +namespace mgard_x_160 { +namespace block_delta_fused { + +// Per-tile status word: top 2 bits = state, low 62 bits = value (a byte count +// or a prefix sum, both well within 62 bits). State INVALID == 0 so a zeroed +// status array starts "not ready". +enum : unsigned long long { + ST_INVALID = 0ULL, + ST_AGGREGATE = 1ULL, + ST_PREFIX = 2ULL +}; +__device__ __forceinline__ unsigned long long +make_status(unsigned long long state, unsigned long long val) { + return (state << 62) | (val & ((1ULL << 62) - 1)); +} +__device__ __forceinline__ unsigned long long st_state(unsigned long long s) { + return s >> 62; +} +__device__ __forceinline__ unsigned long long st_val(unsigned long long s) { + return s & ((1ULL << 62) - 1); +} + +// Exclusive prefix of byte counts for tile `b` via decoupled look-back. +// `bc` is this tile's own byte count. Publishes AGGREGATE before walking back, +// then PREFIX once the exclusive prefix is known. +__device__ __forceinline__ unsigned long long +lookback_exclusive_prefix(volatile unsigned long long *status, unsigned int b, + unsigned long long bc) { + status[b] = make_status(ST_AGGREGATE, bc); + __threadfence(); + unsigned long long excl = 0; + long look = (long)b - 1; + while (look >= 0) { + unsigned long long s; + do { + s = status[look]; + } while (st_state(s) == ST_INVALID); + excl += st_val(s); + if (st_state(s) == ST_PREFIX) + break; // PREFIX value already folds in everything <= look + look--; + } + status[b] = make_status(ST_PREFIX, excl + bc); + __threadfence(); + return excl; +} + +template +__global__ void +encode_kernel(const T *__restrict__ data, SIZE n, SIZE block_size, SIZE nblocks, + bool use_delta, Byte *__restrict__ bitwidth, + Byte *__restrict__ packed, volatile unsigned long long *status, + unsigned int *counter, unsigned long long *total) { + using UT = typename std::make_unsigned::type; + unsigned int b = atomicAdd(counter, 1u); + if (b >= nblocks) + return; + SIZE start = b * block_size; + SIZE len = block_size < (n - start) ? block_size : (n - start); + + // bit-width over (delta+)zigzag + T prev = 0; + UT acc = 0; + for (SIZE i = 0; i < len; i++) { + T x = data[start + i]; + acc |= block_delta::zigzag(use_delta ? (T)(x - prev) : x); + prev = x; + } + int bw = block_delta::bit_length(acc); + bitwidth[b] = (Byte)bw; + unsigned long long bc = (unsigned long long)block_delta::block_bytes(bw, len); + + // fused cross-block scan + unsigned long long excl = lookback_exclusive_prefix(status, b, bc); + + // pack into our (disjoint) byte range + Byte *out = packed + excl; + prev = 0; + UT buf = 0; + int cnt = 0; + size_t pos = 0; + for (SIZE i = 0; i < len; i++) { + T x = data[start + i]; + UT z = block_delta::zigzag(use_delta ? (T)(x - prev) : x); + prev = x; + for (int k = 0; k < bw; k++) { + buf |= (UT)((z >> k) & 1) << cnt; + if (++cnt == 8) { + out[pos++] = (Byte)(buf & 0xff); + buf = 0; + cnt = 0; + } + } + } + if (cnt > 0) + out[pos++] = (Byte)(buf & 0xff); + + if (b == nblocks - 1) + *total = excl + bc; // last tile holds the full packed size +} + +template +__global__ void +decode_kernel(const Byte *__restrict__ packed, SIZE n, SIZE block_size, + SIZE nblocks, bool use_delta, const Byte *__restrict__ bitwidth, + T *__restrict__ data, volatile unsigned long long *status, + unsigned int *counter) { + using UT = typename std::make_unsigned::type; + unsigned int b = atomicAdd(counter, 1u); + if (b >= nblocks) + return; + SIZE start = b * block_size; + SIZE len = block_size < (n - start) ? block_size : (n - start); + int bw = (int)bitwidth[b]; + unsigned long long bc = (unsigned long long)block_delta::block_bytes(bw, len); + + unsigned long long excl = lookback_exclusive_prefix(status, b, bc); + + const Byte *in = packed + excl; + T prev = 0; + Byte cur = 0; + int cnt = 0; + size_t pos = 0; + for (SIZE i = 0; i < len; i++) { + UT z = 0; + for (int k = 0; k < bw; k++) { + if (cnt == 0) { + cur = in[pos++]; + cnt = 8; + } + z |= (UT)(cur & 1) << k; + cur >>= 1; + cnt--; + } + T d = block_delta::unzigzag(z); + prev = use_delta ? (T)(prev + d) : d; + data[start + i] = prev; + } +} + +#ifdef MGARDX_COMPILE_CUDA +using gpuStream_t = cudaStream_t; +#else +using gpuStream_t = hipStream_t; +#endif + +template +inline void launch_encode(const T *data, SIZE n, SIZE block_size, SIZE nblocks, + bool use_delta, Byte *bitwidth, Byte *packed, + unsigned long long *status, unsigned int *counter, + unsigned long long *total, gpuStream_t stream) { + SIZE tpb = 256; + SIZE grid = (nblocks - 1) / tpb + 1; + encode_kernel<<>>( + data, n, block_size, nblocks, use_delta, bitwidth, packed, + (volatile unsigned long long *)status, counter, total); +} + +template +inline void launch_decode(const Byte *packed, SIZE n, SIZE block_size, + SIZE nblocks, bool use_delta, const Byte *bitwidth, + T *data, unsigned long long *status, + unsigned int *counter, gpuStream_t stream) { + SIZE tpb = 256; + SIZE grid = (nblocks - 1) / tpb + 1; + decode_kernel<<>>( + packed, n, block_size, nblocks, use_delta, bitwidth, data, + (volatile unsigned long long *)status, counter); +} + +} // namespace block_delta_fused +} // namespace mgard_x_160 + +#endif // CUDA || HIP + +#endif diff --git a/include/mgard-x-160/Lossless/BlockDelta/BlockDeltaKernels.hpp b/include/mgard-x-160/Lossless/BlockDelta/BlockDeltaKernels.hpp new file mode 100644 index 000000000..57ee4939b --- /dev/null +++ b/include/mgard-x-160/Lossless/BlockDelta/BlockDeltaKernels.hpp @@ -0,0 +1,544 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_BLOCK_DELTA_KERNELS_HPP +#define MGARD_X_160_BLOCK_DELTA_KERNELS_HPP + +#include "../../RuntimeX/RuntimeX.h" + +// Portable (multi-kernel) building blocks for the BlockDelta lossless backend. +// +// Three encoding modes (mirroring cuSZp), selected per call: +// Fixed (0): zigzag(value) -> fixed-length pack at per-block width +// Delta (1): zigzag(value-prev) -> fixed-length pack (default) +// Outlier (2): Delta, but values exceeding a per-block budget width are +// peeled into a side record list so a few large values don't +// widen the whole block. +// +// Each block is padded to a whole byte and owns a disjoint byte range of the +// output (no atomics; trivially parallel). For Fixed/Delta a block's byte range +// is just its packed bitstream. For Outlier it is laid out as: +// [outlier_count : 2B][main FLE : ceil(bw*len/8)B][records : oc * (2B pos + +// sizeof(T)B value)] +// where outlier slots in the main stream store 0 and the full zigzag value +// lives in the record. Records are written in increasing position order so +// decode can patch them with an O(len+oc) single scan. +// +// All functors use a 1-thread-per-data-block mapping (the portable correctness +// reference). The CUDA/HIP fused path implements Fixed/Delta in one kernel. + +namespace mgard_x_160 { + +namespace block_delta { + +enum : Byte { MODE_FIXED = 0, MODE_DELTA = 1, MODE_OUTLIER = 2 }; + +// Bytes per outlier record: 2-byte intra-block position + the value. +template MGARDX_CONT_EXEC constexpr int outlier_rec_bytes() { + return 2 + (int)sizeof(T); +} + +template +MGARDX_CONT_EXEC typename std::make_unsigned::type zigzag(T v) { + using UT = typename std::make_unsigned::type; + constexpr int bits = sizeof(T) * 8; + return (UT)((v << 1) ^ (v >> (bits - 1))); +} + +template +MGARDX_CONT_EXEC T unzigzag(typename std::make_unsigned::type z) { + return (T)((z >> 1) ^ (~(z & 1) + 1)); // (z>>1) ^ -(z&1) +} + +template MGARDX_CONT_EXEC int bit_length(UT acc) { + int n = 0; + while (acc) { + n++; + acc >>= 1; + } + return n; +} + +// Bytes to hold `nvalues` values of `bits` bits each, padded to a byte. +MGARDX_CONT_EXEC size_t block_bytes(int bits, SIZE nvalues) { + return (size_t)(((size_t)bits * nvalues + 7) / 8); +} + +// Per-block sizing shared by host/device. Computes the block's fixed-length +// width `bw`, byte count `bc`, and (Outlier only) outlier count `oc`. +template +MGARDX_CONT_EXEC void size_block(const T *data, SIZE start, SIZE len, Byte mode, + int &bw, size_t &bc, int &oc) { + using UT = typename std::make_unsigned::type; + bool use_delta = (mode != MODE_FIXED); + T prev = 0; + UT acc = 0; + int hist[65]; + if (mode == MODE_OUTLIER) { + for (int k = 0; k < 65; k++) + hist[k] = 0; + } + for (SIZE i = 0; i < len; i++) { + T x = data[start + i]; + UT z = zigzag(use_delta ? (T)(x - prev) : x); + prev = x; + acc |= z; + if (mode == MODE_OUTLIER) + hist[bit_length(z)]++; + } + int bw_max = bit_length(acc); + if (mode != MODE_OUTLIER) { + bw = bw_max; + oc = 0; + bc = block_bytes(bw, len); + return; + } + // suffix sums: suf[k] = #{ bit_length(z) >= k } + int suf[66]; + suf[65] = 0; + for (int k = 64; k >= 0; k--) + suf[k] = suf[k + 1] + hist[k]; + int best_bw = bw_max, best_oc = 0; + size_t best_cost = (size_t)-1; + for (int b = 0; b <= bw_max; b++) { + int o = suf[b + 1]; // values needing > b bits + size_t cost = 2 + block_bytes(b, len) + (size_t)o * outlier_rec_bytes(); + if (cost < best_cost) { + best_cost = cost; + best_bw = b; + best_oc = o; + } + } + bw = best_bw; + oc = best_oc; + bc = 2 + block_bytes(best_bw, len) + (size_t)best_oc * outlier_rec_bytes(); +} + +} // namespace block_delta + +// --------------------------------------------------------------------------- +// Kernel 1: per-block bit-width + byte-count (+ outlier count) -- encode +// sizing. +// --------------------------------------------------------------------------- +template +class BlockBitwidthFunctor : public Functor { +public: + MGARDX_CONT BlockBitwidthFunctor() {} + MGARDX_CONT BlockBitwidthFunctor(SubArray<1, T, DeviceType> data, SIZE n, + SIZE block_size, SIZE nblocks, Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, size_t, DeviceType> bytecount, + SubArray<1, uint16_t, DeviceType> oc) + : data(data), n(n), block_size(block_size), nblocks(nblocks), mode(mode), + bitwidth(bitwidth), bytecount(bytecount), oc(oc) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE b = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (b >= nblocks) + return; + SIZE start = b * block_size; + SIZE len = block_size < (n - start) ? block_size : (n - start); + int bw; + size_t bc; + int ocount; + block_delta::size_block(data.data(), start, len, mode, bw, bc, ocount); + *bitwidth(b) = (Byte)bw; + *bytecount(b) = bc; + *oc(b) = (uint16_t)ocount; + } + + MGARDX_EXEC void Operation2() {} + MGARDX_EXEC void Operation3() {} + MGARDX_EXEC void Operation4() {} + MGARDX_EXEC void Operation5() {} + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T, DeviceType> data; + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, size_t, DeviceType> bytecount; + SubArray<1, uint16_t, DeviceType> oc; +}; + +template +class BlockBitwidthKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + constexpr static bool EnableAutoTuning() { return false; } + using DataType = T; + constexpr static std::string_view Name = "block_delta_bitwidth"; + MGARDX_CONT BlockBitwidthKernel(SubArray<1, T, DeviceType> data, SIZE n, + SIZE block_size, SIZE nblocks, Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, size_t, DeviceType> bytecount, + SubArray<1, uint16_t, DeviceType> oc) + : data(data), n(n), block_size(block_size), nblocks(nblocks), mode(mode), + bitwidth(bitwidth), bytecount(bytecount), oc(oc) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = BlockBitwidthFunctor; + FunctorType functor(data, n, block_size, nblocks, mode, bitwidth, bytecount, + oc); + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (nblocks - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, tbz, tby, tbx, + functor.shared_memory_size(), queue_idx, std::string(Name)); + } + +private: + SubArray<1, T, DeviceType> data; + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, size_t, DeviceType> bytecount; + SubArray<1, uint16_t, DeviceType> oc; +}; + +// --------------------------------------------------------------------------- +// Kernel 2: pack each block into its (disjoint) byte range of the output. +// --------------------------------------------------------------------------- +template +class BlockPackFunctor : public Functor { +public: + MGARDX_CONT BlockPackFunctor() {} + MGARDX_CONT BlockPackFunctor(SubArray<1, T, DeviceType> data, SIZE n, + SIZE block_size, SIZE nblocks, Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, size_t, DeviceType> byte_offset, + SubArray<1, Byte, DeviceType> packed) + : data(data), n(n), block_size(block_size), nblocks(nblocks), mode(mode), + bitwidth(bitwidth), byte_offset(byte_offset), packed(packed) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + using UT = typename std::make_unsigned::type; + SIZE b = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (b >= nblocks) + return; + int bw = (int)*bitwidth(b); + SIZE start = b * block_size; + SIZE len = block_size < (n - start) ? block_size : (n - start); + bool use_delta = (mode != block_delta::MODE_FIXED); + bool outlier = (mode == block_delta::MODE_OUTLIER); + Byte *out = packed(*byte_offset(b)); + size_t main_start = outlier ? 2 : 0; + + // main fixed-length stream + T prev = 0; + UT buf = 0; + int cnt = 0; + size_t pos = main_start; + for (SIZE i = 0; i < len; i++) { + T x = *data(start + i); + UT z = block_delta::zigzag(use_delta ? (T)(x - prev) : x); + prev = x; + UT sv = + (outlier && bw < (int)(sizeof(T) * 8) && z >= ((UT)1 << bw)) ? 0 : z; + for (int k = 0; k < bw; k++) { + buf |= (UT)((sv >> k) & 1) << cnt; + if (++cnt == 8) { + out[pos++] = (Byte)(buf & 0xff); + buf = 0; + cnt = 0; + } + } + } + if (cnt > 0) + out[pos++] = (Byte)(buf & 0xff); + + if (outlier) { + // records + header + int oc = 0; + size_t rp = main_start + block_delta::block_bytes(bw, len); + prev = 0; + for (SIZE i = 0; i < len; i++) { + T x = *data(start + i); + UT z = block_delta::zigzag((T)(x - prev)); + prev = x; + if (bw < (int)(sizeof(T) * 8) && z >= ((UT)1 << bw)) { + out[rp] = (Byte)(i & 0xff); + out[rp + 1] = (Byte)((i >> 8) & 0xff); + rp += 2; + for (int k = 0; k < (int)sizeof(T); k++) + out[rp + k] = (Byte)((z >> (8 * k)) & 0xff); + rp += sizeof(T); + oc++; + } + } + out[0] = (Byte)(oc & 0xff); + out[1] = (Byte)((oc >> 8) & 0xff); + } + } + + MGARDX_EXEC void Operation2() {} + MGARDX_EXEC void Operation3() {} + MGARDX_EXEC void Operation4() {} + MGARDX_EXEC void Operation5() {} + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T, DeviceType> data; + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, size_t, DeviceType> byte_offset; + SubArray<1, Byte, DeviceType> packed; +}; + +template +class BlockPackKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + constexpr static bool EnableAutoTuning() { return false; } + using DataType = T; + constexpr static std::string_view Name = "block_delta_pack"; + MGARDX_CONT BlockPackKernel(SubArray<1, T, DeviceType> data, SIZE n, + SIZE block_size, SIZE nblocks, Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, size_t, DeviceType> byte_offset, + SubArray<1, Byte, DeviceType> packed) + : data(data), n(n), block_size(block_size), nblocks(nblocks), mode(mode), + bitwidth(bitwidth), byte_offset(byte_offset), packed(packed) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = BlockPackFunctor; + FunctorType functor(data, n, block_size, nblocks, mode, bitwidth, + byte_offset, packed); + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (nblocks - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, tbz, tby, tbx, + functor.shared_memory_size(), queue_idx, std::string(Name)); + } + +private: + SubArray<1, T, DeviceType> data; + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, size_t, DeviceType> byte_offset; + SubArray<1, Byte, DeviceType> packed; +}; + +// --------------------------------------------------------------------------- +// Kernel 3: rebuild per-block byte-count from bit-width (+oc) -- decode sizing. +// --------------------------------------------------------------------------- +template +class BlockBytecountFunctor : public Functor { +public: + MGARDX_CONT BlockBytecountFunctor() {} + MGARDX_CONT BlockBytecountFunctor(SIZE n, SIZE block_size, SIZE nblocks, + Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, uint16_t, DeviceType> oc, + SubArray<1, size_t, DeviceType> bytecount) + : n(n), block_size(block_size), nblocks(nblocks), mode(mode), + bitwidth(bitwidth), oc(oc), bytecount(bytecount) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE b = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (b >= nblocks) + return; + SIZE start = b * block_size; + SIZE len = block_size < (n - start) ? block_size : (n - start); + int bw = (int)*bitwidth(b); + if (mode == block_delta::MODE_OUTLIER) { + *bytecount(b) = 2 + block_delta::block_bytes(bw, len) + + (size_t)(*oc(b)) * block_delta::outlier_rec_bytes(); + } else { + *bytecount(b) = block_delta::block_bytes(bw, len); + } + } + + MGARDX_EXEC void Operation2() {} + MGARDX_EXEC void Operation3() {} + MGARDX_EXEC void Operation4() {} + MGARDX_EXEC void Operation5() {} + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, uint16_t, DeviceType> oc; + SubArray<1, size_t, DeviceType> bytecount; +}; + +template +class BlockBytecountKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + constexpr static bool EnableAutoTuning() { return false; } + using DataType = Byte; + constexpr static std::string_view Name = "block_delta_bytecount"; + MGARDX_CONT BlockBytecountKernel(SIZE n, SIZE block_size, SIZE nblocks, + Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, uint16_t, DeviceType> oc, + SubArray<1, size_t, DeviceType> bytecount) + : n(n), block_size(block_size), nblocks(nblocks), mode(mode), + bitwidth(bitwidth), oc(oc), bytecount(bytecount) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = BlockBytecountFunctor; + FunctorType functor(n, block_size, nblocks, mode, bitwidth, oc, bytecount); + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (nblocks - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, tbz, tby, tbx, + functor.shared_memory_size(), queue_idx, std::string(Name)); + } + +private: + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, uint16_t, DeviceType> oc; + SubArray<1, size_t, DeviceType> bytecount; +}; + +// --------------------------------------------------------------------------- +// Kernel 4: unpack each block back into the signed quantized stream. +// --------------------------------------------------------------------------- +template +class BlockUnpackFunctor : public Functor { +public: + MGARDX_CONT BlockUnpackFunctor() {} + MGARDX_CONT BlockUnpackFunctor(SubArray<1, Byte, DeviceType> packed, SIZE n, + SIZE block_size, SIZE nblocks, Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, size_t, DeviceType> byte_offset, + SubArray<1, T, DeviceType> data) + : packed(packed), n(n), block_size(block_size), nblocks(nblocks), + mode(mode), bitwidth(bitwidth), byte_offset(byte_offset), data(data) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + using UT = typename std::make_unsigned::type; + SIZE b = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (b >= nblocks) + return; + int bw = (int)*bitwidth(b); + SIZE start = b * block_size; + SIZE len = block_size < (n - start) ? block_size : (n - start); + bool use_delta = (mode != block_delta::MODE_FIXED); + bool outlier = (mode == block_delta::MODE_OUTLIER); + Byte *in = packed(*byte_offset(b)); + + size_t main_start = 0; + int oc = 0; + if (outlier) { + oc = (int)in[0] | ((int)in[1] << 8); + main_start = 2; + } + size_t rec_start = main_start + block_delta::block_bytes(bw, len); + constexpr int RECSZ = 2 + (int)sizeof(T); + int cursor = 0; + long next_pos = -1; + if (outlier && oc > 0) + next_pos = (long)in[rec_start] | ((long)in[rec_start + 1] << 8); + + T prev = 0; + Byte cur = 0; + int cnt = 0; + size_t pos = main_start; + for (SIZE i = 0; i < len; i++) { + UT z = 0; + for (int k = 0; k < bw; k++) { + if (cnt == 0) { + cur = in[pos++]; + cnt = 8; + } + z |= (UT)(cur & 1) << k; + cur >>= 1; + cnt--; + } + if (outlier && (long)i == next_pos) { + size_t ro = rec_start + (size_t)cursor * RECSZ + 2; + UT v = 0; + for (int k = 0; k < (int)sizeof(T); k++) + v |= (UT)in[ro + k] << (8 * k); + z = v; + cursor++; + next_pos = + (cursor < oc) + ? ((long)in[rec_start + (size_t)cursor * RECSZ] | + ((long)in[rec_start + (size_t)cursor * RECSZ + 1] << 8)) + : -1; + } + T d = block_delta::unzigzag(z); + prev = use_delta ? (T)(prev + d) : d; + *data(start + i) = prev; + } + } + + MGARDX_EXEC void Operation2() {} + MGARDX_EXEC void Operation3() {} + MGARDX_EXEC void Operation4() {} + MGARDX_EXEC void Operation5() {} + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Byte, DeviceType> packed; + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, size_t, DeviceType> byte_offset; + SubArray<1, T, DeviceType> data; +}; + +template +class BlockUnpackKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + constexpr static bool EnableAutoTuning() { return false; } + using DataType = T; + constexpr static std::string_view Name = "block_delta_unpack"; + MGARDX_CONT BlockUnpackKernel(SubArray<1, Byte, DeviceType> packed, SIZE n, + SIZE block_size, SIZE nblocks, Byte mode, + SubArray<1, Byte, DeviceType> bitwidth, + SubArray<1, size_t, DeviceType> byte_offset, + SubArray<1, T, DeviceType> data) + : packed(packed), n(n), block_size(block_size), nblocks(nblocks), + mode(mode), bitwidth(bitwidth), byte_offset(byte_offset), data(data) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = BlockUnpackFunctor; + FunctorType functor(packed, n, block_size, nblocks, mode, bitwidth, + byte_offset, data); + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (nblocks - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, tbz, tby, tbx, + functor.shared_memory_size(), queue_idx, std::string(Name)); + } + +private: + SubArray<1, Byte, DeviceType> packed; + SIZE n, block_size, nblocks; + Byte mode; + SubArray<1, Byte, DeviceType> bitwidth; + SubArray<1, size_t, DeviceType> byte_offset; + SubArray<1, T, DeviceType> data; +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Lossless/CPU.hpp b/include/mgard-x-160/Lossless/CPU.hpp new file mode 100644 index 000000000..06696eeee --- /dev/null +++ b/include/mgard-x-160/Lossless/CPU.hpp @@ -0,0 +1,283 @@ +#ifndef MGARD_X_160_CPU_LOSSLESS_TEMPLATE_HPP +#define MGARD_X_160_CPU_LOSSLESS_TEMPLATE_HPP + +#include +#include +#include + +#include "../RuntimeX/Utilities/Exceptions.h" + +/*! CHECK + * Check that the condition holds. If it doesn't, throw an exception so the + * calling application can handle the failure instead of the whole process + * being terminated. + */ +#define CHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + char mgard_x_check_msg[256]; \ + std::snprintf(mgard_x_check_msg, sizeof(mgard_x_check_msg), \ + "CHECK(" #cond ") failed: " __VA_ARGS__); \ + throw mgard_x_160::ProcessingException(std::string(__FILE__) + ":" + \ + std::to_string(__LINE__) + " " + \ + mgard_x_check_msg); \ + } \ + } while (0) + +/*! CHECK_ZSTD + * Check the zstd error code and die if an error occurred after printing a + * message. + */ +/*! CHECK_ZSTD + * Check the zstd error code and die if an error occurred after printing a + * message. + */ +#define CHECK_ZSTD(fn, ...) \ + do { \ + size_t const err = (fn); \ + CHECK(!ZSTD_isError(err), "%s", ZSTD_getErrorName(err)); \ + } while (0) + +namespace mgard { +void huffman_encoding(long int *quantized_data, const std::size_t n, + unsigned char **out_data_hit, size_t *out_data_hit_size, + unsigned char **out_data_miss, size_t *out_data_miss_size, + unsigned char **out_tree, size_t *out_tree_size); +void huffman_decoding(long int *quantized_data, + const std::size_t quantized_data_size, + unsigned char *out_data_hit, size_t out_data_hit_size, + unsigned char *out_data_miss, size_t out_data_miss_size, + unsigned char *out_tree, size_t out_tree_size); +} // namespace mgard + +namespace mgard_x_160 { + +template +unsigned char *compress_memory_huffman(long int *const src, + const std::size_t srcLen, + std::size_t &outsize) { + unsigned char *out_data_hit = 0; + size_t out_data_hit_size; + unsigned char *out_data_miss = 0; + size_t out_data_miss_size; + unsigned char *out_tree = 0; + size_t out_tree_size; + ::mgard::huffman_encoding(src, srcLen, &out_data_hit, &out_data_hit_size, + &out_data_miss, &out_data_miss_size, &out_tree, + &out_tree_size); + + const size_t total_size = + out_data_hit_size / 8 + 4 + out_data_miss_size + out_tree_size; + unsigned char *payload = (unsigned char *)malloc(total_size); + unsigned char *bufp = payload; + + if (out_tree_size) { + std::memcpy(bufp, out_tree, out_tree_size); + bufp += out_tree_size; + } + + std::memcpy(bufp, out_data_hit, out_data_hit_size / 8 + 4); + bufp += out_data_hit_size / 8 + 4; + + if (out_data_miss_size) { + std::memcpy(bufp, out_data_miss, out_data_miss_size); + bufp += out_data_miss_size; + } + + free(out_tree); + free(out_data_hit); + free(out_data_miss); + + const size_t cBuffSize = ZSTD_compressBound(total_size); + unsigned char *const zstd_buffer = new unsigned char[cBuffSize]; + + const std::size_t cSize = + ZSTD_compress(zstd_buffer, cBuffSize, payload, total_size, 1); + CHECK_ZSTD(cSize); + + free(payload); + payload = 0; + + const std::size_t bufferLen = 3 * sizeof(size_t) + cSize; + unsigned char *const buffer = new unsigned char[bufferLen]; + outsize = bufferLen; + + bufp = buffer; + *(size_t *)bufp = out_tree_size; + bufp += sizeof(size_t); + + *(size_t *)bufp = out_data_hit_size; + bufp += sizeof(size_t); + + *(size_t *)bufp = out_data_miss_size; + bufp += sizeof(size_t); + + { + unsigned char const *const p = zstd_buffer; + std::copy(p, p + cSize, bufp); + } + + { + unsigned char *buf = buffer; + out_tree_size = *(size_t *)buf; + buf += sizeof(size_t); + + out_data_hit_size = *(size_t *)buf; + buf += sizeof(size_t); + + out_data_miss_size = *(size_t *)buf; + buf += sizeof(size_t); + } + + return buffer; +} + +template +void decompress_memory_huffman(unsigned char *const src, + const std::size_t srcLen, long int *const dst, + const std::size_t dstLen) { + unsigned char *out_data_hit = 0; + size_t out_data_hit_size; + unsigned char *out_data_miss = 0; + size_t out_data_miss_size; + unsigned char *out_tree = 0; + size_t out_tree_size; + + unsigned char *buf = src; + + out_tree_size = *(size_t *)buf; + buf += sizeof(size_t); + + out_data_hit_size = *(size_t *)buf; + buf += sizeof(size_t); + + out_data_miss_size = *(size_t *)buf; + buf += sizeof(size_t); + + size_t total_huffman_size = + out_tree_size + out_data_hit_size / 8 + 4 + out_data_miss_size; + unsigned char *huffman_encoding_p = + (unsigned char *)malloc(total_huffman_size); + + size_t const dSize = ZSTD_decompress(huffman_encoding_p, total_huffman_size, + buf, srcLen - 3 * sizeof(size_t)); + CHECK_ZSTD(dSize); + + /* When zstd knows the content size, it will error if it doesn't match. */ + CHECK(total_huffman_size == dSize, + "Impossible because zstd will check this condition!"); + + out_tree = huffman_encoding_p; + out_data_hit = huffman_encoding_p + out_tree_size; + out_data_miss = + huffman_encoding_p + out_tree_size + out_data_hit_size / 8 + 4; + + mgard::huffman_decoding(dst, dstLen, out_data_hit, out_data_hit_size, + out_data_miss, out_data_miss_size, out_tree, + out_tree_size); + + free(huffman_encoding_p); +} + +template +Array<1, Byte, DeviceType> CPUCompress(SubArray<1, C, DeviceType> &input_data) { + + // PrintSubarray("CPUCompress input", input_data); + Timer timer; + if (log::level & log::TIME) + timer.start(); + size_t input_count = input_data.shape(0); + + C *in_data = NULL; + MemoryManager::MallocHost(in_data, input_count, 0); + MemoryManager::Copy1D(in_data, input_data.data(), input_count, 0); + DeviceRuntime::SyncQueue(0); + + std::vector qv(input_count); + for (size_t i = 0; i < input_count; i++) { + qv[i] = (long int)in_data[i]; + } + + std::size_t actual_out_size; + unsigned char *lossless_data = compress_memory_huffman( + qv.data(), qv.size(), actual_out_size); + + uint8_t *out_data = NULL; + MemoryManager::MallocHost(out_data, + actual_out_size + sizeof(size_t), 0); + + *(size_t *)out_data = (size_t)input_count; + std::memcpy(out_data + sizeof(size_t), lossless_data, actual_out_size); + + Array<1, Byte, DeviceType> output_data( + {(SIZE)(actual_out_size + sizeof(size_t))}); + output_data.load(out_data); + + MemoryManager::FreeHost(out_data); + MemoryManager::FreeHost(in_data); + delete[] lossless_data; + + // PrintSubarray("CPUCompress output", SubArray(output_data)); + log::info( + "CPU lossless compress ratio: " + + std::to_string(input_count * sizeof(C)) + "/" + + std::to_string(output_data.shape(0)) + " (" + + std::to_string((double)input_count * sizeof(C) / output_data.shape(0)) + + ")"); + if (log::level & log::TIME) { + DeviceRuntime::SyncDevice(); + timer.end(); + timer.print("CPU lossless"); + timer.clear(); + } + return output_data; +} + +template +Array<1, C, DeviceType> +CPUDecompress(SubArray<1, Byte, DeviceType> &input_data) { + Timer timer; + if (log::level & log::TIME) + timer.start(); + // PrintSubarray("CPUDecompress input", input_data); + size_t input_count = input_data.shape(0); + Byte *in_data = NULL; + MemoryManager::MallocHost(in_data, input_count, 0); + MemoryManager::Copy1D(in_data, input_data.data(), input_count, 0); + DeviceRuntime::SyncQueue(0); + + uint32_t actual_out_count = 0; + actual_out_count = *reinterpret_cast(in_data); + // *oriData = (uint8_t*)malloc(outSize); + C *out_data = NULL; + MemoryManager::MallocHost(out_data, actual_out_count, 0); + DeviceRuntime::SyncQueue(0); + + long int *qv = new long int[actual_out_count]; + size_t out_size = actual_out_count * sizeof(long int); + decompress_memory_huffman( + in_data + sizeof(size_t), input_count - sizeof(size_t), qv, out_size); + + for (size_t i = 0; i < actual_out_count; i++) { + out_data[i] = (C)qv[i]; + } + + Array<1, C, DeviceType> output_data({(SIZE)actual_out_count}); + output_data.load(out_data); + + MemoryManager::FreeHost(out_data); + MemoryManager::FreeHost(in_data); + + // PrintSubarray("CPUDecompress output", SubArray(output_data)); + if (log::level & log::TIME) { + DeviceRuntime::SyncDevice(); + timer.end(); + timer.print("CPU lossless"); + timer.clear(); + } + return output_data; +} + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/LZ4/LZ4.hpp b/include/mgard-x-160/Lossless/LZ4/LZ4.hpp new file mode 100644 index 000000000..5dd22bbcb --- /dev/null +++ b/include/mgard-x-160/Lossless/LZ4/LZ4.hpp @@ -0,0 +1,329 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_LZ4_HPP +#define MGARD_X_160_LZ4_HPP + +#include "../../RuntimeX/Utilities/Serializer.hpp" +#include "LZ4Fused.hpp" // CUDA warp-per-chunk compress kernel (optional fast path) +#include "LZ4Kernels.hpp" + +namespace mgard_x_160 { + +// Portable, nvcomp-free LZ4 (warp-cooperative on GPU via LZ4Fused, the size-1 +// SubGroupScalar functor on CPU). In-place Compress/Decompress(Array<1,Byte>&): +// the composed pipeline feeds it the Huffman output (Huffman_LZ4) or the raw +// quantized byte stream (standalone LZ4), and gets it back. The buffer is +// fully self-describing -- Compress writes the header; Decompress parses it -- +// so there is no separate Serialize stage (unlike BlockDelta, the upstream +// caller already serializes Huffman before handing us the byte stream). +// +// Container layout written by Compress: +// signature(8) | uncompressed_total:size_t | chunk_size:size_t | +// nchunks:size_t | comp_bytes[nchunks]:size_t | packed_bytes:size_t | +// packed[] +// comp_bytes[] is the per-chunk compressed length; Decompress exclusive-scans +// it to recover each chunk's offset into packed[] (same trick as BlockDelta). +template class LZ4 { +public: + LZ4() : initialized(false) {} + + // The warp-per-chunk CUDA compress kernel (LZ4Fused) replaces the portable + // 1-thread-per-chunk functor when available; it keeps its hash table in + // shared memory, so the global htable_array is not needed on that path. True + // for the GPU backend (CUDA/HIP) that has the fused warp-per-chunk path + // compiled in this TU. SERIAL/OpenMP/other always use the portable functor. + static constexpr bool fused_backend() { + bool r = false; +#if defined(MGARDX_COMPILE_CUDA) + r = r || std::is_same::value; +#endif +#if defined(MGARDX_COMPILE_HIP) + r = r || std::is_same::value; +#endif +#if defined(MGARDX_COMPILE_SYCL) + r = r || std::is_same::value; +#endif + return r; + } + + bool uses_fused() const { +#if defined(MGARDX_COMPILE_CUDA) || defined(MGARDX_COMPILE_HIP) || \ + defined(MGARDX_COMPILE_SYCL) + return fused_backend() && lz4_fused::fused_ok(chunk_size); +#else + return false; +#endif + } + + LZ4(SIZE n, SIZE chunk_size) { + Resize(n, chunk_size, 0); + DeviceRuntime::SyncQueue(0); + } + + void Resize(SIZE n, SIZE chunk_size, int queue_idx) { + this->initialized = true; + this->max_size = n; + this->chunk_size = chunk_size; + SIZE max_nchunks = (n - 1) / chunk_size + 1; + this->max_chunk_out = (SIZE)lz4::compress_bound(chunk_size); + + input_data = Array<1, Byte, DeviceType>({n}); + // Global per-chunk hash table only feeds the portable functor path; the + // fused CUDA kernel uses shared memory instead, so skip the (large) alloc. + if (!uses_fused()) + htable_array = Array<1, uint16_t, DeviceType>( + {(SIZE)((size_t)max_nchunks * lz4::HASH_SIZE)}); + scratch_array = Array<1, Byte, DeviceType>( + {(SIZE)((size_t)max_nchunks * max_chunk_out)}); + comp_bytes_array = Array<1, size_t, DeviceType>({max_nchunks}); + byte_offset_array = Array<1, size_t, DeviceType>({max_nchunks + 1}); + DeviceCollective::ScanSumExtended( + max_nchunks, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), scan_workspace, false, queue_idx); + } + + static size_t EstimateMemoryFootprint(SIZE n, SIZE chunk_size) { + SIZE max_nchunks = (n - 1) / chunk_size + 1; + size_t max_chunk_out = lz4::compress_bound(chunk_size); + size_t size = n; // input + size += (size_t)max_nchunks * lz4::HASH_SIZE * sizeof(uint16_t); // htable + size += (size_t)max_nchunks * max_chunk_out; // scratch + size += (size_t)max_nchunks * sizeof(size_t); // comp_bytes + size += (size_t)(max_nchunks + 1) * sizeof(size_t); // offsets + return size; + } + + void Compress(Array<1, Byte, DeviceType> &data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + // Snapshot the input; `data` is reused as the compressed output. + SIZE input_count = data.shape(0); + n = input_count; + nchunks = (n - 1) / chunk_size + 1; + input_data.resize({n}, queue_idx); + MemoryManager::Copy1D(input_data.data(), data.data(), n, + queue_idx); + + SubArray<1, Byte, DeviceType> input_subarray({(SIZE)n}, input_data.data()); + SubArray<1, Byte, DeviceType> scratch_subarray( + {(SIZE)((size_t)nchunks * max_chunk_out)}, scratch_array.data()); + SubArray<1, size_t, DeviceType> comp_bytes_subarray( + {(SIZE)nchunks}, comp_bytes_array.data()); + SubArray<1, size_t, DeviceType> byte_offset_subarray( + {(SIZE)nchunks + 1}, byte_offset_array.data()); + + // 1) compress each chunk into its scratch slot; record per-chunk size. + // CUDA: warp-per-chunk fused kernel (shared-memory hash, parallel match + // scan). Other backends / oversized chunks: portable 1-thread functor. + bool did_fused = false; +#if defined(MGARDX_COMPILE_CUDA) || defined(MGARDX_COMPILE_HIP) || \ + defined(MGARDX_COMPILE_SYCL) + if constexpr (fused_backend()) { + if (lz4_fused::fused_ok(chunk_size)) { + auto stream = DeviceRuntime::GetQueue(queue_idx); + lz4_fused::launch_compress(input_subarray.data(), (SIZE)n, + (int)chunk_size, (SIZE)nchunks, + (SIZE)max_chunk_out, scratch_subarray.data(), + comp_bytes_subarray.data(), stream); + did_fused = true; + } + } +#endif + if (!did_fused) { + SubArray<1, uint16_t, DeviceType> htable_subarray( + {(SIZE)((size_t)nchunks * lz4::HASH_SIZE)}, htable_array.data()); + DeviceLauncher::Execute( + LZ4ChunkCompressKernel( + input_subarray, (SIZE)n, (SIZE)chunk_size, (SIZE)nchunks, + (SIZE)max_chunk_out, htable_subarray, scratch_subarray, + comp_bytes_subarray), + queue_idx); + } + + // 2) exclusive scan of per-chunk sizes -> offsets into packed[] (+ total). + DeviceCollective::ScanSumExtended( + (SIZE)nchunks, comp_bytes_subarray, byte_offset_subarray, + scan_workspace, true, queue_idx); + MemoryManager::Copy1D( + &packed_bytes, byte_offset_subarray.data() + nchunks, 1, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + // 3) size the output and 4) gather chunks into the packed region. + SIZE packed_byte_offset; + SIZE compressed_size = + ComputeLayout(nchunks, packed_bytes, packed_byte_offset); + data.resize({compressed_size}, queue_idx); + SubArray<1, Byte, DeviceType> out_subarray(data); + SubArray<1, Byte, DeviceType> packed_subarray( + {(SIZE)packed_bytes}, (Byte *)out_subarray(packed_byte_offset)); + DeviceLauncher::Execute( + LZ4CondenseKernel(scratch_subarray, (SIZE)nchunks, + (SIZE)max_chunk_out, comp_bytes_subarray, + byte_offset_subarray, packed_subarray), + queue_idx); + + // Header (everything except packed[], which the condense kernel wrote). + size_t n_v = n, chunk_v = chunk_size, nchunks_v = nchunks, + packed_v = packed_bytes; + SIZE byte_offset = 0; + SerializeArray(out_subarray, signature, kSignatureLen, byte_offset, + queue_idx); + SerializeArray(out_subarray, &n_v, 1, byte_offset, queue_idx); + SerializeArray(out_subarray, &chunk_v, 1, byte_offset, queue_idx); + SerializeArray(out_subarray, &nchunks_v, 1, byte_offset, queue_idx); + SerializeArray(out_subarray, comp_bytes_array.data(), nchunks, + byte_offset, queue_idx); + SerializeArray(out_subarray, &packed_v, 1, byte_offset, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + log::info("LZ4 compress ratio: " + std::to_string(input_count) + "/" + + std::to_string(compressed_size) + " (" + + std::to_string((double)input_count / compressed_size) + ")"); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("LZ4 compress", input_count); + timer.clear(); + } + } + + void Decompress(Array<1, Byte, DeviceType> &data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + // Snapshot the compressed input; `data` is reused as the output. + input_data.resize({data.shape(0)}, queue_idx); + MemoryManager::Copy1D(input_data.data(), data.data(), + data.shape(0), queue_idx); + SubArray<1, Byte, DeviceType> in_subarray(input_data); + + // Parse header. comp_bytes_ptr / packed_ptr are zero-copy into in_subarray. + SIZE byte_offset = 0; + Byte *sig = signature_verify; + DeserializeArray(in_subarray, sig, kSignatureLen, byte_offset, false, + queue_idx); + size_t *n_ptr = &n, *chunk_ptr = &chunk_size_runtime, + *nchunks_ptr = &nchunks, *packed_ptr_sz = &packed_bytes; + DeserializeArray(in_subarray, n_ptr, 1, byte_offset, false, + queue_idx); + DeserializeArray(in_subarray, chunk_ptr, 1, byte_offset, false, + queue_idx); + DeserializeArray(in_subarray, nchunks_ptr, 1, byte_offset, false, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + for (int i = 0; i < kSignatureLen; i++) + if (signature_verify[i] != signature[i]) + throw std::runtime_error("LZ4 signature mismatch."); + + size_t *comp_bytes_ptr = nullptr; + DeserializeArray(in_subarray, comp_bytes_ptr, nchunks, byte_offset, + true, queue_idx); // zero-copy device pointer + DeserializeArray(in_subarray, packed_ptr_sz, 1, byte_offset, false, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + Byte *packed_ptr = nullptr; + DeserializeArray(in_subarray, packed_ptr, packed_bytes, byte_offset, + true, queue_idx); + + SubArray<1, size_t, DeviceType> comp_bytes_subarray({(SIZE)nchunks}, + comp_bytes_ptr); + SubArray<1, size_t, DeviceType> byte_offset_subarray( + {(SIZE)nchunks + 1}, byte_offset_array.data()); + SubArray<1, Byte, DeviceType> packed_subarray({(SIZE)packed_bytes}, + packed_ptr); + + // Rebuild per-chunk offsets into packed[] from the stored sizes. + DeviceCollective::ScanSumExtended( + (SIZE)nchunks, comp_bytes_subarray, byte_offset_subarray, + scan_workspace, true, queue_idx); + + data.resize({(SIZE)n}, queue_idx); + SubArray<1, Byte, DeviceType> out_subarray(data); + // CUDA: warp-per-chunk fused decoder (cooperative literal/match copy). + // Other backends / oversized chunks: portable 1-thread decode functor. + bool did_fused = false; +#if defined(MGARDX_COMPILE_CUDA) || defined(MGARDX_COMPILE_HIP) || \ + defined(MGARDX_COMPILE_SYCL) + if constexpr (fused_backend()) { + if (lz4_fused::fused_ok(chunk_size_runtime)) { + auto stream = DeviceRuntime::GetQueue(queue_idx); + lz4_fused::launch_decompress(packed_subarray.data(), + byte_offset_subarray.data(), (SIZE)n, + (int)chunk_size_runtime, (SIZE)nchunks, + out_subarray.data(), stream); + did_fused = true; + } + } +#endif + if (!did_fused) { + DeviceLauncher::Execute( + LZ4ChunkDecompressKernel( + packed_subarray, byte_offset_subarray, (SIZE)n, + (SIZE)chunk_size_runtime, (SIZE)nchunks, out_subarray), + queue_idx); + } + DeviceRuntime::SyncQueue(queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("LZ4 decompress", n); + timer.clear(); + } + } + +private: + // Layout walker; mirrors the header order written by Compress. Returns the + // total compressed size and, via the out-param, where packed[] begins. + SIZE ComputeLayout(size_t nchunks, size_t packed_bytes, + SIZE &packed_byte_offset) { + SIZE off = 0; + advance_with_align(off, kSignatureLen); // signature + advance_with_align(off, 1); // n + advance_with_align(off, 1); // chunk_size + advance_with_align(off, 1); // nchunks + advance_with_align(off, nchunks); // comp_bytes[] + advance_with_align(off, 1); // packed_bytes + align_byte_offset(off); + packed_byte_offset = off; + advance_with_align(off, packed_bytes); // packed[] + return off; + } + +public: + bool initialized; + SIZE max_size; + SIZE chunk_size; + SIZE max_chunk_out; + size_t n = 0; + size_t nchunks = 0; + size_t packed_bytes = 0; + size_t chunk_size_runtime = 0; // chunk_size read back on Decompress + +private: + static constexpr int kSignatureLen = 8; + Byte signature[8] = {'M', 'G', 'X', 'L', 'Z', '4', 'P', '\0'}; + Byte signature_verify[8] = {0}; + + Array<1, Byte, DeviceType> input_data; + Array<1, uint16_t, DeviceType> htable_array; + Array<1, Byte, DeviceType> scratch_array; + Array<1, size_t, DeviceType> comp_bytes_array; + Array<1, size_t, DeviceType> byte_offset_array; + Array<1, Byte, DeviceType> scan_workspace; +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Lossless/LZ4/LZ4Fused.hpp b/include/mgard-x-160/Lossless/LZ4/LZ4Fused.hpp new file mode 100644 index 000000000..6dd2e0b5c --- /dev/null +++ b/include/mgard-x-160/Lossless/LZ4/LZ4Fused.hpp @@ -0,0 +1,177 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_LZ4_FUSED_HPP +#define MGARD_X_160_LZ4_FUSED_HPP + +#include "LZ4Kernels.hpp" // lz4_coop:: cooperative codec (SubGroup<> lives in the DeviceAdapters) + +// GPU launch wrappers for the portable LZ4 backend: one sub-group (warp on +// CUDA, wavefront on HIP) per chunk, running the SAME lz4_coop::compress_chunk +// / decompress_chunk used by every backend -- here with the GPU sub-group, so +// the hash table lives in shared memory and the match scan / byte copies run +// across lanes. The portable functor path (LZ4Kernels) runs the identical codec +// with a size-1 SubGroupScalar. Both __global__ kernels are templated so their +// definitions have weak linkage and don't clash across GPU TUs (a non-template +// __global__ in a header -> nvlink "merge_elf failed"). chunk_size must be +// <= 65535 (uint16 hash slot). The HIP path is UNTESTED (no AMD hardware here). + +#if defined(MGARDX_COMPILE_CUDA) || defined(MGARDX_COMPILE_HIP) + +namespace mgard_x_160 { +namespace lz4_fused { + +// Per-backend sub-group type and stream type. +#if defined(MGARDX_COMPILE_CUDA) +using FusedSubGroup = SubGroup; +using gpuStream_t = cudaStream_t; +#else +using FusedSubGroup = SubGroup; +using gpuStream_t = hipStream_t; +#endif + +static constexpr int WARPS_PER_BLOCK = 4; +static constexpr int SG = FusedSubGroup::size(); + +template +__global__ void compress_kernel(const Byte *__restrict__ input, SIZE n, + int chunk_size, SIZE nchunks, + SIZE max_chunk_out, Byte *__restrict__ scratch, + size_t *__restrict__ comp_bytes) { + constexpr int HS = 1 << lz4::HASH_LOG; + __shared__ uint16_t s_ht[WPB][HS]; + const int warp = threadIdx.x / FusedSubGroup::size(); + const SIZE c = (SIZE)blockIdx.x * WPB + warp; + if (c >= nchunks) + return; + const SIZE start = c * (SIZE)chunk_size; + const int len = + (int)((SIZE)chunk_size < (n - start) ? (SIZE)chunk_size : (n - start)); + FusedSubGroup sg; + int cb = lz4_coop::compress_chunk( + sg, input + start, len, scratch + (size_t)c * max_chunk_out, s_ht[warp]); + if (sg.lane() == 0) + comp_bytes[c] = (size_t)cb; +} + +template +__global__ void decompress_kernel(const Byte *__restrict__ packed, + const size_t *__restrict__ byte_offset, + SIZE n, int chunk_size, SIZE nchunks, + Byte *__restrict__ output) { + const int warp = threadIdx.x / FusedSubGroup::size(); + const SIZE c = (SIZE)blockIdx.x * WPB + warp; + if (c >= nchunks) + return; + const SIZE start = c * (SIZE)chunk_size; + const int outLen = + (int)((SIZE)chunk_size < (n - start) ? (SIZE)chunk_size : (n - start)); + FusedSubGroup sg; + lz4_coop::decompress_chunk(sg, packed + byte_offset[c], output + start, + outLen); +} + +inline void launch_compress(const Byte *input, SIZE n, int chunk_size, + SIZE nchunks, SIZE max_chunk_out, Byte *scratch, + size_t *comp_bytes, gpuStream_t stream) { + dim3 block(WARPS_PER_BLOCK * SG); + dim3 grid((unsigned)((nchunks + WARPS_PER_BLOCK - 1) / WARPS_PER_BLOCK)); + compress_kernel<<>>( + input, n, chunk_size, nchunks, max_chunk_out, scratch, comp_bytes); +} + +inline void launch_decompress(const Byte *packed, const size_t *byte_offset, + SIZE n, int chunk_size, SIZE nchunks, + Byte *output, gpuStream_t stream) { + dim3 block(WARPS_PER_BLOCK * SG); + dim3 grid((unsigned)((nchunks + WARPS_PER_BLOCK - 1) / WARPS_PER_BLOCK)); + decompress_kernel<<>>( + packed, byte_offset, n, chunk_size, nchunks, output); +} + +// Fused chunks are valid only when a position fits the uint16 hash slot. +inline bool fused_ok(SIZE chunk_size) { return chunk_size <= 65535; } + +} // namespace lz4_fused +} // namespace mgard_x_160 + +#endif // CUDA || HIP + +// --------------------------------------------------------------------------- +// SYCL path (oneAPI/DPC++). UNTESTED -- no Intel GPU / SYCL toolchain here. +// SYCL can't use <<<>>>; it submits an nd_range parallel_for with a local +// accessor for the shared hash table. One sub-group per chunk runs the SAME +// lz4_coop codec via SubGroup. reqd_sub_group_size pins the width to 32. +// --------------------------------------------------------------------------- +#if defined(MGARDX_COMPILE_SYCL) +namespace mgard_x_160 { +namespace lz4_fused { + +static constexpr int WARPS_PER_BLOCK = 4; +static constexpr int SYCL_SG = 32; + +inline bool fused_ok(SIZE chunk_size) { return chunk_size <= 65535; } + +inline void launch_compress(const Byte *input, SIZE n, int chunk_size, + SIZE nchunks, SIZE max_chunk_out, Byte *scratch, + size_t *comp_bytes, sycl::queue q) { + constexpr int HS = 1 << lz4::HASH_LOG; + constexpr int WPB = WARPS_PER_BLOCK; + size_t groups = (nchunks + WPB - 1) / WPB; + sycl::range<1> global(groups * WPB * SYCL_SG), local(WPB * SYCL_SG); + q.submit([&](sycl::handler &h) { + sycl::local_accessor ht(sycl::range<1>(WPB * HS), h); + h.parallel_for( + sycl::nd_range<1>(global, local), + [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SYCL_SG)]] { + sycl::sub_group nsg = it.get_sub_group(); + int warp = (int)nsg.get_group_id()[0]; + SIZE c = (SIZE)it.get_group(0) * WPB + warp; + if (c >= nchunks) + return; + SIZE start = c * (SIZE)chunk_size; + int len = (int)((SIZE)chunk_size < (n - start) ? (SIZE)chunk_size + : (n - start)); + SubGroup sg(nsg); + uint16_t *htp = &ht[(size_t)warp * HS]; + int cb = lz4_coop::compress_chunk( + sg, input + start, len, scratch + (size_t)c * max_chunk_out, htp); + if (sg.lane() == 0) + comp_bytes[c] = (size_t)cb; + }); + }); +} + +inline void launch_decompress(const Byte *packed, const size_t *byte_offset, + SIZE n, int chunk_size, SIZE nchunks, + Byte *output, sycl::queue q) { + constexpr int WPB = WARPS_PER_BLOCK; + size_t groups = (nchunks + WPB - 1) / WPB; + sycl::range<1> global(groups * WPB * SYCL_SG), local(WPB * SYCL_SG); + q.submit([&](sycl::handler &h) { + h.parallel_for( + sycl::nd_range<1>(global, local), + [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(SYCL_SG)]] { + sycl::sub_group nsg = it.get_sub_group(); + int warp = (int)nsg.get_group_id()[0]; + SIZE c = (SIZE)it.get_group(0) * WPB + warp; + if (c >= nchunks) + return; + SIZE start = c * (SIZE)chunk_size; + int outLen = (int)((SIZE)chunk_size < (n - start) ? (SIZE)chunk_size + : (n - start)); + SubGroup sg(nsg); + lz4_coop::decompress_chunk(sg, packed + byte_offset[c], + output + start, outLen); + }); + }); +} + +} // namespace lz4_fused +} // namespace mgard_x_160 +#endif // SYCL + +#endif diff --git a/include/mgard-x-160/Lossless/LZ4/LZ4Kernels.hpp b/include/mgard-x-160/Lossless/LZ4/LZ4Kernels.hpp new file mode 100644 index 000000000..db2049782 --- /dev/null +++ b/include/mgard-x-160/Lossless/LZ4/LZ4Kernels.hpp @@ -0,0 +1,505 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_LZ4_KERNELS_HPP +#define MGARD_X_160_LZ4_KERNELS_HPP + +#include "../../RuntimeX/RuntimeX.h" + +// Portable (multi-kernel) building blocks for a self-contained LZ4 backend -- +// a byte-oriented LZ77 dictionary codec, no external nvcomp dependency. +// +// The input byte stream is split into independent chunks of `chunk_size`. Each +// chunk is compressed/decompressed in isolation (its back-references never +// cross a chunk boundary), which makes the whole thing trivially parallel: one +// thread per chunk, no atomics, no cross-chunk state. This mirrors how nvcomp +// batches LZ4 and how BlockDelta gives each data-block a disjoint byte range. +// +// Compression is two phases because a chunk's compressed size is not known +// until it is actually compressed: +// 1) LZ4ChunkCompress : compress chunk c into a worst-case-sized scratch slot +// (scratch + c*max_chunk_out) and record comp_bytes[c]. +// 2) (host) exclusive scan comp_bytes -> byte_offset (final contiguous +// layout) 3) LZ4Condense : copy each chunk's comp_bytes[c] from its +// scratch slot +// to packed[byte_offset[c]] (the gather/compaction +// pass). +// Decompression is single-phase: rebuild byte_offset from the stored +// comp_bytes, then one thread per chunk parses tokens and copies. +// +// NOTE (perf): 1 thread per chunk is the portable correctness reference. With +// large chunks it is low-occupancy and keeps a per-thread hash table in global +// scratch. The optimization is one thread *block* per chunk (cooperative +// match-find + a shared-memory hash table), and/or a fused decoupled-look-back +// pass like BlockDeltaFused. Use a small chunk_size (4-8 KB) for this path. + +namespace mgard_x_160 { + +namespace lz4 { + +// ---- format / tuning constants ------------------------------------------- +enum : int { + MIN_MATCH = 4, // a back-reference must cover >= 4 bytes to pay off + LAST_LITERALS = 5, // final 5 bytes of a chunk are always emitted literally + MFLIMIT = 12, // last match must start >= 12 bytes before chunk end + MAX_DISTANCE = 65535, // 16-bit offset -> 64 KB window + HASH_LOG = 12, // per-chunk hash table: 1<> (32 - HASH_LOG); +} + +// Write an LZ4 length in the 255-continuation form. Returns the new write pos. +MGARDX_EXEC int write_len(Byte *out, int op, int length) { + while (length >= 255) { + out[op++] = (Byte)255; + length -= 255; + } + out[op++] = (Byte)length; + return op; +} + +} // namespace lz4 + +// --------------------------------------------------------------------------- +// ONE cooperative LZ4 codec, written against the portable SubGroup abstraction +// (mgard_x_160::SubGroup / SubGroupScalar, defined in the DeviceAdapter +// headers). A "sub-group" is a set of lanes running in lockstep that can shfl / +// ballot / ffs / sync. At size 1 (CPU) every collective is an identity and this +// exact code degenerates into the greedy serial encoder/decoder. Warp-size is +// handled via SG::size()/full_mask()/the mask typedef, so it is correct for any +// width -- the match scan examines size() positions per step (a wider sub-group +// gives slightly different, still-valid matches => CR can vary marginally with +// width; decode output is width-independent). +// --------------------------------------------------------------------------- +namespace lz4_coop { + +// Cooperative LZ4 encode of one chunk. `ht` is sub-group-shared scratch of +// lz4::HASH_SIZE uint16 slots (0 = empty, else position+1; needs chunk_size +// <= 65535). All lanes of `sg` call this together; returns the byte count. +// At size 1 this is exactly the greedy serial encoder. +template +MGARDX_EXEC int compress_chunk(SG sg, const Byte *in, int len, Byte *out, + uint16_t *ht) { + const int W = SG::size(); + const int lane = sg.lane(); + const int MF = lz4::MFLIMIT, LL = lz4::LAST_LITERALS, MM = lz4::MIN_MATCH; + const int MAXD = lz4::MAX_DISTANCE; + const int HS = (int)lz4::HASH_SIZE; + + for (int i = lane; i < HS; i += W) + ht[i] = 0; + sg.sync(); + + int ip = 0, anchor = 0, op = 0; + while (ip < len - MF) { + const int pos = ip + lane; + // Phase 1: does this lane's position have a valid 4-byte match? (no extend) + int rref = -1; + if (pos < len - MF) { + uint32_t seq = lz4::read4(in + pos); + uint16_t slot = ht[lz4::hash4(seq)]; + if (slot != 0) { + int r = (int)slot - 1; + if (pos - r <= MAXD && lz4::read4(in + r) == seq) + rref = r; + } + } + typename SG::mask_t ballot = sg.ballot(rref >= 0); + + if (ballot == 0) { + if (pos < len - 3) + ht[lz4::hash4(lz4::read4(in + pos))] = (uint16_t)(pos + 1); + sg.sync(); + ip += W; + continue; + } + + const int j = sg.ffs(ballot) - 1; // first (left-most) matching lane + const int mpos = ip + j; + const int r_j = sg.shfl(rref, j); + const int off_j = mpos - r_j; + + // Phase 2: cooperative match-length extension (W bytes per step). + int ml = MM; + { + int base = MM; + while (true) { + int q = base + lane, a = mpos + q, b = r_j + q; + int eq = (a < len - LL && in[b] == in[a]) ? 1 : 0; + typename SG::mask_t good = sg.ballot(eq); + if (good == sg.full_mask()) + base += W; + else { + ml = base + (sg.ffs(~good & sg.full_mask()) - 1); + break; + } + } + } + + // Record literal positions we passed [ip, mpos] for later windows. + if (lane <= j && pos < len - 3) + ht[lz4::hash4(lz4::read4(in + pos))] = (uint16_t)(pos + 1); + sg.sync(); + + // Emit: token | litlen-ext | literals | offset | matchlen-ext. + const int litlen = mpos - anchor; + const int numLitExt = litlen < 15 ? 0 : ((litlen - 15) / 255 + 1); + const int litStart = op + 1 + numLitExt; + const int mlf = ml - MM; + const int numMlExt = mlf < 15 ? 0 : ((mlf - 15) / 255 + 1); + const int after = litStart + litlen + 2 + numMlExt; + if (lane == 0) { + int tHi = litlen < 15 ? litlen : 15, tLo = mlf < 15 ? mlf : 15; + out[op] = (Byte)((tHi << 4) | tLo); + if (litlen >= 15) + lz4::write_len(out, op + 1, litlen - 15); + int offPos = litStart + litlen; + out[offPos] = (Byte)(off_j & 0xff); + out[offPos + 1] = (Byte)((off_j >> 8) & 0xff); + if (mlf >= 15) + lz4::write_len(out, offPos + 2, mlf - 15); + } + for (int k = lane; k < litlen; k += W) // cooperative literal copy + out[litStart + k] = in[anchor + k]; + sg.sync(); + + op = after; + ip = mpos + ml; + anchor = ip; + } + + // Terminal literal-only sequence [anchor, len). + const int litlen = len - anchor; + const int numLitExt = litlen < 15 ? 0 : ((litlen - 15) / 255 + 1); + const int litStart = op + 1 + numLitExt; + if (lane == 0) { + int tHi = litlen < 15 ? litlen : 15; + out[op] = (Byte)(tHi << 4); + if (litlen >= 15) + lz4::write_len(out, op + 1, litlen - 15); + } + for (int k = lane; k < litlen; k += W) + out[litStart + k] = in[anchor + k]; + sg.sync(); + return litStart + litlen; +} + +// Cooperative decode of one chunk. Lane 0 parses the serial token stream; the +// sub-group cooperatively copies literals and the match. Overlap-safe via the +// periodic seed out[op+k] = out[op-offset + (k mod offset)] (reads only the +// already-decoded seed). Driven by output length. Size 1 = serial decoder. +template +MGARDX_EXEC void decompress_chunk(SG sg, const Byte *in, Byte *out, + int outLen) { + const int W = SG::size(); + const int lane = sg.lane(); + const int MM = lz4::MIN_MATCH; + int ip = 0, op = 0; + while (op < outLen) { + int litlen = 0, lit_in = 0, off = 0, mlen = 0, is_term = 0; + if (lane == 0) { + Byte token = in[ip++]; + litlen = token >> 4; + if (litlen == 15) { + Byte b; + do { + b = in[ip++]; + litlen += b; + } while (b == 255); + } + lit_in = ip; + ip += litlen; + if (op + litlen >= outLen) { + is_term = 1; + } else { + off = (int)in[ip] | ((int)in[ip + 1] << 8); + ip += 2; + mlen = (token & 0xf) + MM; + if ((token & 0xf) == 15) { + Byte b; + do { + b = in[ip++]; + mlen += b; + } while (b == 255); + } + } + } + litlen = sg.shfl(litlen, 0); + lit_in = sg.shfl(lit_in, 0); + is_term = sg.shfl(is_term, 0); + off = sg.shfl(off, 0); + mlen = sg.shfl(mlen, 0); + + for (int k = lane; k < litlen; k += W) + out[op + k] = in[lit_in + k]; + op += litlen; + sg.sync(); + if (is_term) + break; + + const int seed = op - off; + if (off >= mlen) { + for (int k = lane; k < mlen; k += W) + out[op + k] = out[seed + k]; + } else { + for (int k = lane; k < mlen; k += W) + out[op + k] = out[seed + (k % off)]; + } + op += mlen; + sg.sync(); + } +} + +} // namespace lz4_coop + +// --------------------------------------------------------------------------- +// Kernel 1: compress each chunk into its scratch slot; record comp_bytes[c]. +// --------------------------------------------------------------------------- +template +class LZ4ChunkCompressFunctor : public Functor { +public: + MGARDX_CONT LZ4ChunkCompressFunctor() {} + MGARDX_CONT + LZ4ChunkCompressFunctor(SubArray<1, Byte, DeviceType> input, SIZE n, + SIZE chunk_size, SIZE nchunks, SIZE max_chunk_out, + SubArray<1, uint16_t, DeviceType> htable, + SubArray<1, Byte, DeviceType> scratch, + SubArray<1, size_t, DeviceType> comp_bytes) + : input(input), n(n), chunk_size(chunk_size), nchunks(nchunks), + max_chunk_out(max_chunk_out), htable(htable), scratch(scratch), + comp_bytes(comp_bytes) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE c = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (c >= nchunks) + return; + SIZE start = c * chunk_size; + int len = (int)(chunk_size < (n - start) ? chunk_size : (n - start)); + Byte *out = scratch((size_t)c * max_chunk_out); + uint16_t *ht = htable((size_t)c * lz4::HASH_SIZE); + int outlen = + lz4_coop::compress_chunk(SubGroupScalar{}, input(start), len, out, ht); + *comp_bytes(c) = (size_t)outlen; + } + + MGARDX_EXEC void Operation2() {} + MGARDX_EXEC void Operation3() {} + MGARDX_EXEC void Operation4() {} + MGARDX_EXEC void Operation5() {} + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Byte, DeviceType> input; + SIZE n, chunk_size, nchunks, max_chunk_out; + SubArray<1, uint16_t, DeviceType> htable; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, size_t, DeviceType> comp_bytes; +}; + +template class LZ4ChunkCompressKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + constexpr static bool EnableAutoTuning() { return false; } + using DataType = Byte; + constexpr static std::string_view Name = "lz4_chunk_compress"; + MGARDX_CONT LZ4ChunkCompressKernel(SubArray<1, Byte, DeviceType> input, + SIZE n, SIZE chunk_size, SIZE nchunks, + SIZE max_chunk_out, + SubArray<1, uint16_t, DeviceType> htable, + SubArray<1, Byte, DeviceType> scratch, + SubArray<1, size_t, DeviceType> comp_bytes) + : input(input), n(n), chunk_size(chunk_size), nchunks(nchunks), + max_chunk_out(max_chunk_out), htable(htable), scratch(scratch), + comp_bytes(comp_bytes) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = LZ4ChunkCompressFunctor; + FunctorType functor(input, n, chunk_size, nchunks, max_chunk_out, htable, + scratch, comp_bytes); + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (nchunks - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, tbz, tby, tbx, + functor.shared_memory_size(), queue_idx, std::string(Name)); + } + +private: + SubArray<1, Byte, DeviceType> input; + SIZE n, chunk_size, nchunks, max_chunk_out; + SubArray<1, uint16_t, DeviceType> htable; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, size_t, DeviceType> comp_bytes; +}; + +// --------------------------------------------------------------------------- +// Kernel 2: gather each chunk's compressed bytes into the contiguous output. +// --------------------------------------------------------------------------- +template +class LZ4CondenseFunctor : public Functor { +public: + MGARDX_CONT LZ4CondenseFunctor() {} + MGARDX_CONT LZ4CondenseFunctor(SubArray<1, Byte, DeviceType> scratch, + SIZE nchunks, SIZE max_chunk_out, + SubArray<1, size_t, DeviceType> comp_bytes, + SubArray<1, size_t, DeviceType> byte_offset, + SubArray<1, Byte, DeviceType> packed) + : scratch(scratch), nchunks(nchunks), max_chunk_out(max_chunk_out), + comp_bytes(comp_bytes), byte_offset(byte_offset), packed(packed) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE c = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (c >= nchunks) + return; + Byte *src = scratch((size_t)c * max_chunk_out); + Byte *dst = packed(*byte_offset(c)); + size_t cb = *comp_bytes(c); + // TUNING: serial per-thread copy. The cooperative (block-per-chunk) version + // copies this with the whole block striding over cb. + for (size_t k = 0; k < cb; k++) + dst[k] = src[k]; + } + + MGARDX_EXEC void Operation2() {} + MGARDX_EXEC void Operation3() {} + MGARDX_EXEC void Operation4() {} + MGARDX_EXEC void Operation5() {} + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Byte, DeviceType> scratch; + SIZE nchunks, max_chunk_out; + SubArray<1, size_t, DeviceType> comp_bytes; + SubArray<1, size_t, DeviceType> byte_offset; + SubArray<1, Byte, DeviceType> packed; +}; + +template class LZ4CondenseKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + constexpr static bool EnableAutoTuning() { return false; } + using DataType = Byte; + constexpr static std::string_view Name = "lz4_condense"; + MGARDX_CONT LZ4CondenseKernel(SubArray<1, Byte, DeviceType> scratch, + SIZE nchunks, SIZE max_chunk_out, + SubArray<1, size_t, DeviceType> comp_bytes, + SubArray<1, size_t, DeviceType> byte_offset, + SubArray<1, Byte, DeviceType> packed) + : scratch(scratch), nchunks(nchunks), max_chunk_out(max_chunk_out), + comp_bytes(comp_bytes), byte_offset(byte_offset), packed(packed) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = LZ4CondenseFunctor; + FunctorType functor(scratch, nchunks, max_chunk_out, comp_bytes, + byte_offset, packed); + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (nchunks - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, tbz, tby, tbx, + functor.shared_memory_size(), queue_idx, std::string(Name)); + } + +private: + SubArray<1, Byte, DeviceType> scratch; + SIZE nchunks, max_chunk_out; + SubArray<1, size_t, DeviceType> comp_bytes; + SubArray<1, size_t, DeviceType> byte_offset; + SubArray<1, Byte, DeviceType> packed; +}; + +// --------------------------------------------------------------------------- +// Kernel 3: decode each chunk back into the contiguous output stream. +// --------------------------------------------------------------------------- +template +class LZ4ChunkDecompressFunctor : public Functor { +public: + MGARDX_CONT LZ4ChunkDecompressFunctor() {} + MGARDX_CONT + LZ4ChunkDecompressFunctor(SubArray<1, Byte, DeviceType> packed, + SubArray<1, size_t, DeviceType> byte_offset, SIZE n, + SIZE chunk_size, SIZE nchunks, + SubArray<1, Byte, DeviceType> output) + : packed(packed), byte_offset(byte_offset), n(n), chunk_size(chunk_size), + nchunks(nchunks), output(output) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE c = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (c >= nchunks) + return; + SIZE start = c * chunk_size; + int len = (int)(chunk_size < (n - start) ? chunk_size : (n - start)); + lz4_coop::decompress_chunk(SubGroupScalar{}, packed(*byte_offset(c)), + output(start), len); + } + + MGARDX_EXEC void Operation2() {} + MGARDX_EXEC void Operation3() {} + MGARDX_EXEC void Operation4() {} + MGARDX_EXEC void Operation5() {} + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Byte, DeviceType> packed; + SubArray<1, size_t, DeviceType> byte_offset; + SIZE n, chunk_size, nchunks; + SubArray<1, Byte, DeviceType> output; +}; + +template class LZ4ChunkDecompressKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + constexpr static bool EnableAutoTuning() { return false; } + using DataType = Byte; + constexpr static std::string_view Name = "lz4_chunk_decompress"; + MGARDX_CONT + LZ4ChunkDecompressKernel(SubArray<1, Byte, DeviceType> packed, + SubArray<1, size_t, DeviceType> byte_offset, SIZE n, + SIZE chunk_size, SIZE nchunks, + SubArray<1, Byte, DeviceType> output) + : packed(packed), byte_offset(byte_offset), n(n), chunk_size(chunk_size), + nchunks(nchunks), output(output) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = LZ4ChunkDecompressFunctor; + FunctorType functor(packed, byte_offset, n, chunk_size, nchunks, output); + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (nchunks - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, tbz, tby, tbx, + functor.shared_memory_size(), queue_idx, std::string(Name)); + } + +private: + SubArray<1, Byte, DeviceType> packed; + SubArray<1, size_t, DeviceType> byte_offset; + SIZE n, chunk_size, nchunks; + SubArray<1, Byte, DeviceType> output; +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Lossless/Lossless.hpp b/include/mgard-x-160/Lossless/Lossless.hpp new file mode 100644 index 000000000..acccf8af2 --- /dev/null +++ b/include/mgard-x-160/Lossless/Lossless.hpp @@ -0,0 +1,279 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "BlockDelta/BlockDelta.hpp" +#include "CPU.hpp" +#include "LZ4/LZ4.hpp" +#include "LosslessCompressorInterface.hpp" +#include "ParallelHuffman/Huffman.hpp" +#include "ParallelRLE/ZeroRunLengthEncoding.hpp" +#include "SymbolRans/SymbolRans.hpp" +#include "Zstd.hpp" +#include "rANS/Rans.hpp" + +#ifndef MGARD_X_160_LOSSLESS_TEMPLATE_HPP +#define MGARD_X_160_LOSSLESS_TEMPLATE_HPP + +namespace mgard_x_160 { + +template +class ComposedLosslessCompressor + : public LosslessCompressorInterface { +public: + using S = typename std::make_signed::type; + using Q = typename std::make_unsigned::type; + + ComposedLosslessCompressor() : initialized(false) {} + + // Whether the configured lossless path actually uses the (workspace-heavy) + // Huffman backend. BlockDelta and standalone LZ4 have their own + // (de)compressors and never touch the Huffman workspace; every other type is + // Huffman / Huffman+LZ4 / Huffman+Zstd (or CPU_Lossless falling through to + // Huffman in Compress()). + static bool uses_huffman(enum lossless_type lossless) { + return lossless != lossless_type::BlockDelta && + lossless != lossless_type::LZ4 && + lossless != lossless_type::ZeroRLE_Rans && + lossless != lossless_type::SymbolRans; + } + + // Worst-case byte size of the RLE0 blob fed to rANS: a (count, symbol) pair + // per element when nothing repeats (uint32 count + T symbol), plus slack. + static SIZE rle_rans_bound(SIZE n) { + return n * (sizeof(uint32_t) + sizeof(T)) + 64; + } + + ComposedLosslessCompressor(SIZE n, Config config) + : initialized(true), n(n), config(config) { + static_assert(!std::is_floating_point::value, + "ComposedLosslessCompressor: Type of T must be integer."); + if (uses_huffman(config.lossless)) { + huffman.Resize(n, config.huff_dict_size, config.huff_block_size, + config.estimate_outlier_ratio, 0); + } + if (config.lossless == lossless_type::Huffman_LZ4) { + lz4.Resize(n * sizeof(H), config.lz4_block_size, 0); + } + if (config.lossless == lossless_type::LZ4) { + lz4.Resize(n * sizeof(T), config.lz4_block_size, 0); + } + if (config.lossless == lossless_type::Huffman_Zstd) { + zstd.Resize(n * sizeof(H), config.zstd_compress_level, 0); + } + if (config.lossless == lossless_type::BlockDelta) { + blockdelta.Resize(n, config.block_delta_block_size, + config.block_delta_mode, 0); + } + if (config.lossless == lossless_type::ZeroRLE_Rans) { + zerorle.Resize(n, 0); + rans.Resize(rle_rans_bound(n), 256, 0); + } + if (config.lossless == lossless_type::SymbolRans) { + symbolrans.Resize(n, config.huff_dict_size, config.estimate_outlier_ratio, + 0); + } + DeviceRuntime::SyncQueue(0); + } + + void Adapt(SIZE n, Config config, int queue_idx) { + this->initialized = true; + this->n = n; + this->config = config; + if (uses_huffman(config.lossless)) { + huffman.Resize(n, config.huff_dict_size, config.huff_block_size, + config.estimate_outlier_ratio, queue_idx); + } + if (config.lossless == lossless_type::Huffman_LZ4) { + lz4.Resize(n * sizeof(H), config.lz4_block_size, queue_idx); + } + if (config.lossless == lossless_type::LZ4) { + lz4.Resize(n * sizeof(T), config.lz4_block_size, queue_idx); + } + if (config.lossless == lossless_type::Huffman_Zstd) { + zstd.Resize(n * sizeof(H), config.zstd_compress_level, queue_idx); + } + if (config.lossless == lossless_type::BlockDelta) { + blockdelta.Resize(n, config.block_delta_block_size, + config.block_delta_mode, queue_idx); + } + if (config.lossless == lossless_type::ZeroRLE_Rans) { + zerorle.Resize(n, queue_idx); + rans.Resize(rle_rans_bound(n), 256, queue_idx); + } + if (config.lossless == lossless_type::SymbolRans) { + symbolrans.Resize(n, config.huff_dict_size, config.estimate_outlier_ratio, + queue_idx); + } + } + + static size_t EstimateMemoryFootprint(SIZE primary_count, Config config) { + size_t size = 0; + if (uses_huffman(config.lossless)) { + size += Huffman::EstimateMemoryFootprint( + primary_count, config.huff_dict_size, config.huff_block_size, + config.estimate_outlier_ratio); + } + if (config.lossless == lossless_type::Huffman_LZ4) { + size += LZ4::EstimateMemoryFootprint( + primary_count * sizeof(H), config.lz4_block_size); + } + if (config.lossless == lossless_type::LZ4) { + size += LZ4::EstimateMemoryFootprint( + primary_count * sizeof(T), config.lz4_block_size); + } + if (config.lossless == lossless_type::Huffman_Zstd) { + size += + Zstd::EstimateMemoryFootprint(primary_count * sizeof(H)); + } + if (config.lossless == lossless_type::BlockDelta) { + size += BlockDeltaLossless::EstimateMemoryFootprint( + primary_count, config.block_delta_block_size); + } + if (config.lossless == lossless_type::ZeroRLE_Rans) { + // RLE0 blob + rANS scratch (~2x the blob) dominate. + size += rle_rans_bound(primary_count) * 3; + } + if (config.lossless == lossless_type::SymbolRans) { + size += SymbolRans::EstimateMemoryFootprint( + primary_count, config.huff_dict_size, config.estimate_outlier_ratio); + } + return size; + } + + void Compress(Array<1, T, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + + if (config.lossless == lossless_type::BlockDelta) { + blockdelta.Compress(original_data, compressed_data, queue_idx); + return; + } + + if (config.lossless == lossless_type::LZ4) { + // LZ4 directly on the raw quantized stream (no Huffman). View the T array + // as bytes into compressed_data, then compress it in place. + SIZE nbytes = original_data.shape(0) * sizeof(T); + compressed_data.resize({nbytes}, queue_idx); + MemoryManager::Copy1D(compressed_data.data(), + (Byte *)original_data.data(), nbytes, + queue_idx); + lz4.Compress(compressed_data, queue_idx); + return; + } + + if (config.lossless == lossless_type::ZeroRLE_Rans) { + // Zero-RLE the quantized stream into a (counts, symbols) byte blob, then + // entropy-code that blob with rANS. Self-contained: the rANS output is + // the final compressed stream (Serialize/Deserialize are no-ops here). + zerorle.Compress(original_data, rle_bytes, 0.0, queue_idx); + rans.Compress(rle_bytes, compressed_data, queue_idx); + return; + } + + if (config.lossless == lossless_type::SymbolRans) { + // Outlier separation + symbol-alphabet rANS over the dict_size primary. + symbolrans.Compress(original_data, compressed_data, queue_idx); + return; + } + + huffman.Compress(original_data, compressed_data, 0.0, queue_idx); + + if (config.lossless == lossless_type::Huffman_LZ4) { + huffman.Serialize(compressed_data, queue_idx); + lz4.Compress(compressed_data, queue_idx); + } + + if (config.lossless == lossless_type::Huffman_Zstd) { + huffman.Serialize(compressed_data, queue_idx); + zstd.Compress(compressed_data, queue_idx); + } + } + + void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + if (config.lossless == lossless_type::Huffman) { + huffman.Serialize(compressed_data, queue_idx); + } + if (config.lossless == lossless_type::BlockDelta) { + blockdelta.Serialize(compressed_data, queue_idx); + } + } + + void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + if (config.lossless == lossless_type::Huffman) { + huffman.Deserialize(compressed_data, queue_idx); + } + if (config.lossless == lossless_type::BlockDelta) { + blockdelta.Deserialize(compressed_data, queue_idx); + } + } + + void Decompress(Array<1, Byte, DeviceType> &compressed_data, + Array<1, T, DeviceType> &decompressed_data, int queue_idx) { + + if (config.lossless == lossless_type::BlockDelta) { + // Deserialize (the memory-movement stage) was already run separately by + // the pipeline; Decompress is computation only. + blockdelta.Decompress(compressed_data, decompressed_data, queue_idx); + return; + } + + if (config.lossless == lossless_type::LZ4) { + // Inverse of the standalone LZ4 path: decompress to the raw byte stream, + // then reinterpret it back into the quantized T array. + lz4.Decompress(compressed_data, queue_idx); + SIZE nbytes = compressed_data.shape(0); + decompressed_data.resize({(SIZE)(nbytes / sizeof(T))}, queue_idx); + MemoryManager::Copy1D((Byte *)decompressed_data.data(), + compressed_data.data(), nbytes, + queue_idx); + return; + } + + if (config.lossless == lossless_type::ZeroRLE_Rans) { + // Inverse of the RLE0 -> rANS path: rANS-decode back to the RLE0 blob, + // then expand it to the quantized stream. + rans.Deserialize(compressed_data, queue_idx); + rans.Decompress(compressed_data, rle_bytes, queue_idx); + zerorle.Deserialize(rle_bytes, queue_idx); + zerorle.Decompress(rle_bytes, decompressed_data, queue_idx); + return; + } + + if (config.lossless == lossless_type::SymbolRans) { + symbolrans.Decompress(compressed_data, decompressed_data, queue_idx); + return; + } + + if (config.lossless == lossless_type::Huffman_LZ4) { + lz4.Decompress(compressed_data, queue_idx); + huffman.Deserialize(compressed_data, queue_idx); + } + + if (config.lossless == lossless_type::Huffman_Zstd) { + zstd.Decompress(compressed_data, queue_idx); + huffman.Deserialize(compressed_data, queue_idx); + } + + huffman.Decompress(compressed_data, decompressed_data, queue_idx); + } + + bool initialized; + SIZE n; + Config config; + Huffman huffman; + LZ4 lz4; + Zstd zstd; + BlockDeltaLossless blockdelta; + parallel_rle::ZeroRunLengthEncoding + zerorle; + rans::Rans rans; + Array<1, Byte, DeviceType> rle_bytes; + SymbolRans symbolrans; +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Lossless/LosslessCompressorInterface.hpp b/include/mgard-x-160/Lossless/LosslessCompressorInterface.hpp new file mode 100644 index 000000000..8e0a6254e --- /dev/null +++ b/include/mgard-x-160/Lossless/LosslessCompressorInterface.hpp @@ -0,0 +1,21 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_LOSSLESS_COMPRESSOR_INTERFACE_HPP +#define MGARD_X_160_LOSSLESS_COMPRESSOR_INTERFACE_HPP +namespace mgard_x_160 { +template class LosslessCompressorInterface { + virtual void Compress(Array<1, T, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, + int queue_idx) = 0; + virtual void Decompress(Array<1, Byte, DeviceType> &compressed_data, + Array<1, T, DeviceType> &decompressed_data, + int queue_idx) = 0; +}; +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/Condense.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/Condense.hpp new file mode 100644 index 000000000..f736eeedb --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/Condense.hpp @@ -0,0 +1,92 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_CONDENSE_TEMPLATE_HPP +#define MGARD_X_160_CONDENSE_TEMPLATE_HPP +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +template +class CondenseFunctor : public Functor { +public: + MGARDX_CONT CondenseFunctor() {} + MGARDX_CONT CondenseFunctor(SubArray<1, H, DeviceType> v, + SubArray<1, size_t, DeviceType> write_offsets, + SubArray<1, size_t, DeviceType> actual_lengths, + SubArray<1, H, DeviceType> condensed_v, + SIZE chunck_size) + : v(v), write_offsets(write_offsets), actual_lengths(actual_lengths), + condensed_v(condensed_v), chunck_size(chunck_size) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE chunck_id = FunctorBase::GetBlockIdX(); + SIZE thread_id = FunctorBase::GetThreadIdX(); + size_t actual_length = *actual_lengths(chunck_id); + size_t write_offset = *write_offsets(chunck_id); + for (SIZE read_offset = thread_id; read_offset < actual_length; + read_offset += FunctorBase::GetBlockDimX()) { + *condensed_v(write_offset + read_offset) = + *v(chunck_id * chunck_size + read_offset); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, H, DeviceType> v; + SubArray<1, size_t, DeviceType> write_offsets; + SubArray<1, size_t, DeviceType> actual_lengths; + SubArray<1, H, DeviceType> condensed_v; + SIZE chunck_size; +}; + +template +class CondenseKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "condense"; + MGARDX_CONT + CondenseKernel(SubArray<1, H, DeviceType> v, + SubArray<1, size_t, DeviceType> write_offsets, + SubArray<1, size_t, DeviceType> actual_lengths, + SubArray<1, H, DeviceType> condensed_v, SIZE chunck_size, + SIZE nchunk) + : v(v), write_offsets(write_offsets), actual_lengths(actual_lengths), + condensed_v(condensed_v), chunck_size(chunck_size), nchunk(nchunk) {} + + MGARDX_CONT + Task> GenTask(int queue_idx) { + using FunctorType = CondenseFunctor; + FunctorType functor(v, write_offsets, actual_lengths, condensed_v, + chunck_size); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = nchunk; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, H, DeviceType> v; + SubArray<1, size_t, DeviceType> write_offsets; + SubArray<1, size_t, DeviceType> actual_lengths; + SubArray<1, H, DeviceType> condensed_v; + SIZE chunck_size; + SIZE nchunk; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/Decode.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/Decode.hpp new file mode 100644 index 000000000..b967175ea --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/Decode.hpp @@ -0,0 +1,207 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_DECODE_TEMPLATE_HPP +#define MGARD_X_160_DECODE_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +template +class DecodeFunctor : public Functor { +public: + MGARDX_CONT DecodeFunctor() {} + MGARDX_CONT DecodeFunctor(SubArray<1, H, DeviceType> densely, + SubArray<1, size_t, DeviceType> dH_meta, + SubArray<1, Q, DeviceType> bcode, SIZE len, + int chunk_size, int n_chunk, + SubArray<1, uint8_t, DeviceType> singleton, + size_t singleton_size) + : densely(densely), dH_meta(dH_meta), bcode(bcode), len(len), + chunk_size(chunk_size), n_chunk(n_chunk), singleton(singleton), + singleton_size(singleton_size) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + if (CACHE_SINGLETION) { + // Opt 3: cooperatively stage only the small, hottest decodebook tables + // (first[] and entry[], together sizeof(H)*2*word_bw bytes ~ 1KB) into + // shared memory. These are touched on every decoded bit. The large keys[] + // table is left in global memory so caching never inflates the per-block + // shared-memory footprint and hurts occupancy (which is what made caching + // the whole decodebook a net loss for large dictionaries). All threads + // participate; the framework inserts a block sync before Operation2. + _s_singleton = (uint8_t *)FunctorBase::GetSharedMemory(); + size_t fe_bytes = sizeof(H) * (2 * sizeof(H) * 8); + uint8_t *src = singleton((IDX)0); + SIZE tid = FunctorBase::GetThreadIdX(); + SIZE nthreads = FunctorBase::GetBlockDimX(); + for (SIZE b = tid; b < (SIZE)fe_bytes; b += nthreads) { + _s_singleton[b] = src[b]; + } + } else { + _s_singleton = singleton((IDX)0); + } + } + + MGARDX_EXEC void Operation2() { + size_t chunk_id = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + if (chunk_id >= n_chunk) + return; + + SIZE densely_offset = *dH_meta(n_chunk + chunk_id); + SIZE bcode_offset = chunk_size * chunk_id; + size_t total_bw = *dH_meta(chunk_id); + + const size_t word_bw = sizeof(H) * 8; + // first[]/entry[] come from _s_singleton (shared when cached, else global); + // keys[] always stays in global memory. + auto first = reinterpret_cast(_s_singleton); + auto entry = first + sizeof(H) * 8; + auto keys = reinterpret_cast(singleton((IDX)0) + + sizeof(H) * (2 * sizeof(H) * 8)); + + // Opt 2: hold the current densely word in a register and refetch from + // global memory only when the bit cursor crosses into the next word, + // instead of re-loading the same word once per bit (up to word_bw redundant + // loads). + size_t cached_word_idx = 0; + H cached_word = *densely(densely_offset); + + H v = (cached_word >> (word_bw - 1)) & 0x1; // get the first bit + size_t l = 1; + size_t i = 0; + size_t idx_bcoded = 0; + while (i < total_bw) { + while (v < first[l]) { // append next i_cb bit + ++i; + size_t idx_word = i / word_bw; + if (idx_word != cached_word_idx) { + cached_word = *densely(densely_offset + idx_word); + cached_word_idx = idx_word; + } + H next_bit = (cached_word >> (word_bw - 1 - (i % word_bw))) & 0x1; + v = (v << 1) | next_bit; + ++l; + } + *bcode(bcode_offset + idx_bcoded) = keys[entry[l] + v - first[l]]; + idx_bcoded++; + { + ++i; + size_t idx_word = i / word_bw; + if (idx_word != cached_word_idx) { + cached_word = *densely(densely_offset + idx_word); + cached_word_idx = idx_word; + } + H next_bit = (cached_word >> (word_bw - 1 - (i % word_bw))) & 0x1; + v = next_bit; + } + l = 1; + } + } + + MGARDX_CONT size_t shared_memory_size() { + if (CACHE_SINGLETION) { + // Only first[]/entry[] are cached (see Operation1), not the full table. + return sizeof(H) * (2 * sizeof(H) * 8); + } else { + return 0; + } + } + +private: + SubArray<1, H, DeviceType> densely; + SubArray<1, size_t, DeviceType> dH_meta; + SubArray<1, Q, DeviceType> bcode; + SIZE len; + int chunk_size; + int n_chunk; + SubArray<1, uint8_t, DeviceType> singleton; + size_t singleton_size; + + uint8_t *_s_singleton; +}; + +template +class DecodeKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + using DataType = H; + constexpr static std::string_view Name = "decode"; + MGARDX_CONT + DecodeKernel(SubArray<1, H, DeviceType> densely, + SubArray<1, size_t, DeviceType> dH_meta, + SubArray<1, Q, DeviceType> bcode, SIZE len, int chunk_size, + int n_chunk, SubArray<1, uint8_t, DeviceType> singleton, + size_t singleton_size) + : densely(densely), dH_meta(dH_meta), bcode(bcode), len(len), + chunk_size(chunk_size), n_chunk(n_chunk), singleton(singleton), + singleton_size(singleton_size) {} + + // NOTE: decode is one independent thread per chunk and is dominated by warp + // straggler divergence (each thread loops until its own chunk's bits are + // exhausted, and the warp runs until its slowest chunk finishes). Larger + // (full-warp) blocks raise occupancy but put 32 chunks under one straggler + // group instead of 16, which measured ~15% SLOWER on NYX 512^3. The autotuned + // (half-warp) default is intentionally left in place. + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = DecodeFunctor; + FunctorType functor(densely, dH_meta, bcode, len, chunk_size, n_chunk, + singleton, singleton_size); + + int nchunk = (len - 1) / chunk_size + 1; + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = R; + tby = C; + tbx = F; + gridz = 1; + gridy = 1; + gridx = (nchunk - 1) / tbx + 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, H, DeviceType> densely; + SubArray<1, size_t, DeviceType> dH_meta; + SubArray<1, Q, DeviceType> bcode; + SIZE len; + int chunk_size; + int n_chunk; + SubArray<1, uint8_t, DeviceType> singleton; + size_t singleton_size; +}; + +template +void Decode(SubArray<1, H, DeviceType> densely, + SubArray<1, size_t, DeviceType> dH_meta, + SubArray<1, Q, DeviceType> bcode, SIZE len, int chunk_size, + int n_chunk, SubArray<1, uint8_t, DeviceType> singleton, + size_t singleton_size, int queue_idx) { + // Opt 3 caches only first[]/entry[] (~1KB) in shared memory, which always + // fits and never hurts occupancy, so the cached path is used unconditionally. + if (DeviceRuntime::PrintKernelConfig) { + std::cout << log::log_info + << "Decode: caching first[]/entry[] in shared memory\n"; + } + DeviceLauncher::Execute( + DecodeKernel(densely, dH_meta, bcode, len, + chunk_size, n_chunk, singleton, + singleton_size), + queue_idx); +} + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/Deflate.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/Deflate.hpp new file mode 100644 index 000000000..ac2eb2fec --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/Deflate.hpp @@ -0,0 +1,148 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_DEFLATE_TEMPLATE_HPP +#define MGARD_X_160_DEFLATE_TEMPLATE_HPP +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +// Jieyang: this kernel rely on whole grid sychronized execution +// For example, adding +// if (thread == 0) { +// __nanosleep(1e9); +// } +// will cause incorrect results + +template +class DeflateFunctor : public Functor { +public: + MGARDX_CONT DeflateFunctor() {} + MGARDX_CONT DeflateFunctor(SubArray<1, H, DeviceType> hcoded, + SubArray<1, size_t, DeviceType> densely_meta, + int PART_SIZE) + : hcoded(hcoded), densely_meta(densely_meta), PART_SIZE(PART_SIZE) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + size_t gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + if (gid >= (hcoded.shape(0) - 1) / PART_SIZE + 1) + return; + uint8_t bitwidth; + size_t densely_coded_lsb_pos = sizeof(H) * 8, total_bitwidth = 0; + size_t ending = (gid + 1) * PART_SIZE <= hcoded.shape(0) + ? PART_SIZE + : hcoded.shape(0) - gid * PART_SIZE; + // if ((gid + 1) * PART_SIZE > hcoded.shape(0)) printf("\n\ngid + // %lu\tending %lu\n\n", gid, ending); + H msb_bw_word_lsb, _1, _2, buffer; + H *current = hcoded(gid * PART_SIZE); + for (size_t i = 0; i < ending; i++) { + + msb_bw_word_lsb = *hcoded(gid * PART_SIZE + i); + bitwidth = *((uint8_t *)&msb_bw_word_lsb + (sizeof(H) - 1)); + + *((uint8_t *)&msb_bw_word_lsb + sizeof(H) - 1) = 0x0; + if (densely_coded_lsb_pos == sizeof(H) * 8) { + // *current = 0x0; // a new unit of data type + buffer = 0x0; + } + if (bitwidth <= densely_coded_lsb_pos) { + densely_coded_lsb_pos -= bitwidth; + // *current |= msb_bw_word_lsb << densely_coded_lsb_pos; + buffer |= msb_bw_word_lsb << densely_coded_lsb_pos; + if (densely_coded_lsb_pos == 0) { + densely_coded_lsb_pos = sizeof(H) * 8; + *current = buffer; + ++current; + } + } else { + // example: we have 5-bit code 11111 but 3 bits left for (*current) + // we put first 3 bits of 11111 to the last 3 bits of (*current) + // and put last 2 bits from MSB of (*(++current)) + // the comment continues with the example + _1 = msb_bw_word_lsb >> (bitwidth - densely_coded_lsb_pos); + _2 = msb_bw_word_lsb + << (sizeof(H) * 8 - (bitwidth - densely_coded_lsb_pos)); + buffer |= _1; + *current = buffer; + // *current |= _1; + // *(++current) = 0x0; + current++; + // *current |= _2; + buffer = 0x0; + buffer = _2; + densely_coded_lsb_pos = + sizeof(H) * 8 - (bitwidth - densely_coded_lsb_pos); + } + total_bitwidth += bitwidth; + } + // if (densely_coded_lsb_pos != sizeof(H) * 8) { + *current = buffer; + // ++current; + // } + *densely_meta(gid) = total_bitwidth; + } + + MGARDX_EXEC void Operation2() {} + + MGARDX_EXEC void Operation3() {} + + MGARDX_EXEC void Operation4() {} + + MGARDX_EXEC void Operation5() {} + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, H, DeviceType> hcoded; + SubArray<1, size_t, DeviceType> densely_meta; + int PART_SIZE; +}; + +template class DeflateKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + using DataType = H; + constexpr static std::string_view Name = "deflate"; + MGARDX_CONT + DeflateKernel(SubArray<1, H, DeviceType> hcoded, + SubArray<1, size_t, DeviceType> densely_meta, int PART_SIZE) + : hcoded(hcoded), densely_meta(densely_meta), PART_SIZE(PART_SIZE) {} + + template + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = DeflateFunctor; + FunctorType functor(hcoded, densely_meta, PART_SIZE); + + auto nchunk = (hcoded.shape(0) - 1) / PART_SIZE + 1; + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = R; + tby = C; + tbx = F; + gridz = 1; + gridy = 1; + gridx = (nchunk - 1) / tbx + 1; + // printf("%u %u %u\n", shape.dataHost()[2], shape.dataHost()[1], + // shape.dataHost()[0]); PrintSubarray("shape", shape); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, H, DeviceType> hcoded; + SubArray<1, size_t, DeviceType> densely_meta; + int PART_SIZE; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/DictionaryShift.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/DictionaryShift.hpp new file mode 100644 index 000000000..c3c28db2c --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/DictionaryShift.hpp @@ -0,0 +1,80 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_DICTIONARY_SHIFT_TEMPLATE_HPP +#define MGARD_X_160_DICTIONARY_SHIFT_TEMPLATE_HPP +#include "../../RuntimeX/RuntimeX.h" + +#define MGARDX_SHIFT_DICT 1 +#define MGARDX_RESTORE_DICT 2 + +namespace mgard_x_160 { + +template +class DictionaryShiftFunctor : public Functor { +public: + MGARDX_CONT DictionaryShiftFunctor() {} + MGARDX_CONT DictionaryShiftFunctor(SubArray<1, T, DeviceType> v, + SIZE dict_size) + : v(v), dict_size(dict_size) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE id = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + if (id < v.shape(0)) { + T value = *v(id); + if constexpr (OP == MGARDX_SHIFT_DICT) { + *v(id) += dict_size / 2; + } else if constexpr (OP == MGARDX_RESTORE_DICT) { + *v(id) -= dict_size / 2; + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T, DeviceType> v; + SIZE dict_size; +}; + +template +class DictionaryShiftKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "DictionaryShift"; + MGARDX_CONT + DictionaryShiftKernel(SubArray<1, T, DeviceType> v, SIZE dict_size) + : v(v), dict_size(dict_size) {} + + MGARDX_CONT + Task> GenTask(int queue_idx) { + using FunctorType = DictionaryShiftFunctor; + FunctorType functor(v, dict_size); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (v.shape(0) - 1) / tbx + 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T, DeviceType> v; + SIZE dict_size; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/EncodeFixedLen.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/EncodeFixedLen.hpp new file mode 100644 index 000000000..cbc2ac5bd --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/EncodeFixedLen.hpp @@ -0,0 +1,96 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_ENCODE_FIXED_LEN_TEMPLATE_HPP +#define MGARD_X_160_ENCODE_FIXED_LEN_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +// Jieyang: this kernel rely on whole grid sychronized execution +// For example, adding +// if (thread == 0) { +// __nanosleep(1e9); +// } +// will cause incorrect results + +template +class EncodeFixedLenFunctor : public Functor { +public: + MGARDX_CONT EncodeFixedLenFunctor() {} + MGARDX_CONT EncodeFixedLenFunctor(SubArray<1, Q, DeviceType> data, + SubArray<1, H, DeviceType> hcoded, + SubArray<1, H, DeviceType> codebook) + : data(data), hcoded(hcoded), codebook(codebook) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + unsigned int gid = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (gid >= data.shape(0)) + return; + *hcoded(gid) = *codebook(*data(gid)); // try to exploit cache? + } + + MGARDX_EXEC void Operation2() {} + + MGARDX_EXEC void Operation3() {} + + MGARDX_EXEC void Operation4() {} + + MGARDX_EXEC void Operation5() {} + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Q, DeviceType> data; + SubArray<1, H, DeviceType> hcoded; + SubArray<1, H, DeviceType> codebook; +}; + +template +class EncodeFixedLenKernel : public Kernel { +public: + constexpr static DIM NumDim = 1; + using DataType = H; + constexpr static std::string_view Name = "encode"; + MGARDX_CONT + EncodeFixedLenKernel(SubArray<1, Q, DeviceType> data, + SubArray<1, H, DeviceType> hcoded, + SubArray<1, H, DeviceType> codebook) + : data(data), hcoded(hcoded), codebook(codebook) {} + + template + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = EncodeFixedLenFunctor; + FunctorType functor(data, hcoded, codebook); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = R; + tby = C; + tbx = F; + gridz = 1; + gridy = 1; + gridx = (data.shape(0) - 1) / tbx + 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Q, DeviceType> data; + SubArray<1, H, DeviceType> hcoded; + SubArray<1, H, DeviceType> codebook; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/EntropyCalculator.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/EntropyCalculator.hpp new file mode 100644 index 000000000..34286e6d8 --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/EntropyCalculator.hpp @@ -0,0 +1,60 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_ENTROPY_CALCULAOR_HPP +#define MGARD_X_160_ENTROPY_CALCULAOR_HPP + +namespace mgard_x_160 { + +template +double CalculateLC(SIZE total_num_elems, SIZE dict_size, + SubArray<1, unsigned int, DeviceType> freq, + SubArray<1, unsigned int, DeviceType> CL, int queue_idx) { + unsigned int *_freq = new unsigned int[dict_size]; + unsigned int *_cl = new unsigned int[dict_size]; + MemoryManager::Copy1D(_freq, freq.data(), dict_size, queue_idx); + MemoryManager::Copy1D(_cl, CL.data(), dict_size, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + double LC = 0; + // for (SIZE i = 0; i < dict_size; i++) { + // std::cout << _freq[i] << " "; + // } + // std::cout << "\n"; + // for (SIZE i = 0; i < dict_size; i++) { + // std::cout << _cl[i] << " "; + // } + // std::cout << "\n"; + for (SIZE i = 0; i < dict_size; i++) { + LC += (double)_freq[i] / total_num_elems * _cl[i]; + } + delete[] _freq; + delete[] _cl; + return LC; +} + +template +double CalculateEntropy(SIZE total_num_elems, SIZE dict_size, + SubArray<1, unsigned int, DeviceType> freq, + int queue_idx) { + unsigned int *_freq = new unsigned int[dict_size]; + MemoryManager::Copy1D(_freq, freq.data(), dict_size, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + double entropy = 0; + // printf("dict size: %u\n", dict_size); + for (SIZE i = 0; i < dict_size; i++) { + double p = (double)_freq[i] / total_num_elems; + // printf("%u ", _freq[i]); + entropy += -1 * log2(p) * p; + } + // printf("\n"); + + delete[] _freq; + return entropy; +} + +} // namespace mgard_x_160 +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/FillArraySequence.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/FillArraySequence.hpp new file mode 100644 index 000000000..ea2427a33 --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/FillArraySequence.hpp @@ -0,0 +1,69 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_FILL_ARRAY_SEQUENCE_TEMPLATE_HPP +#define MGARD_X_160_FILL_ARRAY_SEQUENCE_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +template +class FillArraySequenceFunctor : public Functor { +public: + MGARDX_CONT FillArraySequenceFunctor() {} + MGARDX_CONT FillArraySequenceFunctor(SubArray<1, T, DeviceType> array) + : array(array) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + unsigned int thread = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (thread < array.shape(0)) { + *array(thread) = thread; + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T, DeviceType> array; +}; + +template +class FillArraySequenceKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "filling array sequence"; + MGARDX_CONT + FillArraySequenceKernel(SubArray<1, T, DeviceType> array) : array(array) {} + + MGARDX_CONT + Task> GenTask(int queue_idx) { + using FunctorType = FillArraySequenceFunctor; + FunctorType functor(array); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = DeviceRuntime::GetMaxNumThreadsPerTB(); + gridz = 1; + gridy = 1; + gridx = (array.shape(0) / tbx) + 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T, DeviceType> array; +}; +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/GenerateCL.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/GenerateCL.hpp new file mode 100644 index 000000000..955cb0de3 --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/GenerateCL.hpp @@ -0,0 +1,936 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-GPU: MultiGrid Adaptive Reduction oT Data Accelerated by GPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_GENERATE_CL_TEMPLATE_HPP +#define MGARD_X_160_GENERATE_CL_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +#include + +namespace mgard_x_160 { + +#define MAX(X, Y) (((X) > (Y)) ? (X) : (Y)) +#define MIN(X, Y) (((X) < (Y)) ? (X) : (Y)) +#define MOD(a, b) ((((a) % (b)) + (b)) % (b)) + +#define _iNodesFront 0 +#define _iNodesRear 1 +#define _lNodesCur 2 +#define _iNodesSize 3 +#define _curLeavesNum 4 +#define _minFreq 5 +#define _tempLength 6 +#define _mergeFront 7 +#define _mergeRear 8 + +template +class GenerateCLFunctor : public HuffmanCLCustomizedFunctor { +public: + MGARDX_CONT GenerateCLFunctor() {} + MGARDX_CONT GenerateCLFunctor( + SubArray<1, T, DeviceType> histogram, SubArray<1, T, DeviceType> CL, + int dict_size, + /* Global Arrays */ + SubArray<1, T, DeviceType> lNodesFreq, + SubArray<1, int, DeviceType> lNodesLeader, + SubArray<1, T, DeviceType> iNodesFreq, + SubArray<1, int, DeviceType> iNodesLeader, + SubArray<1, T, DeviceType> tempFreq, + SubArray<1, int, DeviceType> tempIsLeaf, + SubArray<1, int, DeviceType> tempIndex, + SubArray<1, T, DeviceType> copyFreq, + SubArray<1, int, DeviceType> copyIsLeaf, + SubArray<1, int, DeviceType> copyIndex, + SubArray<1, uint32_t, DeviceType> diagonal_path_intersections, + SubArray<1, int, DeviceType, false, true> status, + // Width of the per-block diagonal merge-path search (Operations 6-9) + // and of the thread block that runs it. Read from the device's real + // warp/wavefront size (DeviceRuntime::GetWarpSize(), host-side only, + // so it is threaded through as a plain functor member rather than + // read from device code) instead of assuming the CUDA-oriented + // MGARDX_WARP_SIZE=32: the merge-path search here uses shared memory + // with an explicit sync between operations (not warp-implicit + // shuffle/ballot lockstep), so it is safe to size to whatever the + // hardware actually reports, letting each block search a full + // wavefront's worth of the diagonal per step. + SIZE warp_size) + : histogram(histogram), CL(CL), dict_size(dict_size), + lNodesFreq(lNodesFreq), lNodesLeader(lNodesLeader), + iNodesFreq(iNodesFreq), iNodesLeader(iNodesLeader), tempFreq(tempFreq), + tempIsLeaf(tempIsLeaf), tempIndex(tempIndex), copyFreq(copyFreq), + copyIsLeaf(copyIsLeaf), copyIndex(copyIndex), + diagonal_path_intersections(diagonal_path_intersections), + status(status), warp_size(warp_size) { + HuffmanCLCustomizedFunctor(); + } + + MGARDX_EXEC void Operation1() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + + /* Initialization */ + // All leaf nodes have no leader at beginning + if (i < dict_size) { + *lNodesLeader((IDX)i) = -1; + *CL((IDX)i) = 0; + } + + if (i == 0) { + // Front of internal node list + (*status((IDX)_iNodesFront)) = 0; + // End of internal node list + (*status((IDX)_iNodesRear)) = 0; + // Front of unprocessed leaf node list + (*status((IDX)_lNodesCur)) = 0; + // Size of internal node list + (*status((IDX)_iNodesSize)) = 0; + } + } + + MGARDX_CONT_EXEC bool LoopCondition1() { + + // if (i == 0) { + // printf("************LoopCondition1 _lNodesCur(%d) (dict_size)%u + // (_iNodesSize)%d\n", (*status((IDX)_lNodesCur)), dict_size, + // (*status((IDX)_iNodesSize))); + // } + // Continue if we have not processed all leaf nodes OR + // if unmeraged internal nodes is greater than 1 (more that just root) + return (*status((IDX)_lNodesCur)) < dict_size || + (*status((IDX)_iNodesSize)) > 1; + } + + MGARDX_EXEC void Operation2() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + /* Combine two least frequent nodes on same level */ + if (i == 0) { + T midFreq[4]; + int midIsLeaf[4]; + for (int j = 0; j < 4; ++j) { + midFreq[j] = UINT_MAX; + midIsLeaf[j] = 0; + } + + // Get two nodes from leaf node if possible + if ((*status((IDX)_lNodesCur)) < dict_size) { + midFreq[0] = *lNodesFreq((*status((IDX)_lNodesCur))); + midIsLeaf[0] = 1; + } + if ((*status((IDX)_lNodesCur)) < dict_size - 1) { + midFreq[1] = *lNodesFreq((*status((IDX)_lNodesCur)) + 1); + midIsLeaf[1] = 1; + } + + // Get two nodes from internal node if possible + if ((*status((IDX)_iNodesSize)) >= 1) { + midFreq[2] = *iNodesFreq((*status((IDX)_iNodesFront))); + midIsLeaf[2] = 0; + } + if ((*status((IDX)_iNodesSize)) >= 2) { + midFreq[3] = + *iNodesFreq(MOD((*status((IDX)_iNodesFront)) + 1, dict_size)); + midIsLeaf[3] = 0; + } + + // printf("midFreq: %u %u %u %u\n", midFreq[0], midFreq[1], + // midFreq[2], midFreq[3]); + // printf("midIsLeaf: %d %d %d %d\n", midIsLeaf[0], midIsLeaf[1], + // midIsLeaf[2], midIsLeaf[3]); + + /* Select the minimum of minimums - sorting network */ + /* TODO There's likely a good 1-warp faster way to do this */ + { + T tempFreq; + int tempIsLeaf; + if (midFreq[1] > midFreq[3]) { + tempFreq = midFreq[1]; + midFreq[1] = midFreq[3]; + midFreq[3] = tempFreq; + tempIsLeaf = midIsLeaf[1]; + midIsLeaf[1] = midIsLeaf[3]; + midIsLeaf[3] = tempIsLeaf; + } + if (midFreq[0] > midFreq[2]) { + tempFreq = midFreq[0]; + midFreq[0] = midFreq[2]; + midFreq[2] = tempFreq; + tempIsLeaf = midIsLeaf[0]; + midIsLeaf[0] = midIsLeaf[2]; + midIsLeaf[2] = tempIsLeaf; + } + if (midFreq[0] > midFreq[1]) { + tempFreq = midFreq[0]; + midFreq[0] = midFreq[1]; + midFreq[1] = tempFreq; + tempIsLeaf = midIsLeaf[0]; + midIsLeaf[0] = midIsLeaf[1]; + midIsLeaf[1] = tempIsLeaf; + } + if (midFreq[2] > midFreq[3]) { + tempFreq = midFreq[2]; + midFreq[2] = midFreq[3]; + midFreq[3] = tempFreq; + tempIsLeaf = midIsLeaf[2]; + midIsLeaf[2] = midIsLeaf[3]; + midIsLeaf[3] = tempIsLeaf; + } + if (midFreq[1] > midFreq[2]) { + tempFreq = midFreq[1]; + midFreq[1] = midFreq[2]; + midFreq[2] = tempFreq; + tempIsLeaf = midIsLeaf[1]; + midIsLeaf[1] = midIsLeaf[2]; + midIsLeaf[2] = tempIsLeaf; + } + } + + // printf("mine: (*status((IDX)_lNodesCur)): %u, + // (*status((IDX)_iNodesSize)): %u\n", (*status((IDX)_lNodesCur)), + // (*status((IDX)_iNodesSize))); printf("mine: midFreq[0]: %u, midFreq[1]: + // %u\n", midFreq[0], midFreq[1]); printf("mine: midIsLeaf[0]: %u, + // midIsLeaf[1]: %u\n", midIsLeaf[0], midIsLeaf[1]); + + // Calculate the merged frequency. + // If we have only 1 node then do not merge + (*status((IDX)_minFreq)) = midFreq[0]; + if (midFreq[1] < UINT_MAX) { + (*status((IDX)_minFreq)) += midFreq[1]; + } + + // printf("merged freq: %u\n", *status((IDX)_minFreq)); + + // Create a internal node in the end + // Assign merged frequency + // Assign no leader at beginning + // _iNodesRear needs to be increased by 1 later + *iNodesFreq((IDX)(*status((IDX)_iNodesRear))) = (*status((IDX)_minFreq)); + *iNodesLeader((IDX)(*status((IDX)_iNodesRear))) = -1; + + // printf("new internal node : %d\n", *status((IDX)_iNodesRear)); + + // printf("mine: iNodesLeader(0.leader) = %d, (*status((IDX)_iNodesRear)): + // %u\n", *iNodesLeader(IDX(0)), (*status((IDX)_iNodesRear))); + + /* If the smallest node is leaf */ + if (midFreq[0] < UINT_MAX) { + if (midIsLeaf[0]) { + // printf("mid[0] is leaf\n"); + *lNodesLeader((IDX)(*status((IDX)_lNodesCur))) = + (*status((IDX)_iNodesRear)); + // printf("update leader of leaf %d to just created internal node + // %d\n", *status((IDX)_lNodesCur), + // *lNodesLeader((IDX)(*status((IDX)_lNodesCur)))); + ++(*CL((IDX)(*status((IDX)_lNodesCur)))); + ++(*status((IDX)_lNodesCur)); + // printf("remove mid[0] from unprocessed leaf node list. _lNodesCur = + // %d\n", *status((IDX)_lNodesCur)); printf("update CL(%d) = %u\n", + // *status((IDX)_lNodesCur-1), + // *CL((IDX)(*status((IDX)_lNodesCur-1)))); + } else { + // printf("mid[0] is internal node\n"); + // Update its leader to the internal node we just created + *iNodesLeader((IDX)(*status((IDX)_iNodesFront))) = + (*status((IDX)_iNodesRear)); + // printf("update leader of internal node %d to just created internal + // node %d\n", *status((IDX)_iNodesFront), + // *iNodesLeader((IDX)(*status((IDX)_iNodesFront)))); + (*status((IDX)_iNodesFront)) = + MOD((*status((IDX)_iNodesFront)) + 1, dict_size); + // printf("remove mid[0] from unprocessed internal node list. + // _iNodesFront = %d\n", *status((IDX)_iNodesFront)); + } + } + + if (midFreq[1] < UINT_MAX) { + /* If the second smallest node is leaf */ + if (midIsLeaf[1]) { + // printf("mid[1] is leaf\n"); + *lNodesLeader((IDX)(*status((IDX)_lNodesCur))) = + (*status((IDX)_iNodesRear)); + // printf("update leader of leaf %d to just created internal node + // %d\n", + // *status((IDX)_lNodesCur), + // *lNodesLeader((IDX)(*status((IDX)_lNodesCur)))); + ++(*CL((IDX)(*status((IDX)_lNodesCur)))); + ++(*status((IDX)_lNodesCur)); + // printf("remove mid[1] from unprocessed leaf node list. _lNodesCur = + // %d\n", *status((IDX)_lNodesCur)); printf("update CL(%d) = %u\n", + // *status((IDX)_lNodesCur-1), + // *CL((IDX)(*status((IDX)_lNodesCur-1)))); + } else { + // printf("mid[1] is internal node\n"); + *iNodesLeader((IDX)(*status((IDX)_iNodesFront))) = + (*status((IDX)_iNodesRear)); + // printf("update leader of internal node %d to just created internal + // node %d\n", + // *status((IDX)_iNodesFront), + // *iNodesLeader((IDX)(*status((IDX)_iNodesFront)))); + // printf("remove mid[1] from unprocessed internal node list. + // _iNodesFront = %d\n", *status((IDX)_iNodesFront)); + (*status((IDX)_iNodesFront)) = + MOD((*status((IDX)_iNodesFront)) + 1, dict_size); + // printf("remove mid[1] from unprocessed internal node list. + // _iNodesFront = %d\n", *status((IDX)_iNodesFront)); + } + } + + // (*status((IDX)_iNodesRear)) = MOD((*status((IDX)_iNodesRear)) + 1, + // dict_size); + + // Update internal node list size + (*status((IDX)_iNodesSize)) = + MOD((*status((IDX)_iNodesRear)) - (*status((IDX)_iNodesFront)), + dict_size); + + // printf("update number of unprocessed internal node (not including just + // created) _iNodesSize: %d\n", *status((IDX)_iNodesSize)); + + // printf("mine: iNodesLeader(0.leader) = %d, (*status((IDX)_iNodesRear)): + // %u\n", *iNodesLeader(IDX(0)), (*status((IDX)_iNodesRear))); + + // Reset total number of leaf nodes to be parallel merged + (*status((IDX)_curLeavesNum)) = 0; + } + } + + MGARDX_EXEC void Operation3() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + /* Select elements to copy -- parallelized */ + // For all unprocessed leaf node + if (i >= (*status((IDX)_lNodesCur)) && i < dict_size) { + // Parallel component + int threadCurLeavesNum; + // If the frequency of a leaf node <= the frequency of just two merged + // nodes + if (*lNodesFreq((IDX)i) <= (*status((IDX)_minFreq))) { + // Make it start from 1 + threadCurLeavesNum = i - (*status((IDX)_lNodesCur)) + 1; + // Atomic max -- Largest valid index + // Find the maximum threadCurLeavesNum that satisify our condition + // Now we find the number of leaf nodes to be parallel merged + Atomic::Max( + status((IDX)_curLeavesNum), threadCurLeavesNum); + } + } + } + + MGARDX_EXEC void Operation4() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + // Copy all leaf nodes to be parallel merged in a temp buffer + if (i - (*status((IDX)_lNodesCur)) < (*status((IDX)_curLeavesNum))) { + *copyFreq((IDX)i - (*status((IDX)_lNodesCur))) = *lNodesFreq((IDX)i); + *copyIndex((IDX)i - (*status((IDX)_lNodesCur))) = i; + *copyIsLeaf((IDX)i - (*status((IDX)_lNodesCur))) = 1; + } + // if (i == 0) { + // printf("prepare parallel merge. copy %d leaf nodes\n", + // *status((IDX)_curLeavesNum)); + // } + } + + MGARDX_EXEC void Operation5() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + // if (!thread) { + // printf("(*status((IDX)_curLeavesNum)): %d\n", + // (*status((IDX)_curLeavesNum))); + // } + + /* Updates Iterators */ + if (i == 0) { + // All internal nodes participate in parallel merge + (*status((IDX)_mergeRear)) = (*status((IDX)_iNodesRear)); + (*status((IDX)_mergeFront)) = (*status((IDX)_iNodesFront)); + + // If total number of node to be merged are even + // printf("total nodes to be merged: leaf(%d), internal(%d)\n", + // *status((IDX)_curLeavesNum), *status((IDX)_iNodesSize)); + if (((*status((IDX)_curLeavesNum)) + (*status((IDX)_iNodesSize))) % 2 == + 0) { + // printf("total number of nodes is even, so empty internal list since + // all will be merged\n"); + (*status((IDX)_iNodesFront)) = (*status((IDX)_iNodesRear)); + } + /* Odd number of nodes to merge - leave out one*/ + // Remove one internal node (keep it for the next outer iteration) if: + // (a) there are no leaf nodes in the batch (_curLeavesNum == 0), OR + // (b) the first leaf node NOT in the batch has a frequency <= the + // highest-frequency internal node in the batch, meaning it is + // cheaper to defer that internal node. + // + // BUG FIX: The original code read histogram[_lNodesCur + _curLeavesNum] + // unconditionally. When _curLeavesNum == dict_size - _lNodesCur (i.e., + // all remaining leaf nodes qualify for the batch), the index equals + // dict_size, which is one past the end of the nz_dict_size-sized + // histogram array, causing a GPU memory access fault. + // FIX: Guard the histogram access with the bounds check + // (_lNodesCur + _curLeavesNum < dict_size). + // When there is no next leaf outside the batch, the condition is treated + // as false and execution falls through to the else branch (remove one + // leaf from the batch instead). + else if (((*status((IDX)_iNodesSize)) != 0) and + ((*status((IDX)_curLeavesNum)) == 0 or + (((*status((IDX)_lNodesCur)) + (*status((IDX)_curLeavesNum)) < + dict_size) and + (*histogram((IDX)(*status((IDX)_lNodesCur)) + + (*status((IDX)_curLeavesNum))) <= + *iNodesFreq((IDX)MOD((*status((IDX)_iNodesRear)) - 1, + dict_size)))))) { + (*status((IDX)_mergeRear)) = + MOD((*status((IDX)_mergeRear)) - 1, dict_size); + (*status((IDX)_iNodesFront)) = + MOD((*status((IDX)_iNodesRear)) - 1, dict_size); + // printf("remove one internal node from merging list. _mergeRear = + // %d\n", *status((IDX)_mergeRear)); printf("add one to the internal + // node list. _iNodesFront = %d\n", *status((IDX)_iNodesFront)); + } else { + (*status((IDX)_iNodesFront)) = (*status((IDX)_iNodesRear)); + --(*status((IDX)_curLeavesNum)); + // printf("empty internal list since all will be merged\n"); + // printf("remove highest frequency leaf node from merge list. + // _curLeavesNum = %d\n", *status((IDX)_curLeavesNum)); + } + + // Exclude all participating leaf nodes from unprocessed leaf node list + (*status((IDX)_lNodesCur)) = + (*status((IDX)_lNodesCur)) + (*status((IDX)_curLeavesNum)); + + // printf("exclude all participating leaf nodes from unprocessed leaf node + // list. _lNodesCur = %d\n", *status((IDX)_lNodesCur)); + + // Update _iNodesRear for the new internal node (in Operation2) + (*status((IDX)_iNodesRear)) = + MOD((*status((IDX)_iNodesRear)) + 1, dict_size); + // printf("include the new internal node. _iNodesRear = %d\n", + // *status((IDX)_iNodesRear)); + + // Calculate number of node participating parallel merge + (*status((IDX)_tempLength)) = + ((*status((IDX)_curLeavesNum)) - 0) + + MOD((*status((IDX)_mergeRear)) - (*status((IDX)_mergeFront)), + dict_size); + // printf("final number of nodes for parallel merge = %d\n", + // *status((IDX)_tempLength)); + } + } + + MGARDX_CONT_EXEC bool BranchCondition1() { + // If we have nodes to be parallel merged + return (*status((IDX)_tempLength)) > 0; + } + + MGARDX_EXEC void Operation6() { + + int32_t *sm = (int32_t *)FunctorBase::GetSharedMemory(); + x_top = &sm[0]; + y_top = &sm[1]; + x_bottom = &sm[2]; + y_bottom = &sm[3]; + found = &sm[4]; + oneorzero = &sm[5]; + + // (*status((IDX)_tempLength)) = (cEnd - 0) + MOD((*status((IDX)_mergeRear)) + // - (*status((IDX)_mergeFront)), dict_size); if + // ((*status((IDX)_tempLength)) == 0) return; + A_length = (*status((IDX)_curLeavesNum)) - 0; + B_length = MOD((*status((IDX)_mergeRear)) - (*status((IDX)_mergeFront)), + dict_size); + + // if (!thread) { + // printf("A_length: %d, B_length: %d, (*status((IDX)_tempLength)): %d\n", + // A_length, B_length, (*status((IDX)_tempLength))); + // } + // Calculate combined index around the MergePath "matrix" + combinedIndex = ((uint64_t)FunctorBase::GetBlockIdX() * + ((uint64_t)A_length + (uint64_t)B_length)) / + (uint64_t)FunctorBase::GetGridDimX(); + + // if (!FunctorBase::GetThreadIdX()) { + // printf("A_length: %d, B_length: %d, (*status((IDX)_tempLength)): %d, + // combinedIndex: %d\n", A_length, B_length, (*status((IDX)_tempLength)), + // combinedIndex); + // } + threadOffset = FunctorBase::GetThreadIdX() - warp_size / 2; + + if (FunctorBase::GetThreadIdX() < warp_size) { + // Figure out the coordinates of our diagonal + if (A_length >= B_length) { + *x_top = MIN(combinedIndex, A_length); + *y_top = combinedIndex > A_length ? combinedIndex - (A_length) : 0; + *x_bottom = *y_top; + *y_bottom = *x_top; + } else { + *y_bottom = MIN(combinedIndex, B_length); + *x_bottom = combinedIndex > B_length ? combinedIndex - (B_length) : 0; + *y_top = *x_bottom; + *x_top = *y_bottom; + } + } + *found = 0; + } + + MGARDX_EXEC bool LoopCondition2() { + // printf("%u LoopCondition2\n", FunctorBase::GetBlockIdX()); + return !(*found); + } + + MGARDX_EXEC void Operation7() { + // Update our coordinates within the 32-wide section of the diagonal + // if (!FunctorBase::GetThreadIdX()) { + // printf("x %d %d y: %d %d\n", *x_top, *x_bottom, *y_top, *y_bottom); + // } + current_x = *x_top - ((*x_top - *x_bottom) >> 1) - threadOffset; + current_y = *y_top + ((*y_bottom - *y_top) >> 1) + threadOffset; + getfrom_x = current_x + 0 - 1; + // Below statement is a more efficient, divmodless version of the following + // int32_t getfrom_y = MOD((*status((IDX)_mergeFront)) + current_y, + // dict_size); + getfrom_y = (*status((IDX)_mergeFront)) + current_y; + + if (FunctorBase::GetThreadIdX() < warp_size) { + if (getfrom_y >= dict_size) + getfrom_y -= dict_size; + + // Are we a '1' or '0' with respect to A[x] <= B[x] + if (current_x > (int32_t)A_length or current_y < 0) { + oneorzero[FunctorBase::GetThreadIdX()] = 0; + } else if (current_y >= (int32_t)B_length || current_x < 1) { + oneorzero[FunctorBase::GetThreadIdX()] = 1; + } else { + oneorzero[FunctorBase::GetThreadIdX()] = + (*copyFreq(getfrom_x) <= *iNodesFreq(getfrom_y)) ? 1 : 0; + } + } + } + + MGARDX_EXEC void Operation8() { + // if (!FunctorBase::GetThreadIdX()) + // printf("%d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d \n\ + // %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d \n\n", + // oneorzero[0], oneorzero[1], oneorzero[2], oneorzero[3], + // oneorzero[4], oneorzero[5], oneorzero[6], oneorzero[7], + // oneorzero[8], oneorzero[9], oneorzero[10], oneorzero[11], + // oneorzero[12], oneorzero[13], oneorzero[14], oneorzero[15], + // oneorzero[16], oneorzero[17], oneorzero[18], oneorzero[19], + // oneorzero[20], oneorzero[21], oneorzero[22], oneorzero[23], + // oneorzero[24], oneorzero[25], oneorzero[26], oneorzero[27], + // oneorzero[28], oneorzero[29], oneorzero[30], oneorzero[31]); + // If we find the meeting of the '1's and '0's, we found the + // intersection of the path and diagonal + if (FunctorBase::GetThreadIdX() > 0 and // + FunctorBase::GetThreadIdX() < warp_size and // + (oneorzero[FunctorBase::GetThreadIdX()] != + oneorzero[FunctorBase::GetThreadIdX() - 1]) // + ) { + // printf("found\n"); + *found = 1; + + *diagonal_path_intersections( + (IDX)FunctorBase::GetBlockIdX()) = current_x; + *diagonal_path_intersections((IDX)FunctorBase::GetBlockIdX() + + FunctorBase::GetGridDimX() + 1) = + current_y; + } + } + + MGARDX_EXEC void Operation9() { + // Adjust the search window on the diagonal + if (FunctorBase::GetThreadIdX() == warp_size / 2) { + if (oneorzero[warp_size - 1] != 0) { + *x_bottom = current_x; + *y_bottom = current_y; + } else { + *x_top = current_x; + *y_top = current_y; + } + } + } + + // end of loop 2 + + MGARDX_EXEC void Operation10() { + // Set the boundary diagonals (through 0,0 and A_length,B_length) + if (FunctorBase::GetThreadIdX() == 0 && + FunctorBase::GetBlockIdX() == 0) { + *diagonal_path_intersections((IDX)0) = 0; + *diagonal_path_intersections((IDX)FunctorBase::GetGridDimX() + + 1) = 0; + *diagonal_path_intersections( + (IDX)FunctorBase::GetGridDimX()) = A_length; + *diagonal_path_intersections((IDX)FunctorBase::GetGridDimX() + + FunctorBase::GetGridDimX() + 1) = + B_length; + } + } + + MGARDX_EXEC void Operation11() { + if (FunctorBase::GetThreadIdX() == 0) { + // Boundaries + int x_block_top = *diagonal_path_intersections( + (IDX)FunctorBase::GetBlockIdX()); + int y_block_top = *diagonal_path_intersections( + (IDX)FunctorBase::GetBlockIdX() + + FunctorBase::GetGridDimX() + 1); + int x_block_stop = *diagonal_path_intersections( + (IDX)FunctorBase::GetBlockIdX() + 1); + int y_block_stop = *diagonal_path_intersections( + (IDX)FunctorBase::GetBlockIdX() + + FunctorBase::GetGridDimX() + 2); + + // Actual indexes + int x_start = x_block_top + 0; + int x_end = x_block_stop + 0; + int y_start = MOD((*status((IDX)_mergeFront)) + y_block_top, dict_size); + int y_end = MOD((*status((IDX)_mergeFront)) + y_block_stop, dict_size); + + int offset = x_block_top + y_block_top; + + // printf("x_block_top: %d y_block_top: %d, offset: %d\n", x_block_top, + // y_block_top, offset); + + int dummy; // Unused result + // TODO optimize serial merging of each partition + int len = 0; + + int iterCopy = x_start, iterINodes = y_start; + + while (iterCopy < x_end && MOD(y_end - iterINodes, dict_size) > 0) { + if (*copyFreq((IDX)iterCopy) <= *iNodesFreq((IDX)iterINodes)) { + *tempFreq((IDX)offset + len) = *copyFreq((IDX)iterCopy); + *tempIndex((IDX)offset + len) = *copyIndex((IDX)iterCopy); + *tempIsLeaf((IDX)offset + len) = *copyIsLeaf((IDX)iterCopy); + ++iterCopy; + } else { + *tempFreq((IDX)offset + len) = *iNodesFreq((IDX)iterINodes); + *tempIndex((IDX)offset + len) = iterINodes; + *tempIsLeaf((IDX)offset + len) = 0; + iterINodes = MOD(iterINodes + 1, dict_size); + } + ++len; + } + + while (iterCopy < x_end) { + *tempFreq((IDX)offset + len) = *copyFreq((IDX)iterCopy); + *tempIndex((IDX)offset + len) = *copyIndex((IDX)iterCopy); + *tempIsLeaf((IDX)offset + len) = *copyIsLeaf((IDX)iterCopy); + ++iterCopy; + ++len; + } + while (MOD(y_end - iterINodes, dict_size) > 0) { + *tempFreq((IDX)offset + len) = *iNodesFreq((IDX)iterINodes); + *tempIndex((IDX)offset + len) = iterINodes; + *tempIsLeaf((IDX)offset + len) = 0; + iterINodes = MOD(iterINodes + 1, dict_size); + ++len; + } + + // for (int i = 0; i < len; i++) { + // if (*tempIsLeaf((IDX)offset+i) == 2 ) { + // printf("*copyIsLeaf((IDX)iterCopy) = 2\n"); + // } + // } + + // if (FunctorBase::GetThreadIdX() == 0) { + // printf("FunctorBase::GetGridDimX(): %llu, offset: %d, + // len: %d\n", FunctorBase::GetGridDimX(), offset, len); + // printf("leaf: %d %d\n", *tempIsLeaf((IDX)2 * 4), *tempIsLeaf((IDX)2 * + // 4 + 1)); + // } + } + + // cg::this_grid().sync(); + // if (FunctorBase::GetThreadIdX() == 0) { + // // printf("FunctorBase::GetGridDimX(): %u, offset: %d, len: + // %d\n", FunctorBase::GetGridDimX(), offset, len); + // printf("leaf: %d %d\n", *tempIsLeaf((IDX)2 * 4), *tempIsLeaf((IDX)2 * 4 + // + 1)); + // } + } + + // end of parallel merge + + MGARDX_EXEC void Operation12() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + // if (thread == 0) { + // printf("leaf: %d %d\n", *tempIsLeaf((IDX)2 * 4), *tempIsLeaf((IDX)2 * 4 + // + 1)); + // } + /* Melding phase -- New */ + // if (i == 0) { + // printf("parallel merge\n"); + // } + if (i < (*status((IDX)_tempLength)) / 2) { + // Index of new internal nodes + int ind = MOD((*status((IDX)_iNodesRear)) + i, dict_size); + // printf("Melding(i=%d): %u(%d) %u(%d)\n", i, *tempFreq((IDX)2 * i), + // *tempIsLeaf((IDX)2 * i), *tempFreq((IDX)2 * i + 1), *tempIsLeaf((IDX)2 + // * i + 1)); + // Calculate the combined frequency of the new internel node + *iNodesFreq((IDX)ind) = *tempFreq((IDX)2 * i) + *tempFreq((IDX)2 * i + 1); + // New internel node should not have leader + *iNodesLeader((IDX)ind) = -1; + + // If a merge node is leaf node, increase CL by 1 + if (*tempIsLeaf((IDX)2 * i)) { + *lNodesLeader((IDX)*tempIndex((IDX)2 * i)) = ind; + ++(*CL(*tempIndex(2 * i))); + } else { + *iNodesLeader((IDX)*tempIndex((IDX)2 * i)) = ind; + } + if (*tempIsLeaf((IDX)2 * i + 1)) { + *lNodesLeader((IDX)*tempIndex((IDX)2 * i + 1)) = ind; + ++(*CL(*tempIndex(2 * i + 1))); + } else { + *iNodesLeader((IDX)*tempIndex((IDX)2 * i + 1)) = ind; + } + } + } + + MGARDX_EXEC void Operation13() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (i == 0) { + // printf("update _iNodesRear to include new internel nodes\n"); + (*status((IDX)_iNodesRear)) = + MOD((*status((IDX)_iNodesRear)) + ((*status((IDX)_tempLength)) / 2), + dict_size); + } + } + + MGARDX_EXEC void Operation14() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + /* Update leaders */ + // if (thread == 0) { + // printf("mine: iNodesLeader(0.leader) = %d, (*status((IDX)_iNodesRear)): + // %u\n", *iNodesLeader(IDX(0)), (*status((IDX)_iNodesRear))); + // } + if (i < dict_size) { + // For all leaf node that is already merged, we check if we need to update + // their leader and CL + if (*lNodesLeader((IDX)i) != -1) { + // If the previous leader had leader (new highest leader) + if (*iNodesLeader((IDX)*lNodesLeader((IDX)i)) != -1) { + // We keep doing this to make sure to track highest leader + // After this '*lNodesLeader((IDX)i)' should always points to the + // highest leader + *lNodesLeader((IDX)i) = *iNodesLeader((IDX)*lNodesLeader((IDX)i)); + // If we update leader, it means we have a new leader result from a + // merge so, we need to update CL + ++(*CL((IDX)i)); + // printf("update CL(%d):%d\n", i, *CL(i)); + } + } + } + } + + MGARDX_EXEC void Operation15() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (i == 0) { + (*status((IDX)_iNodesSize)) = + MOD((*status((IDX)_iNodesRear)) - (*status((IDX)_iNodesFront)), + dict_size); + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t sm_size = 0; + sm_size += 5 * sizeof(int32_t); + sm_size += warp_size * sizeof(int32_t); + return sm_size; + } + +private: + SubArray<1, T, DeviceType> histogram; + SubArray<1, T, DeviceType> CL; + int dict_size; + /* Global Arrays */ + SubArray<1, T, DeviceType> lNodesFreq; + SubArray<1, int, DeviceType> lNodesLeader; + SubArray<1, T, DeviceType> iNodesFreq; + SubArray<1, int, DeviceType> iNodesLeader; + SubArray<1, T, DeviceType> tempFreq; + SubArray<1, int, DeviceType> tempIsLeaf; + SubArray<1, int, DeviceType> tempIndex; + SubArray<1, T, DeviceType> copyFreq; + SubArray<1, int, DeviceType> copyIsLeaf; + SubArray<1, int, DeviceType> copyIndex; + SubArray<1, uint32_t, DeviceType> diagonal_path_intersections; + SubArray<1, int, DeviceType, false, true> status; + SIZE warp_size; + + int32_t *x_top; + int32_t *y_top; + int32_t *x_bottom; + int32_t *y_bottom; + int32_t *found; + int32_t *oneorzero; + + unsigned int i; + + int threadOffset; + uint32_t A_length; + uint32_t B_length; + int32_t combinedIndex; + int32_t current_x; + int32_t current_y; + int32_t getfrom_x; + int32_t getfrom_y; +}; + +template +class GenerateCLKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "generate codeword length"; + MGARDX_CONT + GenerateCLKernel( + SubArray<1, T, DeviceType> histogram, SubArray<1, T, DeviceType> CL, + int dict_size, + /* Global Arrays */ + SubArray<1, T, DeviceType> lNodesFreq, + SubArray<1, int, DeviceType> lNodesLeader, + SubArray<1, T, DeviceType> iNodesFreq, + SubArray<1, int, DeviceType> iNodesLeader, + SubArray<1, T, DeviceType> tempFreq, + SubArray<1, int, DeviceType> tempIsLeaf, + SubArray<1, int, DeviceType> tempIndex, + SubArray<1, T, DeviceType> copyFreq, + SubArray<1, int, DeviceType> copyIsLeaf, + SubArray<1, int, DeviceType> copyIndex, + SubArray<1, uint32_t, DeviceType> diagonal_path_intersections, + SubArray<1, int, DeviceType, false, true> status) + : histogram(histogram), CL(CL), dict_size(dict_size), + lNodesFreq(lNodesFreq), lNodesLeader(lNodesLeader), + iNodesFreq(iNodesFreq), iNodesLeader(iNodesLeader), tempFreq(tempFreq), + tempIsLeaf(tempIsLeaf), tempIndex(tempIndex), copyFreq(copyFreq), + copyIsLeaf(copyIsLeaf), copyIndex(copyIndex), + diagonal_path_intersections(diagonal_path_intersections), + status(status) {} + + // Block width of GenerateCL's merge-path search, and the number of uint32 + // entries diagonal_path_intersections must hold for it. + // + // These have to agree with GenTask below and with whoever allocates the + // array (HuffmanWorkspace), so they live here as the single definition. + // The kernel writes indices up to 2 * gridDim.x + 1 (see Operation7/8), and + // GenTask launches ceil(dict_size / block width) blocks -- sizing the array + // from an occupancy estimate instead overflows it whenever the dictionary + // needs more blocks than the device is estimated to run at once. + MGARDX_CONT static SIZE BlockWidth() { + if constexpr (std::is_same::value) { + // Empirically tuned on MI300: the merge-path search here uses shared + // memory + explicit inter-Operation sync barriers rather than raw + // warp-shuffle/ballot, so its "width" is really just a block size and + // isn't tied to the hardware wavefront. A direct sweep (64/128/256/ + // 512/1024) on the real 64-wide CDNA wavefront found throughput + // climbing well past 64 -- 64 (the true wavefront size) measures + // ~86 GB/s, 512 measures 110-114 GB/s, a ~30% further gain. 1024 (the + // hardware max block width) also completes safely here (unlike + // OutlierSeparator's tbx, which hangs at 1024) but measures ~111 GB/s, + // matching 512 within noise, so 512 is used as the smallest width that + // captures the full gain. Revisit with a fresh sweep if this kernel's + // algorithm changes. + return 512; + } else { + return DeviceRuntime::GetWarpSize(); + } + } + + MGARDX_CONT static SIZE DiagonalPathIntersectionsSize(SIZE dict_size) { + SIZE gridx = (dict_size - 1) / BlockWidth() + 1; + // Keep the old occupancy-derived size as a floor so no device shrinks. + SIZE occupancy_blocks = + (DeviceRuntime::GetMaxNumThreadsPerTB() / + DeviceRuntime::GetWarpSize()) * + DeviceRuntime::GetNumSMs(); + return 2 * (std::max(gridx, occupancy_blocks) + 1); + } + + MGARDX_CONT + Task> GenTask(int queue_idx) { + using FunctorType = GenerateCLFunctor; + SIZE warp_size = BlockWidth(); + FunctorType Functor(histogram, CL, dict_size, lNodesFreq, lNodesLeader, + iNodesFreq, iNodesLeader, tempFreq, tempIsLeaf, + tempIndex, copyFreq, copyIsLeaf, copyIndex, + diagonal_path_intersections, status, warp_size); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = Functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = warp_size; + + int cg_blocks_sm = + DeviceRuntime::GetOccupancyMaxActiveBlocksPerSM( + Functor, tbx, sm_size); + int cg_mblocks = cg_blocks_sm * DeviceRuntime::GetNumSMs(); + + gridz = 1; + gridy = 1; + gridx = cg_mblocks; + + int tthreads = tbx * gridx; + if (tthreads >= dict_size) { + if (DeviceRuntime::PrintKernelConfig) { + std::cout << log::log_info << "GenerateCL: using Cooperative Groups\n"; + } + Functor.use_CG = true; + } else { + if (DeviceRuntime::PrintKernelConfig) { + std::cout << log::log_info + << "GenerateCL: not using Cooperative Groups\n"; + } + Functor.use_CG = false; + } + gridx = (dict_size - 1) / tbx + 1; + + return Task(Functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T, DeviceType> histogram; + SubArray<1, T, DeviceType> CL; + int dict_size; + SubArray<1, T, DeviceType> lNodesFreq; + SubArray<1, int, DeviceType> lNodesLeader; + SubArray<1, T, DeviceType> iNodesFreq; + SubArray<1, int, DeviceType> iNodesLeader; + SubArray<1, T, DeviceType> tempFreq; + SubArray<1, int, DeviceType> tempIsLeaf; + SubArray<1, int, DeviceType> tempIndex; + SubArray<1, T, DeviceType> copyFreq; + SubArray<1, int, DeviceType> copyIsLeaf; + SubArray<1, int, DeviceType> copyIndex; + SubArray<1, uint32_t, DeviceType> diagonal_path_intersections; + SubArray<1, int, DeviceType, false, true> status; +}; + +#undef MOD +#undef MIN +#undef MAX + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/GenerateCW.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/GenerateCW.hpp new file mode 100644 index 000000000..7887516be --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/GenerateCW.hpp @@ -0,0 +1,319 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_GENERATE_CW_TEMPLATE_HPP +#define MGARD_X_160_GENERATE_CW_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +// GenerateCW Locals +#define _CCL 0 +#define _CDPI 1 +#define _newCDPI 2 + +#define _updateEnd 3 +#define _curEntryVal 4 +#define _numCCL 5 + +template +class GenerateCWFunctor : public HuffmanCWCustomizedFunctor { +public: + MGARDX_CONT GenerateCWFunctor() {} + MGARDX_CONT GenerateCWFunctor( + SubArray<1, T, DeviceType> CL, SubArray<1, H, DeviceType> CW, + SubArray<1, H, DeviceType> first, SubArray<1, H, DeviceType> entry, + SIZE dict_size, SubArray<1, int, DeviceType, false, true> status) + : CL(CL), CW(CW), first(first), entry(entry), dict_size(dict_size), + status(status) { + HuffmanCWCustomizedFunctor(); + } + + MGARDX_EXEC void Operation1() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + // i = thread; // Porting convenience + type_bw = sizeof(H) * 8; + + /* Reverse in place - Probably a more CUDA-appropriate way */ + if (i < dict_size / 2) { + T temp = *CL((IDX)i); + *CL((IDX)i) = *CL((IDX)dict_size - i - 1); + *CL((IDX)dict_size - i - 1) = temp; + } + } + + MGARDX_EXEC void Operation2() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (i == 0) { + (*status((IDX)_CCL)) = *CL((IDX)0); + (*status((IDX)_CDPI)) = 0; + (*status((IDX)_newCDPI)) = dict_size - 1; + *entry((IDX)(*status((IDX)_CCL))) = 0; + + // Edge case -- only one input symbol + *CW((IDX)(*status((IDX)_CDPI))) = 0; + *first((IDX)(*status((IDX)_CCL))) = + *CW((IDX)(*status((IDX)_CDPI))) ^ + (((H)1 << (H)*CL((IDX)(*status((IDX)_CDPI)))) - 1); + *entry((IDX)(*status((IDX)_CCL)) + 1) = 1; + } + } + + MGARDX_EXEC void Operation3() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + // Initialize first and entry arrays + if (i < (*status((IDX)_CCL))) { + // Initialization of first to Max ensures that unused code + // lengths are skipped over in decoding. + *first((IDX)i) = std::numeric_limits::max(); + *entry((IDX)i) = 0; + } + } + + MGARDX_CONT_EXEC bool LoopCondition1() { + // if (! thread) + // printf("thread: %u, (*status((IDX)_CDPI)): %d, (*status((IDX)_newCDPI)): + // %d, dict_size: %u\n", thread, (*status((IDX)_CDPI)), + // (*status((IDX)_newCDPI)), dict_size); + return (*status((IDX)_CDPI)) < dict_size - 1; + } + + MGARDX_EXEC void Operation4() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + // (*status((IDX)_CDPI)) update + if (i < dict_size - 1 && *CL((IDX)i + 1) > (*status((IDX)_CCL))) { + Atomic::Min( + &(*status((IDX)_newCDPI)), (int)i); + } + } + + MGARDX_EXEC void Operation5() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + type_bw = sizeof(H) * 8; + // Last element to update + (*status((IDX)_updateEnd)) = ((*status((IDX)_newCDPI)) >= dict_size - 1) + ? type_bw + : *CL((IDX)(*status((IDX)_newCDPI)) + 1); + // Fill base + (*status((IDX)_curEntryVal)) = *entry((IDX)(*status((IDX)_CCL))); + // Number of elements of length (*status((IDX)_CCL)) + (*status((IDX)_numCCL)) = + ((*status((IDX)_newCDPI)) - (*status((IDX)_CDPI)) + 1); + + // Get first codeword + if (i == 0) { + if ((*status((IDX)_CDPI)) == 0) { + *CW((IDX)(*status((IDX)_newCDPI))) = 0; + } else { + *CW((IDX)(*status((IDX)_newCDPI))) = + *CW((IDX)(*status((IDX)_CDPI))); // Pre-stored + } + } + } + + MGARDX_EXEC void Operation6() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + type_bw = sizeof(H) * 8; + if (i < dict_size) { + // Parallel canonical codeword generation + if (i >= (*status((IDX)_CDPI)) && i < (*status((IDX)_newCDPI))) { + *CW((IDX)i) = + *CW((IDX)(*status((IDX)_newCDPI))) + ((*status((IDX)_newCDPI)) - i); + } + } + + // Update entry and first arrays in O(1) time + // Jieyang: not useful? + if (i > (*status((IDX)_CCL)) && i < (*status((IDX)_updateEnd))) { + *entry((IDX)i) = (*status((IDX)_curEntryVal)) + (*status((IDX)_numCCL)); + } + // Add number of entries to next (*status((IDX)_CCL)) + if (i == 0) { + if ((*status((IDX)_updateEnd)) < type_bw) { + *entry((IDX)(*status((IDX)_updateEnd))) = + (*status((IDX)_curEntryVal)) + (*status((IDX)_numCCL)); + } + } + } + + MGARDX_EXEC void Operation7() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + // Update first array in O(1) time + if (i == (*status((IDX)_CCL))) { + // Flip least significant CL[(*status((IDX)_CDPI))] bits + *first((IDX)(*status((IDX)_CCL))) = + *CW((IDX)(*status((IDX)_CDPI))) ^ + (((H)1 << (H)*CL((IDX)(*status((IDX)_CDPI)))) - 1); + // printf("first[%d]: %llu\n", (*status((IDX)_CCL)), + // first[(*status((IDX)_CCL))]); + } + if (i > (*status((IDX)_CCL)) && i < (*status((IDX)_updateEnd))) { + *first((IDX)i) = std::numeric_limits::max(); + } + } + + MGARDX_EXEC void Operation8() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (i == 0) { + if ((*status((IDX)_newCDPI)) < dict_size - 1) { + int CLDiff = *CL((IDX)(*status((IDX)_newCDPI)) + 1) - + *CL((IDX)(*status((IDX)_newCDPI))); + // Add and shift -- Next canonical code + + *CW((IDX)(*status((IDX)_newCDPI)) + 1) = + ((*CW((IDX)(*status((IDX)_CDPI))) + 1) << CLDiff); + (*status((IDX)_CCL)) = *CL((IDX)(*status((IDX)_newCDPI)) + 1); + + H temp = (*CW((IDX)(*status((IDX)_CDPI))) + 1); + + // printf("CLDiff: %d %llu <- %llu\n", CLDiff, (H)(temp << CLDiff), + // temp); + + ++(*status((IDX)_newCDPI)); + } + + // Update (*status((IDX)_CDPI)) to (*status((IDX)_newCDPI)) after codeword + // length increase + (*status((IDX)_CDPI)) = (*status((IDX)_newCDPI)); + (*status((IDX)_newCDPI)) = dict_size - 1; + // printf("thread: %u, (*status((IDX)_CDPI)): %d, + // (*status((IDX)_newCDPI)): %d, dict_size: %u\n", thread, + // (*status((IDX)_CDPI)), + // (*status((IDX)_newCDPI)), dict_size); + } + } + + MGARDX_EXEC void Operation9() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + // encoding CL into CW (highest 8 bits) + if (i < dict_size) { + *CW((IDX)i) = (*CW((IDX)i) | + (((H)*CL((IDX)i) & (H)0xffu) << ((sizeof(H) * 8) - 8))) ^ + (((H)1 << (H)*CL((IDX)i)) - 1); + // printf("flip: %llu ^ 1 << %u -> %llu\n", *CW((IDX)i), *CL((IDX)i), + // (*CW((IDX)i)) ^ (((H)1 << (H)*CL((IDX)i)) - 1) ); *CW((IDX)i) = + // (*CW((IDX)i)) ^ + // (((H)1 << (H)*CL((IDX)i)) - 1); + } + } + + MGARDX_EXEC void Operation10() { + i = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + /* Reverse partial codebook */ + + if (i < dict_size / 2) { + // printf("Reverse\n"); + H temp = *CW((IDX)i); + *CW((IDX)i) = *CW((IDX)dict_size - i - 1); + *CW((IDX)dict_size - i - 1) = temp; + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T, DeviceType> CL; + SubArray<1, H, DeviceType> CW; + SubArray<1, H, DeviceType> first; + SubArray<1, H, DeviceType> entry; + SubArray<1, int, DeviceType, false, true> status; + SIZE dict_size; + + // unsigned int thread; + unsigned int i; + size_t type_bw; + + // int (*status((IDX)_updateEnd)); + // int (*status((IDX)_curEntryVal)); + // int (*status((IDX)_numCCL)); +}; + +template +class GenerateCWKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "generate codeword"; + MGARDX_CONT + GenerateCWKernel(SubArray<1, T, DeviceType> CL, SubArray<1, H, DeviceType> CW, + SubArray<1, H, DeviceType> first, + SubArray<1, H, DeviceType> entry, SIZE dict_size, + SubArray<1, int, DeviceType, false, true> status) + : CL(CL), CW(CW), first(first), entry(entry), dict_size(dict_size), + status(status) {} + + MGARDX_CONT + Task> GenTask(int queue_idx) { + using FunctorType = GenerateCWFunctor; + FunctorType Functor(CL, CW, first, entry, dict_size, status); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = Functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = DeviceRuntime::GetMaxNumThreadsPerTB(); + + int nz_nblocks = (dict_size / tbx) + 1; + int cg_blocks_sm = + DeviceRuntime::GetOccupancyMaxActiveBlocksPerSM( + Functor, tbx, sm_size); + int cg_mblocks = cg_blocks_sm * DeviceRuntime::GetNumSMs(); + gridz = 1; + gridy = 1; + gridx = cg_mblocks; + + int cw_tthreads = gridx * tbx; + if (cw_tthreads >= dict_size) { + if (DeviceRuntime::PrintKernelConfig) { + std::cout << log::log_info << "GenerateCW: using Cooperative Groups\n"; + } + Functor.use_CG = true; + } else { + if (DeviceRuntime::PrintKernelConfig) { + std::cout << log::log_info + << "GenerateCW: not using Cooperative Groups\n"; + } + Functor.use_CG = false; + } + gridx = (dict_size - 1) / tbx + 1; + + return Task(Functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T, DeviceType> CL; + SubArray<1, H, DeviceType> CW; + SubArray<1, H, DeviceType> first; + SubArray<1, H, DeviceType> entry; + SubArray<1, int, DeviceType, false, true> status; + SIZE dict_size; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/GetCodebook.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/GetCodebook.hpp new file mode 100644 index 000000000..61589b349 --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/GetCodebook.hpp @@ -0,0 +1,156 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../../RuntimeX/Utilities/Exceptions.h" +#include "EntropyCalculator.hpp" +#include "FillArraySequence.hpp" +#include "GenerateCL.hpp" +#include "GenerateCW.hpp" +#include "GetFirstNonzeroIndex.hpp" +#include "HuffmanWorkspace.hpp" +#include "ReorderByIndex.hpp" +#include "ReverseArray.hpp" + +#include +#include + +#ifndef MGARD_X_160_GET_CODEBOOK_TEMPLATE_HPP +#define MGARD_X_160_GET_CODEBOOK_TEMPLATE_HPP + +namespace mgard_x_160 { +// Parallel codebook generation wrapper +template +void GetCodebook(int dict_size, size_t primary_count, + SubArray<1, unsigned int, DeviceType> _d_freq_subarray, + SubArray<1, H, DeviceType> _d_codebook_subarray, + SubArray<1, uint8_t, DeviceType> _d_decode_meta_subarray, + HuffmanWorkspace &workspace, + int queue_idx) { + // Metadata + auto type_bw = sizeof(H) * 8; + + SubArray<1, H, DeviceType> _d_first_subarray( + {(SIZE)type_bw}, (H *)_d_decode_meta_subarray((IDX)0)); + SubArray<1, H, DeviceType> _d_entry_subarray( + {(SIZE)type_bw}, (H *)_d_decode_meta_subarray(sizeof(H) * type_bw)); + SubArray<1, Q, DeviceType> _d_qcode_subarray( + {(SIZE)dict_size}, (Q *)_d_decode_meta_subarray(sizeof(H) * 2 * type_bw)); + + // Sort Qcodes by frequency + DeviceLauncher::Execute( + FillArraySequenceKernel(_d_qcode_subarray), queue_idx); + + MemoryManager::Copy1D(workspace._d_freq_copy_subarray.data(), + _d_freq_subarray.data(), dict_size, + queue_idx); + MemoryManager::Copy1D(workspace._d_qcode_copy_subarray.data(), + _d_qcode_subarray.data(), dict_size, + queue_idx); + DeviceCollective::SortByKey( + (SIZE)dict_size, workspace._d_freq_copy_subarray, + workspace._d_qcode_copy_subarray, _d_freq_subarray, _d_qcode_subarray, + workspace.sort_by_key_workspace, true, queue_idx); + + DeviceLauncher::Execute( + GetFirstNonzeroIndexKernel( + _d_freq_subarray, workspace.first_nonzero_index_subarray), + queue_idx); + + unsigned int first_nonzero_index; + MemoryManager().Copy1D( + &first_nonzero_index, workspace.first_nonzero_index_subarray(IDX(0)), 1, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + if (debug_print_huffman) { + PrintSubarray("SortByKey::_d_freq_subarray", _d_freq_subarray); + PrintSubarray("SortByKey::_d_qcode_subarray", _d_qcode_subarray); + // std::cout << "first_nonzero_index: " << first_nonzero_index << std::endl; + } + + // DumpSubArray("freq_"+std::to_string(workspace.huff_array.shape(0))+".dat", + // _d_freq_subarray); + + int nz_dict_size = dict_size - first_nonzero_index; + + SubArray<1, unsigned int, DeviceType> _nz_d_freq_subarray( + {(SIZE)nz_dict_size}, _d_freq_subarray(first_nonzero_index)); + SubArray<1, H, DeviceType> _nz_d_codebook_subarray( + {(SIZE)nz_dict_size}, _d_codebook_subarray(first_nonzero_index)); + + DeviceLauncher::Execute( + GenerateCLKernel( + _nz_d_freq_subarray, workspace.CL_subarray, nz_dict_size, + _nz_d_freq_subarray, workspace.lNodesLeader_subarray, + workspace.iNodesFreq_subarray, workspace.iNodesLeader_subarray, + workspace.tempFreq_subarray, workspace.tempIsLeaf_subarray, + workspace.tempIndex_subarray, workspace.copyFreq_subarray, + workspace.copyIsLeaf_subarray, workspace.copyIndex_subarray, + workspace.diagonal_path_intersections_subarray, + workspace.status_subarray), + queue_idx); + + unsigned int max_CL; + MemoryManager().Copy1D(&max_CL, workspace.CL_subarray(IDX(0)), 1, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + if (log::level & log::INFO) { + // PrintSubarray("GenerateCL::CL_subarray", workspace.CL_subarray); + // std::cout << "GenerateCL: max_CL: " << max_CL << std::endl; + double LC = CalculateLC(primary_count, nz_dict_size, _nz_d_freq_subarray, + workspace.CL_subarray, queue_idx); + double entropy = CalculateEntropy(primary_count, nz_dict_size, + _nz_d_freq_subarray, queue_idx); + log::dbg("LC: " + std::to_string(LC)); + log::dbg("Entropy: " + std::to_string(entropy)); + } + + // DumpSubArray("cl_"+std::to_string(workspace.huff_array.shape(0))+".dat", + // _d_freq_subarray); + + int max_CW_bits = (sizeof(H) * 8) - 8; + if (max_CL > max_CW_bits) { + // Throw (instead of exit) so callers can catch and fall back to another + // lossless backend (e.g. raw Zstd) or retry with a smaller huff_dict_size. + // A longer dictionary makes the tree deeper, so a degenerate/low-entropy + // input can produce codewords exceeding the H-type budget (sizeof(H)*8 - + // 8). + throw ProcessingException( + "Cannot store all Huffman codewords in " + + std::to_string(max_CW_bits + 8) + + "-bit representation; representation requires at least " + + std::to_string(max_CL + 8) + + " bits (longest codeword: " + std::to_string(max_CL) + " bits)"); + } + + DeviceLauncher::Execute( + GenerateCWKernel( + workspace.CL_subarray, _nz_d_codebook_subarray, _d_first_subarray, + _d_entry_subarray, nz_dict_size, workspace.status_subarray), + queue_idx); + + DeviceLauncher::Execute( + ReverseArrayKernel(_d_codebook_subarray), queue_idx); + DeviceLauncher::Execute( + ReverseArrayKernel(_d_qcode_subarray), queue_idx); + + MemoryManager().Copy1D(workspace._d_codebook_subarray_org.data(), + _d_codebook_subarray.data(), + _d_codebook_subarray.shape(0), queue_idx); + + DeviceLauncher::Execute( + ReorderByIndexKernel(workspace._d_codebook_subarray_org, + _d_codebook_subarray, + _d_qcode_subarray), + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); +} + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/GetFirstNonzeroIndex.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/GetFirstNonzeroIndex.hpp new file mode 100644 index 000000000..13fb6b4ec --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/GetFirstNonzeroIndex.hpp @@ -0,0 +1,78 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_GET_FIRST_NONZERO_INDEX_TEMPLATE_HPP +#define MGARD_X_160_GET_FIRST_NONZERO_INDEX_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +template +class GetFirstNonzeroIndexFunctor : public Functor { +public: + MGARDX_CONT GetFirstNonzeroIndexFunctor() {} + MGARDX_CONT GetFirstNonzeroIndexFunctor(SubArray<1, T, DeviceType> array, + SubArray<1, T, DeviceType> result) + : array(array), result(result) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + unsigned int thread = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (thread < array.shape(0) && *array(thread) != 0) { + Atomic::Min(result((IDX)0), thread); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T, DeviceType> array; + SubArray<1, T, DeviceType> result; +}; + +template +class GetFirstNonzeroIndexKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "get first non-zero"; + MGARDX_CONT + GetFirstNonzeroIndexKernel(SubArray<1, T, DeviceType> array, + SubArray<1, T, DeviceType> result) + : array(array), result(result) {} + + MGARDX_CONT + Task> GenTask(int queue_idx) { + using FunctorType = GetFirstNonzeroIndexFunctor; + FunctorType functor(array, result); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = DeviceRuntime::GetMaxNumThreadsPerTB(); + gridz = 1; + gridy = 1; + gridx = (array.shape(0) / tbx) + 1; + // printf("%u %u %u\n", shape.dataHost()[2], shape.dataHost()[1], + // shape.dataHost()[0]); PrintSubarray("shape", shape); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T, DeviceType> array; + SubArray<1, T, DeviceType> result; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/Histogram.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/Histogram.hpp new file mode 100644 index 000000000..479d80b4e --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/Histogram.hpp @@ -0,0 +1,251 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_HUFFMAN_HISTOGRAM_TEMPLATE_HPP +#define MGARD_X_160_HUFFMAN_HISTOGRAM_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { +template +class HistogramFunctor : public Functor { +public: + MGARDX_CONT HistogramFunctor() {} + MGARDX_CONT + HistogramFunctor(SubArray<1, T, DeviceType> input_data, + SubArray<1, int, DeviceType, false, true> local_histogram, + SubArray<1, Q, DeviceType> output, SIZE N, int bins, + int RPerBlock, SIZE warp_size) + : input_data(input_data), local_histogram(local_histogram), + output(output), N(N), bins(bins), RPerBlock(RPerBlock), + warp_size(warp_size) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + if (CACHE_HISTOGRAM) { + Hs = (int *)FunctorBase::GetSharedMemory(); + } else { + Hs = local_histogram(FunctorBase::GetBlockIdX() * RPerBlock * + bins); + } + + // Grid-stride partitioning granularity: this is arithmetic work + // partitioning (which "warp group" a thread belongs to for load + // balancing), not a hardware lockstep/shuffle assumption, so it is safe + // to size to the real warp/wavefront (DeviceRuntime::GetWarpSize(), + // threaded through as warp_size) rather than the CUDA-oriented + // MGARDX_WARP_SIZE=32 constant. Matching the real wavefront also keeps + // each warp's global-memory accesses (input_data(i) below) coalesced. + warpid = (int)(FunctorBase::GetThreadIdX() / warp_size); + lane = FunctorBase::GetThreadIdX() % warp_size; + warps_block = FunctorBase::GetBlockDimX() / warp_size; + + off_rep = (bins) * (FunctorBase::GetThreadIdX() % RPerBlock); + + begin = (N / warps_block) * warpid + + warp_size * FunctorBase::GetBlockIdX() + lane; + end = (N / warps_block) * (warpid + 1); + step = warp_size * FunctorBase::GetGridDimX(); + + // final warp handles data outside of the warps_block partitions + if (warpid >= warps_block - 1) + end = N; + + if (CACHE_HISTOGRAM) { + for (unsigned int pos = FunctorBase::GetThreadIdX(); + pos < (bins)*RPerBlock; + pos += FunctorBase::GetBlockDimX()) { + Hs[pos] = 0; + } + } + } + + MGARDX_EXEC void Operation2() { + for (unsigned int i = begin; i < end; i += step) { + int d = *input_data(i); + if (CACHE_HISTOGRAM) { + Atomic::Add( + &Hs[off_rep + d], 1); + } else { + Atomic::Add( + &Hs[off_rep + d], 1); + } + } + } + + MGARDX_EXEC void Operation3() { + for (unsigned int pos = FunctorBase::GetThreadIdX(); pos < bins; + pos += FunctorBase::GetBlockDimX()) { + int sum = 0; + for (int base = 0; base < (bins)*RPerBlock; base += bins) { + sum += Hs[base + pos]; + } + Atomic::Add( + output(pos), (Q)sum); + } + } + + MGARDX_EXEC void Operation4() {} + + MGARDX_EXEC void Operation5() {} + + MGARDX_CONT size_t shared_memory_size() { + if (CACHE_HISTOGRAM) { + size_t size = 0; + size = (bins)*RPerBlock * sizeof(int); + return size; + } else { + return 0; + } + } + +private: + SubArray<1, T, DeviceType> input_data; + SubArray<1, int, DeviceType, false, true> local_histogram; + SubArray<1, Q, DeviceType> output; + SIZE N; + int bins; + int RPerBlock; + SIZE warp_size; + + int *Hs; + + unsigned int warpid; + unsigned int lane; + unsigned int warps_block; + + unsigned int off_rep; + + unsigned int begin; + unsigned int end; + unsigned int step; +}; + +template +class HistogramKernel : public Kernel { +public: + constexpr static std::string_view Name = "histogram"; + constexpr static bool EnableAutoTuning() { return false; } + + MGARDX_CONT + HistogramKernel(SubArray<1, T, DeviceType> input_data, + SubArray<1, int, DeviceType, false, true> local_histogram, + SubArray<1, Q, DeviceType> output, SIZE N, int bins, + int RPerBlock, int threadsPerBlock, int numBlocks) + : input_data(input_data), local_histogram(local_histogram), + output(output), N(N), bins(bins), RPerBlock(RPerBlock), + threadsPerBlock(threadsPerBlock), numBlocks(numBlocks) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = HistogramFunctor; + + FunctorType functor(input_data, local_histogram, output, N, bins, RPerBlock, + DeviceRuntime::GetWarpSize()); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = threadsPerBlock; + gridz = 1; + gridy = 1; + gridx = numBlocks; + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T, DeviceType> input_data; + SubArray<1, int, DeviceType, false, true> local_histogram; + SubArray<1, Q, DeviceType> output; + SIZE N; + int bins; + int RPerBlock; + int threadsPerBlock; + int numBlocks; +}; + +template +MGARDX_CONT void ExecutionConfig(int N, int bins, int RPerBlock, + int &threadsPerBlock, int &numBlocks) { + int numSMs = DeviceRuntime::GetNumSMs(); + int numBuckets = bins; + int numValues = N; + int itemsPerThread = 1; + numBlocks = numSMs; + + threadsPerBlock = + ((((numValues / (numBlocks * itemsPerThread)) + 1) / 64) + 1) * 64; + while (threadsPerBlock > DeviceRuntime::GetMaxNumThreadsPerTB()) { + if (RPerBlock <= 1) { + threadsPerBlock = DeviceRuntime::GetMaxNumThreadsPerTB(); + } else { + RPerBlock /= 2; + numBlocks *= 2; + threadsPerBlock = + ((((numValues / (numBlocks * itemsPerThread)) + 1) / 64) + 1) * 64; + } + } +} + +template +MGARDX_CONT void Histogram(SubArray<1, T, DeviceType> input_data, + SubArray<1, Q, DeviceType> output, SIZE N, int bins, + int queue_idx) { + int maxbytes = DeviceRuntime::GetMaxSharedMemorySize(); + SubArray<1, int, DeviceType, false, true> local_histogram; + if (bins * sizeof(int) < maxbytes) { + if (DeviceRuntime::PrintKernelConfig) { + std::cout << log::log_info + << "Histogram: using shared memory for local histogram\n"; + } + using FunctorType = HistogramFunctor; + using TaskType = Task; + int RPerBlock = (maxbytes / (int)sizeof(int)) / (bins); + if (std::is_same::value || + std::is_same::value) { + RPerBlock = 1; + } + int threadsPerBlock, numBlocks; + ExecutionConfig(N, bins, RPerBlock, threadsPerBlock, numBlocks); + DeviceLauncher::Execute( + HistogramKernel(input_data, local_histogram, + output, N, bins, RPerBlock, + threadsPerBlock, numBlocks), + queue_idx); + } else { + if (DeviceRuntime::PrintKernelConfig) { + std::cout << log::log_info + << "Histogram: using global memory for local histogram\n"; + } + using FunctorType = HistogramFunctor; + using TaskType = Task; + int RPerBlock = 2; + int threadsPerBlock, numBlocks; + ExecutionConfig(N, bins, RPerBlock, threadsPerBlock, numBlocks); + Array<1, int, DeviceType, false, true> local_histogram_array( + {(SIZE)RPerBlock * bins * numBlocks}); + local_histogram_array.memset(0); + // TODO: can we not sync all queues? + DeviceRuntime::SyncAllQueues(); + local_histogram = SubArray(local_histogram_array); + DeviceLauncher::Execute( + HistogramKernel(input_data, local_histogram, + output, N, bins, RPerBlock, + threadsPerBlock, numBlocks), + queue_idx); + // Make sure the work is does before we free local_histogram_array + DeviceRuntime::SyncAllQueues(); + } +} + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/Huffman.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/Huffman.hpp new file mode 100644 index 000000000..88e6f1645 --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/Huffman.hpp @@ -0,0 +1,638 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_HUFFMAN_TEMPLATE_HPP +#define MGARD_X_160_HUFFMAN_TEMPLATE_HPP + +static bool debug_print_huffman = false; + +#include "../LosslessCompressorInterface.hpp" + +#include "../../RuntimeX/Utilities/Serializer.hpp" +#include "Condense.hpp" +#include "Decode.hpp" +#include "Deflate.hpp" +#include "DictionaryShift.hpp" +#include "EncodeFixedLen.hpp" +#include "GetCodebook.hpp" +#include "Histogram.hpp" +#include "HuffmanWorkspace.hpp" +#include "OutlierSeparator.hpp" +#include "ParallelDeflate.hpp" + +#include +using namespace std::chrono; + +namespace mgard_x_160 { + +template +class Huffman { +public: + Huffman() : initialized(false) {} + + Huffman(SIZE max_size, int dict_size, int chunk_size, + double estimated_outlier_ratio) + : initialized(true), max_size(max_size), dict_size(dict_size), + chunk_size(chunk_size) { + workspace = HuffmanWorkspace( + max_size, dict_size, chunk_size, estimated_outlier_ratio); + } + + void Resize(SIZE max_size, int dict_size, int chunk_size, + double estimated_outlier_ratio, int queue_idx) { + this->initialized = true; + this->max_size = max_size; + this->dict_size = dict_size; + this->chunk_size = chunk_size; + workspace.resize(max_size, dict_size, chunk_size, estimated_outlier_ratio, + queue_idx); + } + + static size_t EstimateMemoryFootprint(SIZE primary_count, SIZE dict_size, + SIZE chunk_size, + double estimated_outlier_ratio = 1) { + return HuffmanWorkspace::EstimateMemoryFootprint( + primary_count, dict_size, chunk_size, estimated_outlier_ratio); + } + + double EstimateCR(Array<1, Q, DeviceType> &primary_data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + SubArray primary_subarray(primary_data); + workspace.reset(queue_idx); + + primary_count = primary_subarray.shape(0); + + Histogram(primary_subarray, + workspace.freq_subarray, + primary_count, dict_size, queue_idx); + auto type_bw = sizeof(H) * 8; + + SubArray<1, H, DeviceType> _d_first_subarray( + {(SIZE)type_bw}, (H *)workspace.decodebook_subarray((IDX)0)); + SubArray<1, H, DeviceType> _d_entry_subarray( + {(SIZE)type_bw}, + (H *)workspace.decodebook_subarray(sizeof(H) * type_bw)); + SubArray<1, Q, DeviceType> _d_qcode_subarray( + {(SIZE)dict_size}, + (Q *)workspace.decodebook_subarray(sizeof(H) * 2 * type_bw)); + + // Sort Qcodes by frequency + DeviceLauncher::Execute( + FillArraySequenceKernel(_d_qcode_subarray), queue_idx); + + MemoryManager::Copy1D(workspace._d_freq_copy_subarray.data(), + workspace.freq_subarray.data(), dict_size, + queue_idx); + MemoryManager::Copy1D(workspace._d_qcode_copy_subarray.data(), + _d_qcode_subarray.data(), dict_size, + queue_idx); + DeviceCollective::SortByKey( + (SIZE)dict_size, workspace._d_freq_copy_subarray, + workspace._d_qcode_copy_subarray, workspace.freq_subarray, + _d_qcode_subarray, workspace.sort_by_key_workspace, true, queue_idx); + + DeviceLauncher::Execute( + GetFirstNonzeroIndexKernel( + workspace.freq_subarray, workspace.first_nonzero_index_subarray), + queue_idx); + + unsigned int first_nonzero_index; + MemoryManager().Copy1D( + &first_nonzero_index, workspace.first_nonzero_index_subarray(IDX(0)), 1, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + int nz_dict_size = dict_size - first_nonzero_index; + + SubArray<1, unsigned int, DeviceType> _nz_d_freq_subarray( + {(SIZE)nz_dict_size}, workspace.freq_subarray(first_nonzero_index)); + SubArray<1, H, DeviceType> _nz_d_codebook_subarray( + {(SIZE)nz_dict_size}, workspace.codebook_subarray(first_nonzero_index)); + + DeviceLauncher::Execute( + GenerateCLKernel( + _nz_d_freq_subarray, workspace.CL_subarray, nz_dict_size, + _nz_d_freq_subarray, workspace.lNodesLeader_subarray, + workspace.iNodesFreq_subarray, workspace.iNodesLeader_subarray, + workspace.tempFreq_subarray, workspace.tempIsLeaf_subarray, + workspace.tempIndex_subarray, workspace.copyFreq_subarray, + workspace.copyIsLeaf_subarray, workspace.copyIndex_subarray, + workspace.diagonal_path_intersections_subarray, + workspace.status_subarray), + queue_idx); + + unsigned int *_freq = new unsigned int[dict_size]; + unsigned int *_cl = new unsigned int[dict_size]; + MemoryManager::Copy1D(_freq, workspace.freq_subarray.data(), + dict_size, queue_idx); + MemoryManager::Copy1D(_cl, workspace.CL_subarray.data(), + dict_size, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + double LC = 0; + for (SIZE i = 0; i < dict_size; i++) { + LC += (double)_freq[i] * _cl[i]; + } + delete[] _freq; + delete[] _cl; + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Huffman estimate CR", primary_count * sizeof(Q)); + timer.clear(); + } + + double CR = (double)(sizeof(Q) * primary_count) / (LC / 8 + 2000); + return CR; + } + + bool CompressPrimary(Array<1, Q, DeviceType> &primary_data, + Array<1, Byte, DeviceType> &compressed_data, + float target_cr, int queue_idx) { + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SubArray primary_subarray(primary_data); + workspace.reset(queue_idx); + + primary_count = primary_subarray.shape(0); + + // Per-stage timing scaffolding (disabled; uncomment the block below and the + // mark() calls to profile each stage under log::TIME). Each mark() syncs + // the queue, prints the elapsed time of the stage just finished against the + // primary input size, and restarts the stage timer. Note: this serializes + // the pipeline (one sync per stage). + // Timer timer_stage; + // auto mark = [&](const char *name) { + // if (log::level & log::TIME) { + // DeviceRuntime::SyncQueue(queue_idx); + // timer_stage.end(); + // timer_stage.print(name, primary_count * sizeof(Q)); + // timer_stage.clear(); + // timer_stage.start(); + // } + // }; + // if (log::level & log::TIME) { + // DeviceRuntime::SyncQueue(queue_idx); + // timer_stage.start(); + // } + + Histogram(primary_subarray, + workspace.freq_subarray, + primary_count, dict_size, queue_idx); + // mark("Huffman stage: histogram"); + + if (debug_print_huffman) { + PrintSubarray("Histogram::freq_subarray", workspace.freq_subarray); + } + + GetCodebook(dict_size, primary_count, workspace.freq_subarray, + workspace.codebook_subarray, workspace.decodebook_subarray, + workspace, queue_idx); + // mark("Huffman stage: codebook"); + + if (target_cr > 1.0) { + // Encoded bits = sum over symbols of frequency x codeword length. After + // GetCodebook, freq_subarray holds the frequencies sorted ascending and + // CL_subarray has been reversed by GenerateCW, so they cannot be paired + // by index. Use the unsorted histogram (_d_freq_copy_subarray) and the + // symbol-indexed codebook, whose top byte is the codeword length (see + // deflate_bitwidth). + std::vector freq(dict_size); + std::vector codebook(dict_size); + MemoryManager::Copy1D(freq.data(), + workspace._d_freq_copy_subarray.data(), + dict_size, queue_idx); + MemoryManager::Copy1D(codebook.data(), + workspace.codebook_subarray.data(), + dict_size, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + double LC = 0; + for (SIZE i = 0; i < dict_size; i++) { + LC += (double)freq[i] * (double)(codebook[i] >> (sizeof(H) * 8 - 8)); + } + double estimated_cr = + (double)(sizeof(Q) * primary_count) / (LC / 8 + 2000); + log::info("Huffman estimated CR: " + std::to_string(estimated_cr) + + " (target: " + std::to_string(target_cr) + ")"); + if (estimated_cr < target_cr) { + return false; + } + } + + if (debug_print_huffman) { + PrintSubarray("GetCodebook::codebook_subarray", + workspace.codebook_subarray); + PrintSubarray("GetCodebook::decodebook_subarray", + workspace.decodebook_subarray); + } + // Encoding is fused into the deflate kernels below: instead of first + // materializing huff[i] = codebook[data[i]] into a primary_count-sized + // array and reading it back twice, GroupBits and Pack look up + // codebook[data[i]] on the fly. This removes a full pass and the huff_array + // allocation. + + // Parallel deflate sizing. The actual bit-packing into the final buffer is + // deferred to Serialize (once the output layout is known) so that each + // chunk can be written directly to its final offset, eliminating a separate + // condense/gather pass. Here we only compute the output geometry: + // per-group bit sums -> scan -> per-chunk bit lengths + word counts + // -> scan -> per-chunk word offsets (and the total ddata word count). + SIZE groups_per_chunk = + (chunk_size - 1) / GetDeflateGroupSize(chunk_size) + 1; + auto nchunk = (primary_count - 1) / chunk_size + 1; + SIZE ngroups = (SIZE)(nchunk * groups_per_chunk); + + DeviceLauncher::Execute( + DeflateGroupBitsKernel( + primary_subarray, workspace.codebook_subarray, + workspace.deflate_group_bits_subarray, primary_count, chunk_size, + groups_per_chunk, ngroups), + queue_idx); + // mark("Huffman stage: deflate group-bits"); + DeviceCollective::ScanSumExtended( + ngroups, workspace.deflate_group_bits_subarray, + workspace.deflate_group_offsets_subarray, + workspace.deflate_group_scan_workspace, true, queue_idx); + DeviceLauncher::Execute( + DeflateChunkMetaKernel( + workspace.deflate_group_offsets_subarray, + workspace.huff_bitwidths_subarray, + workspace.deflate_chunk_words_subarray, (SIZE)nchunk, + groups_per_chunk), + queue_idx); + DeviceCollective::ScanSumExtended( + (SIZE)nchunk, workspace.deflate_chunk_words_subarray, + workspace.deflate_chunk_word_offsets_subarray, + workspace.deflate_chunk_scan_workspace, true, queue_idx); + // Total number of H-words in the densely packed stream. + MemoryManager::Copy1D( + &ddata_size, + workspace.deflate_chunk_word_offsets_subarray.data() + nchunk, 1, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + // mark("Huffman stage: deflate sizing (scans+meta)"); + + // Pack the densely-coded bitstream directly into its final location in the + // output buffer (zero-copy): the byte offset where the packed stream lands + // is fully determined by the sizes of the preceding sections, all known + // here, so we can size the output now and have the pack kernel write + // straight to it. Serialize then only fills in the surrounding metadata. + // Each group writes MSB-first to its final intra-chunk offset; words fully + // owned by a group use plain stores while the (<=2) words shared with + // neighbouring groups use atomicOr, so the destination must be zeroed + // first. + SIZE packed_byte_offset; + SIZE compressed_size = ComputeSerializedLayout(packed_byte_offset); + compressed_data.resize({compressed_size}, queue_idx); + SubArray<1, Byte, DeviceType> compressed_data_subarray(compressed_data); + SubArray<1, H, DeviceType> packed_subarray( + {(SIZE)ddata_size}, (H *)compressed_data_subarray(packed_byte_offset)); + MemoryManager::Memset1D(packed_subarray.data(), ddata_size, 0, + queue_idx); + DeviceLauncher::Execute( + DeflatePackKernel( + primary_subarray, workspace.codebook_subarray, + workspace.deflate_group_offsets_subarray, + workspace.deflate_chunk_word_offsets_subarray, packed_subarray, + primary_count, chunk_size, groups_per_chunk, ngroups), + queue_idx); + // mark("Huffman stage: deflate pack"); + + // Serialize(compressed_data, queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Huffman compress", primary_count * sizeof(Q)); + timer.clear(); + } + return true; + } + + // Walks the serialized layout, returning the total compressed size and, via + // packed_byte_offset, the byte offset at which the densely-packed Huffman + // stream lands. Must stay in lockstep with the section order written by + // Serialize. Depends only on values known after CompressPrimary's sizing + // pass (primary_count, dict_size, chunk_size, ddata_size, outlier_count). + SIZE ComputeSerializedLayout(SIZE &packed_byte_offset) { + auto nchunk = (primary_count - 1) / chunk_size + 1; + size_t type_bw = sizeof(H) * 8; + size_t huffmeta_size = 2 * nchunk; + SIZE byte_offset = 0; + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, huffmeta_size); + advance_with_align(byte_offset, 1); + advance_with_align( + byte_offset, (sizeof(H) * (2 * type_bw) + sizeof(Q) * dict_size)); + advance_with_align(byte_offset, 1); + align_byte_offset(byte_offset); + packed_byte_offset = byte_offset; + advance_with_align(byte_offset, ddata_size); + // outlier + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, outlier_count); + advance_with_align(byte_offset, outlier_count); + return byte_offset; + } + + void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + auto nchunk = (primary_count - 1) / chunk_size + 1; + + // MGARD 1.6.0 layout, which has no leading signature. + // out_meta: |primary count|dict size|chunk size|huffmeta size| + // huffmeta|decodebook size|decodebook|ddata size| + // out_data: |huffman data| + // huffmeta = |per-chunk bit lengths|per-chunk word offsets|, both already + // computed on the device during CompressPrimary. ddata_size (total packed + // word count) was also computed there. + + size_t type_bw = sizeof(H) * 8; + size_t decodebook_size = workspace.decodebook_subarray.shape(0); + size_t huffmeta_size = 2 * nchunk; + + // The output buffer was already sized and the packed Huffman stream already + // written into it by CompressPrimary (zero-copy). Here we only fill in the + // metadata sections around it; the packed region is skipped, not rewritten. + SubArray compressed_data_subarray(compressed_data); + + SIZE byte_offset = 0; + SerializeArray(compressed_data_subarray, &primary_count, 1, + byte_offset, queue_idx); + SerializeArray(compressed_data_subarray, &dict_size, 1, byte_offset, + queue_idx); + SerializeArray(compressed_data_subarray, &chunk_size, 1, byte_offset, + queue_idx); + SerializeArray(compressed_data_subarray, &huffmeta_size, 1, + byte_offset, queue_idx); + // huffmeta first half: per-chunk bit lengths (device). + SerializeArray(compressed_data_subarray, + workspace.huff_bitwidths_subarray.data(), nchunk, + byte_offset, queue_idx); + // huffmeta second half: per-chunk word offsets (device). The extended scan + // holds nchunk+1 entries; the first nchunk are the chunk start offsets. + SerializeArray(compressed_data_subarray, + workspace.deflate_chunk_word_offsets_subarray.data(), + nchunk, byte_offset, queue_idx); + SerializeArray(compressed_data_subarray, &decodebook_size, 1, + byte_offset, queue_idx); + SerializeArray(compressed_data_subarray, + workspace.decodebook_subarray.data(), + (sizeof(H) * (2 * type_bw) + sizeof(Q) * dict_size), + byte_offset, queue_idx); + SerializeArray(compressed_data_subarray, &ddata_size, 1, + byte_offset, queue_idx); + + // The densely-packed Huffman stream is already in place (written directly + // by CompressPrimary); just advance past it. Must mirror + // ComputeSerializedLayout. + advance_with_align(byte_offset, ddata_size); + + // outlier + SerializeArray(compressed_data_subarray, &outlier_count, 1, + byte_offset, queue_idx); + SerializeArray(compressed_data_subarray, + workspace.outlier_idx_subarray.data(), + outlier_count, byte_offset, queue_idx); + SerializeArray(compressed_data_subarray, + workspace.outlier_subarray.data(), outlier_count, + byte_offset, queue_idx); + + DeviceRuntime::SyncQueue(queue_idx); + + log::dbg("Huffman block size: " + std::to_string(chunk_size)); + log::dbg("Huffman dictionary size: " + std::to_string(dict_size)); + log::dbg("Huffman compress ratio (primary): " + + std::to_string(primary_count * sizeof(Q)) + "/" + + std::to_string(ddata_size * sizeof(H)) + " (" + + std::to_string((double)primary_count * sizeof(Q) / + (ddata_size * sizeof(H))) + + ")"); + log::info( + "Huffman compress ratio: " + std::to_string(primary_count * sizeof(Q)) + + "/" + std::to_string(compressed_data.shape(0)) + " (" + + std::to_string((double)primary_count * sizeof(Q) / + compressed_data.shape(0)) + + ")"); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Huffman serialize", compressed_data.shape(0)); + timer.clear(); + } + } + + void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + SubArray compressed_subarray(compressed_data); + + size_t *primary_count_ptr = &primary_count; + int *dict_size_ptr = &dict_size; + int *chunk_size_ptr = &chunk_size; + size_t *huffmeta_size_ptr = &huffmeta_size; + size_t *decodebook_size_ptr = &decodebook_size; + size_t *ddata_size_ptr = &ddata_size; + ATOMIC_IDX *outlier_count_ptr = &outlier_count; + + SIZE byte_offset = 0; + DeserializeArray(compressed_subarray, primary_count_ptr, 1, + byte_offset, false, queue_idx); + DeserializeArray(compressed_subarray, dict_size_ptr, 1, byte_offset, + false, queue_idx); + DeserializeArray(compressed_subarray, chunk_size_ptr, 1, byte_offset, + false, queue_idx); + DeserializeArray(compressed_subarray, huffmeta_size_ptr, 1, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + DeserializeArray(compressed_subarray, huffmeta, huffmeta_size, + byte_offset, true, queue_idx); + DeserializeArray(compressed_subarray, decodebook_size_ptr, 1, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + DeserializeArray(compressed_subarray, decodebook, decodebook_size, + byte_offset, true, queue_idx); + DeserializeArray(compressed_subarray, ddata_size_ptr, 1, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + DeserializeArray(compressed_subarray, ddata, ddata_size, byte_offset, + true, queue_idx); + + // outlier + DeserializeArray(compressed_subarray, outlier_count_ptr, 1, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + DeserializeArray(compressed_subarray, outlier_idx, + outlier_count, byte_offset, true, queue_idx); + DeserializeArray(compressed_subarray, outlier, outlier_count, + byte_offset, true, queue_idx); + workspace.outlier_idx_subarray = + SubArray<1, ATOMIC_IDX, DeviceType>({(SIZE)outlier_count}, outlier_idx); + workspace.outlier_subarray = + SubArray<1, S, DeviceType>({(SIZE)outlier_count}, outlier); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Huffman deserialize", compressed_data.shape(0)); + timer.clear(); + } + } + + void DecompressPrimary(Array<1, Byte, DeviceType> &compressed_data, + Array<1, Q, DeviceType> &primary_data, int queue_idx) { + + // Deserialize(compressed_data, queue_idx); + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SubArray<1, H, DeviceType> ddata_subarray({(SIZE)ddata_size}, ddata); + SubArray<1, size_t, DeviceType> huffmeta_subarray({(SIZE)huffmeta_size}, + huffmeta); + primary_data.resize({(SIZE)primary_count}); + SubArray primary_subarray(primary_data); + + SubArray<1, uint8_t, DeviceType> decodebook_subarray( + {(SIZE)decodebook_size}, decodebook); + + int nchunk = (primary_count - 1) / chunk_size + 1; + Decode( + ddata_subarray, huffmeta_subarray, primary_subarray, primary_count, + chunk_size, nchunk, decodebook_subarray, decodebook_size, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Huffman decompress", primary_count * sizeof(Q)); + timer.clear(); + } + } + + bool Compress(Array<1, S, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, float target_cr, + int queue_idx) { + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + ATOMIC_IDX zero = 0; + MemoryManager::Copy1D(workspace.outlier_count_subarray.data(), + &zero, 1, queue_idx); + + // The dictionary shift (+dict_size/2) is folded into the quantizer, so the + // incoming data already lives in the non-negative dictionary range. A + // single block-aggregated pass separates the out-of-range outliers. + DeviceLauncher::Execute( + OutlierSeparatorKernel( + SubArray(original_data), dict_size, + workspace.outlier_count_subarray, workspace.outlier_idx_subarray, + workspace.outlier_subarray), + queue_idx); + MemoryManager::Copy1D( + &outlier_count, workspace.outlier_count_subarray.data(), 1, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + log::info( + "Outlier ratio: " + std::to_string(outlier_count) + "/" + + std::to_string(original_data.shape(0)) + " (" + + std::to_string((double)100 * outlier_count / original_data.shape(0)) + + "%)"); + if (outlier_count > workspace.outlier_subarray.shape(0)) { + throw std::runtime_error("Not enough workspace for outliers."); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Huffman outlier", original_data.shape(0) * sizeof(S)); + timer.clear(); + } + + // Cast to unsigned type + Array<1, Q, DeviceType> primary_data({original_data.shape(0)}, + (Q *)original_data.data()); + return CompressPrimary(primary_data, compressed_data, target_cr, queue_idx); + } + + void Decompress(Array<1, Byte, DeviceType> &compressed_data, + Array<1, S, DeviceType> &decompressed_data, int queue_idx) { + + // Cast to unsigned type. + // We must use the correct size to avoid resize to new allocation + Array<1, Q, DeviceType> primary_data({decompressed_data.shape(0)}, + (Q *)decompressed_data.data()); + DecompressPrimary(compressed_data, primary_data, queue_idx); + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + // Restore the outliers; the dictionary shift is undone later in the + // dequantizer (mirroring the quantizer), so no shift pass is needed here. + DeviceLauncher::Execute( + OutlierSeparatorKernel( + decompressed_data, dict_size, workspace.outlier_count_subarray, + workspace.outlier_idx_subarray, workspace.outlier_subarray), + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Huffman outlier", decompressed_data.shape(0) * sizeof(S)); + timer.clear(); + } + } + + bool initialized; + SIZE max_size; + size_t primary_count; + ATOMIC_IDX outlier_count = 0; + int dict_size; + int chunk_size; + size_t huffmeta_size; + size_t decodebook_size; + size_t ddata_size; + size_t *huffmeta; + uint8_t *decodebook; + ATOMIC_IDX *outlier_idx; + S *outlier; + H *ddata; + HuffmanWorkspace workspace; +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/HuffmanWorkspace.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/HuffmanWorkspace.hpp new file mode 100644 index 000000000..2c8f6e0f1 --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/HuffmanWorkspace.hpp @@ -0,0 +1,379 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_HUFFMAN_WORKSPACE_HPP +#define MGARD_X_160_HUFFMAN_WORKSPACE_HPP + +#include "../../Hierarchy/Hierarchy.h" +#include "../../RuntimeX/RuntimeXPublic.h" +#include "GenerateCL.hpp" +#include "ParallelDeflate.hpp" + +namespace mgard_x_160 { + +template +class HuffmanWorkspace { +public: + HuffmanWorkspace() { + // By defualt it is not pre-allocated + pre_allocated = false; + } + + void initialize_subarray() { + outlier_count_subarray = SubArray(outlier_count_array); + outlier_idx_subarray = SubArray(outlier_idx_array); + outlier_subarray = SubArray(outlier_array); + + freq_subarray = SubArray(freq_array); + codebook_subarray = SubArray(codebook_array); + decodebook_subarray = SubArray(decodebook_array); + huff_bitwidths_subarray = SubArray(huff_bitwidths_array); + condense_write_offsets_subarray = SubArray(condense_write_offsets_array); + condense_actual_lengths_subarray = SubArray(condense_actual_lengths_array); + + deflate_group_bits_subarray = SubArray(deflate_group_bits_array); + deflate_group_offsets_subarray = SubArray(deflate_group_offsets_array); + deflate_chunk_words_subarray = SubArray(deflate_chunk_words_array); + deflate_chunk_word_offsets_subarray = + SubArray(deflate_chunk_word_offsets_array); + + // Codebook + first_nonzero_index_subarray = SubArray(first_nonzero_index_array); + sort_by_key_workspace_subarray = SubArray(sort_by_key_workspace); + _d_freq_copy_subarray = SubArray(_d_freq_copy_array); + _d_qcode_copy_subarray = SubArray(_d_qcode_copy_array); + CL_subarray = SubArray(CL_array); + lNodesLeader_subarray = SubArray(lNodesLeader_array); + iNodesFreq_subarray = SubArray(iNodesFreq_array); + iNodesLeader_subarray = SubArray(iNodesLeader_array); + tempFreq_subarray = SubArray(tempFreq_array); + tempIsLeaf_subarray = SubArray(tempIsLeaf_array); + tempIndex_subarray = SubArray(tempIndex_array); + copyFreq_subarray = SubArray(copyFreq_array); + copyIsLeaf_subarray = SubArray(copyIsLeaf_array); + copyIndex_subarray = SubArray(copyIndex_array); + _d_codebook_subarray_org = SubArray(_d_codebook_array_org); + status_subarray = SubArray(status_array); + diagonal_path_intersections_subarray = + SubArray(diagonal_path_intersections_array); + } + + static size_t EstimateMemoryFootprint(SIZE primary_count, SIZE dict_size, + SIZE chunk_size, + double estimated_outlier_ratio = 1) { + size_t size = 0; + size += sizeof(ATOMIC_IDX); + size += primary_count * estimated_outlier_ratio * sizeof(ATOMIC_IDX); + size += primary_count * estimated_outlier_ratio * sizeof(S); + + size += dict_size * sizeof(unsigned int); + size += dict_size * sizeof(H); + size_t type_bw = sizeof(H) * 8; + size_t decodebook_size = sizeof(H) * (2 * type_bw) + sizeof(Q) * dict_size; + size += decodebook_size * sizeof(uint8_t); + size_t nchunk = (primary_count - 1) / chunk_size + 1; + size += nchunk * sizeof(size_t); + size += nchunk * sizeof(size_t); + size += nchunk * sizeof(size_t); + + // Parallel deflate + SIZE groups_per_chunk = + (chunk_size - 1) / GetDeflateGroupSize(chunk_size) + 1; + size_t ngroups = nchunk * groups_per_chunk; + size += ngroups * sizeof(size_t); // group_bits + size += (ngroups + 1) * sizeof(size_t); // group_offsets + size += nchunk * sizeof(size_t); // chunk_words + size += (nchunk + 1) * sizeof(size_t); // chunk_word_offsets + { + Array<1, Byte, DeviceType> tmp_group_scan, tmp_chunk_scan; + DeviceCollective::ScanSumExtended( + (SIZE)ngroups, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), tmp_group_scan, false, 0); + DeviceCollective::ScanSumExtended( + (SIZE)nchunk, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), tmp_chunk_scan, false, 0); + size += tmp_group_scan.shape(0); + size += tmp_chunk_scan.shape(0); + } + + size += sizeof(unsigned int); + Array<1, Byte, DeviceType> tmp; + DeviceCollective::SortByKey( + dict_size, SubArray<1, unsigned int, DeviceType>(), + SubArray<1, Q, DeviceType>(), SubArray<1, unsigned int, DeviceType>(), + SubArray<1, Q, DeviceType>(), tmp, false, 0); + size += tmp.shape(0); + size += dict_size * sizeof(unsigned int); + size += dict_size * sizeof(Q); + size += sizeof(unsigned int) * dict_size * 4; + size += sizeof(int) * dict_size * 6; + size += sizeof(H) * dict_size; + size += sizeof(int) * 16; + size += + GenerateCLKernel::DiagonalPathIntersectionsSize(dict_size) * + sizeof(uint32_t); + size += 7 * sizeof(Byte); // signature_verify_array + return size; + } + + void allocate(SIZE primary_count, SIZE dict_size, SIZE chunk_size, + double estimated_outlier_ratio) { + + outlier_count_array = Array<1, ATOMIC_IDX, DeviceType>({1}); + outlier_idx_array = Array<1, ATOMIC_IDX, DeviceType>( + {(SIZE)(primary_count * estimated_outlier_ratio)}); + outlier_array = Array<1, S, DeviceType>( + {(SIZE)(primary_count * estimated_outlier_ratio)}); + + freq_array = Array<1, unsigned int, DeviceType>({dict_size}); + freq_array.hostAllocate(false); + codebook_array = Array<1, H, DeviceType>({dict_size}); + size_t type_bw = sizeof(H) * 8; + size_t decodebook_size = sizeof(H) * (2 * type_bw) + sizeof(Q) * dict_size; + decodebook_array = Array<1, uint8_t, DeviceType>({(SIZE)decodebook_size}); + size_t nchunk = (primary_count - 1) / chunk_size + 1; + huff_bitwidths_array = Array<1, size_t, DeviceType>({(SIZE)nchunk}); + condense_write_offsets_array = Array<1, size_t, DeviceType>({(SIZE)nchunk}); + condense_actual_lengths_array = + Array<1, size_t, DeviceType>({(SIZE)nchunk}); + + // Parallel deflate + SIZE groups_per_chunk = + (chunk_size - 1) / GetDeflateGroupSize(chunk_size) + 1; + SIZE ngroups = (SIZE)(nchunk * groups_per_chunk); + deflate_group_bits_array = Array<1, size_t, DeviceType>({ngroups}); + deflate_group_offsets_array = Array<1, size_t, DeviceType>({ngroups + 1}); + deflate_chunk_words_array = Array<1, size_t, DeviceType>({(SIZE)nchunk}); + deflate_chunk_word_offsets_array = + Array<1, size_t, DeviceType>({(SIZE)nchunk + 1}); + DeviceCollective::ScanSumExtended( + ngroups, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), deflate_group_scan_workspace, false, + 0); + DeviceCollective::ScanSumExtended( + (SIZE)nchunk, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), deflate_chunk_scan_workspace, false, + 0); + // Codebook + first_nonzero_index_array = Array<1, unsigned int, DeviceType>({1}); + // first_nonzero_index_array.hostCopy(); // Create host allocation + // Allocate workspace + DeviceCollective::SortByKey( + dict_size, SubArray<1, unsigned int, DeviceType>(), + SubArray<1, Q, DeviceType>(), SubArray<1, unsigned int, DeviceType>(), + SubArray<1, Q, DeviceType>(), sort_by_key_workspace, false, 0); + _d_freq_copy_array = Array<1, unsigned int, DeviceType>({(SIZE)dict_size}); + _d_qcode_copy_array = Array<1, Q, DeviceType>({(SIZE)dict_size}); + CL_array = Array<1, unsigned int, DeviceType>({dict_size}); + CL_array.hostAllocate(false); + lNodesLeader_array = Array<1, int, DeviceType>({dict_size}); + iNodesFreq_array = Array<1, unsigned int, DeviceType>({dict_size}); + iNodesLeader_array = Array<1, int, DeviceType>({dict_size}); + tempFreq_array = Array<1, unsigned int, DeviceType>({dict_size}); + tempIsLeaf_array = Array<1, int, DeviceType>({dict_size}); + tempIndex_array = Array<1, int, DeviceType>({dict_size}); + copyFreq_array = Array<1, unsigned int, DeviceType>({dict_size}); + copyIsLeaf_array = Array<1, int, DeviceType>({dict_size}); + copyIndex_array = Array<1, int, DeviceType>({dict_size}); + _d_codebook_array_org = Array<1, H, DeviceType>({dict_size}); + status_array = Array<1, int, DeviceType, false, false>( + {(SIZE)16}); // non-managed: atomicMin on managed mem unreliable on ROCm + diagonal_path_intersections_array = Array<1, uint32_t, DeviceType>( + {GenerateCLKernel:: + DiagonalPathIntersectionsSize(dict_size)}); + + signature_verify_array = Array<1, Byte, DeviceType>({(SIZE)7}); + signature_verify_array.hostAllocate(false); + + // outlier_count_array.memset(0); + // outlier_idx_array.memset(0); + // outlier_array.memset(0); + + initialize_subarray(); + + pre_allocated = true; + } + + void resize(SIZE primary_count, SIZE dict_size, SIZE chunk_size, + double estimated_outlier_ratio, int queue_idx) { + + outlier_count_array.resize({1}, queue_idx); + outlier_idx_array.resize({(SIZE)(primary_count * estimated_outlier_ratio)}, + queue_idx); + outlier_array.resize({(SIZE)(primary_count * estimated_outlier_ratio)}, + queue_idx); + + freq_array.resize({dict_size}, queue_idx); + freq_array.hostAllocate(false); + codebook_array.resize({dict_size}, queue_idx); + size_t type_bw = sizeof(H) * 8; + size_t decodebook_size = sizeof(H) * (2 * type_bw) + sizeof(Q) * dict_size; + decodebook_array.resize({(SIZE)decodebook_size}, queue_idx); + size_t nchunk = (primary_count - 1) / chunk_size + 1; + huff_bitwidths_array.resize({(SIZE)nchunk}, queue_idx); + condense_write_offsets_array.resize({(SIZE)nchunk}, queue_idx); + condense_actual_lengths_array.resize({(SIZE)nchunk}, queue_idx); + + // Parallel deflate + SIZE groups_per_chunk = + (chunk_size - 1) / GetDeflateGroupSize(chunk_size) + 1; + SIZE ngroups = (SIZE)(nchunk * groups_per_chunk); + deflate_group_bits_array.resize({ngroups}, queue_idx); + deflate_group_offsets_array.resize({ngroups + 1}, queue_idx); + deflate_chunk_words_array.resize({(SIZE)nchunk}, queue_idx); + deflate_chunk_word_offsets_array.resize({(SIZE)nchunk + 1}, queue_idx); + DeviceCollective::ScanSumExtended( + ngroups, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), deflate_group_scan_workspace, false, + queue_idx); + DeviceCollective::ScanSumExtended( + (SIZE)nchunk, SubArray<1, size_t, DeviceType>(), + SubArray<1, size_t, DeviceType>(), deflate_chunk_scan_workspace, false, + queue_idx); + // Codebook + first_nonzero_index_array.resize({1}, queue_idx); + // first_nonzero_index_array.hostCopy(); // Create host allocation + // // Allocate workspace + DeviceCollective::SortByKey( + dict_size, SubArray<1, unsigned int, DeviceType>(), + SubArray<1, Q, DeviceType>(), SubArray<1, unsigned int, DeviceType>(), + SubArray<1, Q, DeviceType>(), sort_by_key_workspace, false, queue_idx); + + _d_freq_copy_array.resize({(SIZE)dict_size}, queue_idx); + _d_qcode_copy_array.resize({(SIZE)dict_size}, queue_idx); + CL_array.resize({dict_size}, queue_idx); + CL_array.hostAllocate(false); + lNodesLeader_array.resize({dict_size}, queue_idx); + iNodesFreq_array.resize({dict_size}, queue_idx); + iNodesLeader_array.resize({dict_size}, queue_idx); + tempFreq_array.resize({dict_size}, queue_idx); + tempIsLeaf_array.resize({dict_size}, queue_idx); + tempIndex_array.resize({dict_size}, queue_idx); + copyFreq_array.resize({dict_size}, queue_idx); + copyIsLeaf_array.resize({dict_size}, queue_idx); + copyIndex_array.resize({dict_size}, queue_idx); + _d_codebook_array_org.resize({dict_size}, queue_idx); + status_array.resize({(SIZE)16}, queue_idx); + diagonal_path_intersections_array.resize( + {GenerateCLKernel:: + DiagonalPathIntersectionsSize(dict_size)}, + queue_idx); + + signature_verify_array.resize({(SIZE)7}, queue_idx); + signature_verify_array.hostAllocate(false, queue_idx); + + // outlier_count_array.memset(0, queue_idx); + // outlier_idx_array.memset(0, queue_idx); + // outlier_array.memset(0, queue_idx); + initialize_subarray(); + pre_allocated = true; + } + + void reset(int queue_idx) { + outlier_count_array.memset(0, queue_idx); + freq_array.memset(0, queue_idx); + codebook_array.memset(0, queue_idx); + decodebook_array.memset(0xff, queue_idx); + // huff_array.memset(0, queue_idx); + huff_bitwidths_array.memset(0, queue_idx); + first_nonzero_index_array.memset(0xff, queue_idx); + CL_array.memset(0, queue_idx); + } + + HuffmanWorkspace(SIZE primary_count, SIZE dict_size, SIZE chunk_size, + double estimated_outlier_ratio) { + allocate(primary_count, dict_size, chunk_size, estimated_outlier_ratio); + } + + bool pre_allocated; + + Array<1, ATOMIC_IDX, DeviceType> outlier_count_array; + Array<1, ATOMIC_IDX, DeviceType> outlier_idx_array; + Array<1, S, DeviceType> outlier_array; + + Array<1, unsigned int, DeviceType> freq_array; + Array<1, H, DeviceType> codebook_array; + Array<1, uint8_t, DeviceType> decodebook_array; + Array<1, size_t, DeviceType> huff_bitwidths_array; + Array<1, size_t, DeviceType> condense_write_offsets_array; + Array<1, size_t, DeviceType> condense_actual_lengths_array; + + // Parallel deflate (cooperative bit-packing). group_bits holds per-group bit + // sums; group_offsets is their extended exclusive scan. chunk_words holds + // per-chunk H-word counts; chunk_word_offsets is their extended exclusive + // scan (the last element is the total ddata word count). + Array<1, size_t, DeviceType> deflate_group_bits_array; + Array<1, size_t, DeviceType> deflate_group_offsets_array; + Array<1, size_t, DeviceType> deflate_chunk_words_array; + Array<1, size_t, DeviceType> deflate_chunk_word_offsets_array; + Array<1, Byte, DeviceType> deflate_group_scan_workspace; + Array<1, Byte, DeviceType> deflate_chunk_scan_workspace; + + // Codebook + Array<1, unsigned int, DeviceType> first_nonzero_index_array; + Array<1, Byte, DeviceType> sort_by_key_workspace; + Array<1, unsigned int, DeviceType> _d_freq_copy_array; + Array<1, Q, DeviceType> _d_qcode_copy_array; + Array<1, unsigned int, DeviceType> CL_array; + Array<1, int, DeviceType> lNodesLeader_array; + Array<1, unsigned int, DeviceType> iNodesFreq_array; + Array<1, int, DeviceType> iNodesLeader_array; + Array<1, unsigned int, DeviceType> tempFreq_array; + Array<1, int, DeviceType> tempIsLeaf_array; + Array<1, int, DeviceType> tempIndex_array; + Array<1, unsigned int, DeviceType> copyFreq_array; + Array<1, int, DeviceType> copyIsLeaf_array; + Array<1, int, DeviceType> copyIndex_array; + Array<1, H, DeviceType> _d_codebook_array_org; + Array<1, int, DeviceType, false, true> status_array; + Array<1, uint32_t, DeviceType> diagonal_path_intersections_array; + + // Holds the deserialized signature during Verify. Only its pinned host copy + // is used (read back via dataHost()); the device buffer is unused but comes + // along with the Array and is freed automatically with the workspace. + Array<1, Byte, DeviceType> signature_verify_array; + + SubArray<1, ATOMIC_IDX, DeviceType> outlier_count_subarray; + SubArray<1, ATOMIC_IDX, DeviceType> outlier_idx_subarray; + SubArray<1, S, DeviceType> outlier_subarray; + + SubArray<1, unsigned int, DeviceType> freq_subarray; + SubArray<1, H, DeviceType> codebook_subarray; + SubArray<1, uint8_t, DeviceType> decodebook_subarray; + SubArray<1, size_t, DeviceType> huff_bitwidths_subarray; + SubArray<1, size_t, DeviceType> condense_write_offsets_subarray; + SubArray<1, size_t, DeviceType> condense_actual_lengths_subarray; + + SubArray<1, size_t, DeviceType> deflate_group_bits_subarray; + SubArray<1, size_t, DeviceType> deflate_group_offsets_subarray; + SubArray<1, size_t, DeviceType> deflate_chunk_words_subarray; + SubArray<1, size_t, DeviceType> deflate_chunk_word_offsets_subarray; + + // Codebook + SubArray<1, unsigned int, DeviceType> first_nonzero_index_subarray; + SubArray<1, Byte, DeviceType> sort_by_key_workspace_subarray; + SubArray<1, unsigned int, DeviceType> _d_freq_copy_subarray; + SubArray<1, Q, DeviceType> _d_qcode_copy_subarray; + SubArray<1, unsigned int, DeviceType> CL_subarray; + SubArray<1, int, DeviceType> lNodesLeader_subarray; + SubArray<1, unsigned int, DeviceType> iNodesFreq_subarray; + SubArray<1, int, DeviceType> iNodesLeader_subarray; + SubArray<1, unsigned int, DeviceType> tempFreq_subarray; + SubArray<1, int, DeviceType> tempIsLeaf_subarray; + SubArray<1, int, DeviceType> tempIndex_subarray; + SubArray<1, unsigned int, DeviceType> copyFreq_subarray; + SubArray<1, int, DeviceType> copyIsLeaf_subarray; + SubArray<1, int, DeviceType> copyIndex_subarray; + SubArray<1, H, DeviceType> _d_codebook_subarray_org; + SubArray<1, int, DeviceType, false, true> status_subarray; + SubArray<1, uint32_t, DeviceType> diagonal_path_intersections_subarray; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/OutlierSeparator.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/OutlierSeparator.hpp new file mode 100644 index 000000000..3b269155c --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/OutlierSeparator.hpp @@ -0,0 +1,186 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_OUTLIER_SEPARATOR_TEMPLATE_HPP +#define MGARD_X_160_OUTLIER_SEPARATOR_TEMPLATE_HPP +#include "../../RuntimeX/RuntimeX.h" + +#define MGARDX_SEPARATE_OUTLIER 1 +#define MGARDX_RESTORE_OUTLIER 2 + +namespace mgard_x_160 { + +template +class OutlierSeparatorFunctor : public Functor { +public: + MGARDX_CONT OutlierSeparatorFunctor() {} + MGARDX_CONT + OutlierSeparatorFunctor(SubArray<1, T, DeviceType> v, SIZE dict_size, + SubArray<1, ATOMIC_IDX, DeviceType> outlier_count, + SubArray<1, ATOMIC_IDX, DeviceType> outlier_index, + SubArray<1, T, DeviceType> outlier_value) + : v(v), dict_size(dict_size), outlier_count(outlier_count), + outlier_index(outlier_index), outlier_value(outlier_value) { + Functor(); + } + + // SEPARATE uses block-aggregated atomics: every outlier first reserves a slot + // in a block-local counter (shared memory, block scope), then a single thread + // reserves the whole block's contiguous output range with one device-scope + // atomic. This collapses up to blockDim global atomics on the single counter + // address into one per block, removing the contention that made this the + // bottleneck. The functor framework inserts a block barrier between + // consecutive Operation*() phases. + MGARDX_EXEC void Operation1() { + id = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + if constexpr (OP == MGARDX_SEPARATE_OUTLIER) { + // sm[0]: block-local outlier count, sm[1]: block base offset + sm = (ATOMIC_IDX *)FunctorBase::GetSharedMemory(); + if (FunctorBase::GetThreadIdX() == 0) { + sm[0] = 0; + } + is_outlier = false; + if (id < v.shape(0)) { + value = *v(id); + if (value < 0 || value >= dict_size) { + is_outlier = true; + } + } + } else if constexpr (OP == MGARDX_RESTORE_OUTLIER) { + // Grid is sized to the outlier count, so this is a sparse scatter. + if (id < outlier_value.shape(0)) { + ATOMIC_IDX index = *outlier_index(id); + T val = *outlier_value(id); + *v(index) = val; + } + } + } + + // Reserve a block-local slot for each outlier. + MGARDX_EXEC void Operation2() { + if constexpr (OP == MGARDX_SEPARATE_OUTLIER) { + if (is_outlier) { + local_offset = Atomic::Add(&sm[0], (ATOMIC_IDX)1); + } + } + } + + // One global atomic per block reserves the block's output range. + MGARDX_EXEC void Operation3() { + if constexpr (OP == MGARDX_SEPARATE_OUTLIER) { + if (FunctorBase::GetThreadIdX() == 0) { + sm[1] = Atomic::Add(outlier_count((IDX)0), sm[0]); + } + } + } + + // Scatter outliers to their global slots and zero them in the primary stream. + MGARDX_EXEC void Operation4() { + if constexpr (OP == MGARDX_SEPARATE_OUTLIER) { + if (is_outlier) { + ATOMIC_IDX outlier_write_index = sm[1] + local_offset; + if (outlier_write_index < outlier_index.shape(0)) { + *outlier_index(outlier_write_index) = id; + *outlier_value(outlier_write_index) = value; + *v(id) = 0; + } + } + } + } + + MGARDX_EXEC void Operation5() {} + + MGARDX_CONT size_t shared_memory_size() { + if constexpr (OP == MGARDX_SEPARATE_OUTLIER) { + return 2 * sizeof(ATOMIC_IDX); + } else { + return 0; + } + } + +private: + SubArray<1, T, DeviceType> v; + SIZE dict_size; + SubArray<1, ATOMIC_IDX, DeviceType> outlier_count; + SubArray<1, ATOMIC_IDX, DeviceType> outlier_index; + SubArray<1, T, DeviceType> outlier_value; + + // Per-thread state carried across the Operation*() phases (SEPARATE only). + SIZE id; + bool is_outlier; + T value; + ATOMIC_IDX local_offset; + ATOMIC_IDX *sm; +}; + +template +class OutlierSeparatorKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "OutlierSeparator"; + MGARDX_CONT + OutlierSeparatorKernel(SubArray<1, T, DeviceType> v, SIZE dict_size, + SubArray<1, ATOMIC_IDX, DeviceType> outlier_count, + SubArray<1, ATOMIC_IDX, DeviceType> outlier_index, + SubArray<1, T, DeviceType> outlier_value) + : v(v), dict_size(dict_size), outlier_count(outlier_count), + outlier_index(outlier_index), outlier_value(outlier_value) {} + + MGARDX_CONT + Task> GenTask(int queue_idx) { + using FunctorType = OutlierSeparatorFunctor; + FunctorType functor(v, dict_size, outlier_count, outlier_index, + outlier_value); + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + if constexpr (std::is_same::value) { + // Empirically tuned on MI300: a direct sweep (64/128/256/512/768/896/ + // 960/1024) found throughput climbing steeply with block size -- + // 256 (the original) measures ~224 GB/s, 960 measures ~815 GB/s, a + // >3.5x improvement with no logic change. 1024 (the hardware max) + // reliably hung the kernel launch on this system, so 960 is used as + // the largest safely-tested value rather than chasing the last ~7% + // theoretical gain from 1024 at the risk of a hang. This tuning is + // HIP/CDNA3-specific and untested on other backends, so it is guarded + // here rather than applied unconditionally. + tbx = 960; + } else { + tbx = 256; + } + gridz = 1; + gridy = 1; + // RESTORE is a sparse scatter over the outliers only (outlier_value is + // sized to the outlier count after deserialization), so launch just enough + // threads instead of one per element of the full array. + SIZE launch_count; + if constexpr (OP == MGARDX_RESTORE_OUTLIER) { + launch_count = outlier_value.shape(0); + } else { + launch_count = v.shape(0); + } + gridx = (launch_count == 0) ? 1 : (launch_count - 1) / tbx + 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T, DeviceType> v; + SIZE dict_size; + SubArray<1, ATOMIC_IDX, DeviceType> outlier_count; + SubArray<1, ATOMIC_IDX, DeviceType> outlier_index; + SubArray<1, T, DeviceType> outlier_value; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/ParallelDeflate.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/ParallelDeflate.hpp new file mode 100644 index 000000000..b02ce14a1 --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/ParallelDeflate.hpp @@ -0,0 +1,412 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_PARALLEL_DEFLATE_TEMPLATE_HPP +#define MGARD_X_160_PARALLEL_DEFLATE_TEMPLATE_HPP +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +// Number of consecutive symbols packed by a single thread ("group"). Each +// Huffman chunk is split into ceil(chunk_size / group_size) groups so that +// the bit-packing of one chunk is shared by many threads instead of a single +// one. Must stay small enough to keep good parallelism but large enough that +// most output words a group produces are fully owned (written with a plain +// store) rather than shared at the boundaries (written with an atomicOr). +// +// Empirically the pack kernel is fastest when a full warp covers about one +// Huffman chunk, i.e. group_size ~= huff_block_size / 32: for the default +// huff_block_size = 1024 that measured ~3.6x faster than a fixed 256 on NYX +// 512^3 on Hopper. That "one warp per chunk" reasoning does not transfer +// directly to a 64-wide CDNA wavefront, though: group_size = chunk_size/64 +// = 16 measured *worse* on MI300 (~73 GB/s) than the original 32 (~74 +// GB/s), and a direct sweep (16/32/64/128/256/512) found the real peak at +// group_size = 128 (~86 GB/s) -- larger, not smaller, than Hopper's tuning. +// Fewer, bigger groups apparently win here because each group boundary +// costs an atomicOr-merged word instead of a plain store (see DeflatePack +// below); halving group_size doubles the boundary count for the same data, +// and on MI300 that atomic-merge overhead outweighs the extra parallelism +// from narrower groups. Revisit with a fresh sweep if huff_block_size +// changes (currently group_size = chunk_size/8, matching the measured +// optimum at the default chunk_size = 1024). +template +MGARDX_CONT SIZE GetDeflateGroupSize(SIZE chunk_size) { + SIZE group_size; + if constexpr (std::is_same::value) { + group_size = chunk_size / 8; + } else { + SIZE warp_size = DeviceRuntime::GetWarpSize(); + group_size = chunk_size / warp_size; + } + return group_size > 0 ? group_size : 1; +} + +// Helper: extract the per-symbol bitwidth, stored in the most-significant byte +// of each fixed-length Huffman codeword (codebook[symbol]). +template MGARDX_EXEC uint8_t deflate_bitwidth(H word) { + return *((uint8_t *)&word + (sizeof(H) - 1)); +} + +// Phase 1 (sizing): each thread sums the bitwidths of the symbols in one group. +// The fixed-length codeword for symbol s is codebook[s], so we look it up on +// the fly instead of reading a materialized per-symbol array (encode is fused +// here). Groups never cross chunk boundaries, so a later per-chunk reduction +// over the scanned group sums yields per-chunk bit lengths. +template +class DeflateGroupBitsFunctor : public Functor { +public: + MGARDX_CONT DeflateGroupBitsFunctor() {} + MGARDX_CONT DeflateGroupBitsFunctor( + SubArray<1, Q, DeviceType> data, SubArray<1, H, DeviceType> codebook, + SubArray<1, size_t, DeviceType> group_bits, size_t primary_count, + SIZE chunk_size, SIZE groups_per_chunk, SIZE ngroups, + SIZE deflate_group_size) + : data(data), codebook(codebook), group_bits(group_bits), + primary_count(primary_count), chunk_size(chunk_size), + groups_per_chunk(groups_per_chunk), ngroups(ngroups), + deflate_group_size(deflate_group_size) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + if (gid >= ngroups) + return; + SIZE chunk_id = gid / groups_per_chunk; + SIZE local = gid % groups_per_chunk; + size_t sym_base = + (size_t)chunk_id * chunk_size + (size_t)local * deflate_group_size; + size_t chunk_end = (size_t)(chunk_id + 1) * chunk_size; + if (chunk_end > primary_count) + chunk_end = primary_count; + size_t sym_end = sym_base + deflate_group_size; + if (sym_end > chunk_end) + sym_end = chunk_end; + size_t bits = 0; + for (size_t i = sym_base; i < sym_end; i++) { + bits += deflate_bitwidth(*codebook(*data(i))); + } + *group_bits(gid) = bits; + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Q, DeviceType> data; + SubArray<1, H, DeviceType> codebook; + SubArray<1, size_t, DeviceType> group_bits; + size_t primary_count; + SIZE chunk_size; + SIZE groups_per_chunk; + SIZE ngroups; + SIZE deflate_group_size; +}; + +template +class DeflateGroupBitsKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "deflate_group_bits"; + MGARDX_CONT + DeflateGroupBitsKernel(SubArray<1, Q, DeviceType> data, + SubArray<1, H, DeviceType> codebook, + SubArray<1, size_t, DeviceType> group_bits, + size_t primary_count, SIZE chunk_size, + SIZE groups_per_chunk, SIZE ngroups) + : data(data), codebook(codebook), group_bits(group_bits), + primary_count(primary_count), chunk_size(chunk_size), + groups_per_chunk(groups_per_chunk), ngroups(ngroups) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = DeflateGroupBitsFunctor; + SIZE deflate_group_size = GetDeflateGroupSize(chunk_size); + FunctorType functor(data, codebook, group_bits, primary_count, chunk_size, + groups_per_chunk, ngroups, deflate_group_size); + SIZE tbx; + if constexpr (std::is_same::value) { + // Empirically tuned on MI300: this kernel has no shared memory or + // cross-thread communication (each thread independently sums bitwidths + // for its own group), so block size is a pure occupancy knob. A direct + // sweep (256/512/768/960) found a non-monotonic curve -- 256 and 512 + // both measure ~113 GB/s (combined with DeflatePack below, via the + // CLI's "Huffman compress" timer), 768 measures ~120-124 GB/s, and 960 + // drops back to ~111 GB/s. 768 is the peak, likely an occupancy sweet + // spot for this kernel's register/LDS footprint rather than a value + // that keeps climbing like OutlierSeparator's. This tuning is + // HIP/CDNA3-specific and untested on other backends, so it is guarded + // here rather than applied unconditionally. Revisit with a fresh + // sweep if this kernel's per-thread work changes. + tbx = 768; + } else { + tbx = 256; + } + size_t sm_size = functor.shared_memory_size(); + SIZE gridx = (ngroups - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, 1, 1, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Q, DeviceType> data; + SubArray<1, H, DeviceType> codebook; + SubArray<1, size_t, DeviceType> group_bits; + size_t primary_count; + SIZE chunk_size; + SIZE groups_per_chunk; + SIZE ngroups; +}; + +// Phase 2 (per-chunk meta): from the extended exclusive scan of group bit sums, +// compute each chunk's total bit length and its output length in H-words. +template +class DeflateChunkMetaFunctor : public Functor { +public: + MGARDX_CONT DeflateChunkMetaFunctor() {} + MGARDX_CONT + DeflateChunkMetaFunctor(SubArray<1, size_t, DeviceType> group_offsets, + SubArray<1, size_t, DeviceType> chunk_bits, + SubArray<1, size_t, DeviceType> chunk_words, + SIZE nchunk, SIZE groups_per_chunk) + : group_offsets(group_offsets), chunk_bits(chunk_bits), + chunk_words(chunk_words), nchunk(nchunk), + groups_per_chunk(groups_per_chunk) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE chunk_id = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + if (chunk_id >= nchunk) + return; + size_t total_bits = *group_offsets((chunk_id + 1) * groups_per_chunk) - + *group_offsets(chunk_id * groups_per_chunk); + *chunk_bits(chunk_id) = total_bits; + *chunk_words(chunk_id) = (total_bits + sizeof(H) * 8 - 1) / (sizeof(H) * 8); + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, size_t, DeviceType> group_offsets; + SubArray<1, size_t, DeviceType> chunk_bits; + SubArray<1, size_t, DeviceType> chunk_words; + SIZE nchunk; + SIZE groups_per_chunk; +}; + +template +class DeflateChunkMetaKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "deflate_chunk_meta"; + MGARDX_CONT + DeflateChunkMetaKernel(SubArray<1, size_t, DeviceType> group_offsets, + SubArray<1, size_t, DeviceType> chunk_bits, + SubArray<1, size_t, DeviceType> chunk_words, + SIZE nchunk, SIZE groups_per_chunk) + : group_offsets(group_offsets), chunk_bits(chunk_bits), + chunk_words(chunk_words), nchunk(nchunk), + groups_per_chunk(groups_per_chunk) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = DeflateChunkMetaFunctor; + FunctorType functor(group_offsets, chunk_bits, chunk_words, nchunk, + groups_per_chunk); + SIZE tbx = 256; + size_t sm_size = functor.shared_memory_size(); + SIZE gridx = (nchunk - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, 1, 1, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, size_t, DeviceType> group_offsets; + SubArray<1, size_t, DeviceType> chunk_bits; + SubArray<1, size_t, DeviceType> chunk_words; + SIZE nchunk; + SIZE groups_per_chunk; +}; + +// Phase 3 (pack): each thread densely packs one group's symbols, MSB-first, +// directly into the final compressed buffer at the chunk's word offset plus the +// group's intra-chunk bit offset. Output words fully covered by the group are +// written with a plain store; the (at most two) words shared with neighbouring +// groups are merged with atomicOr. The destination region must be zeroed first +// so the atomicOr merges only contribute new bits. +template +class DeflatePackFunctor : public Functor { +public: + MGARDX_CONT DeflatePackFunctor() {} + MGARDX_CONT + DeflatePackFunctor(SubArray<1, Q, DeviceType> data, + SubArray<1, H, DeviceType> codebook, + SubArray<1, size_t, DeviceType> group_offsets, + SubArray<1, size_t, DeviceType> chunk_word_offsets, + SubArray<1, H, DeviceType> condensed, size_t primary_count, + SIZE chunk_size, SIZE groups_per_chunk, SIZE ngroups, + SIZE deflate_group_size) + : data(data), codebook(codebook), group_offsets(group_offsets), + chunk_word_offsets(chunk_word_offsets), condensed(condensed), + primary_count(primary_count), chunk_size(chunk_size), + groups_per_chunk(groups_per_chunk), ngroups(ngroups), + deflate_group_size(deflate_group_size) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + SIZE gid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + if (gid >= ngroups) + return; + SIZE chunk_id = gid / groups_per_chunk; + SIZE local = gid % groups_per_chunk; + size_t sym_base = + (size_t)chunk_id * chunk_size + (size_t)local * deflate_group_size; + if (sym_base >= primary_count) + return; + size_t chunk_end = (size_t)(chunk_id + 1) * chunk_size; + if (chunk_end > primary_count) + chunk_end = primary_count; + size_t sym_end = sym_base + deflate_group_size; + if (sym_end > chunk_end) + sym_end = chunk_end; + + const size_t bpw = sizeof(H) * 8; + size_t group_start_bit = + *group_offsets(gid) - *group_offsets(chunk_id * groups_per_chunk); + size_t chunk_base_word = *chunk_word_offsets(chunk_id); + + size_t lsb_pos = bpw - (group_start_bit % bpw); + size_t cur = chunk_base_word + group_start_bit / bpw; + H buffer = 0; + bool first_word = true; + bool left_shared = (group_start_bit % bpw) != 0; + + for (size_t i = sym_base; i < sym_end; i++) { + H word = *codebook(*data(i)); // encode fused in: codeword for symbol + uint8_t bitwidth = deflate_bitwidth(word); + *((uint8_t *)&word + (sizeof(H) - 1)) = 0x0; // clear bitwidth byte + if (lsb_pos == bpw) + buffer = 0x0; // start of a fresh output word + if (bitwidth <= lsb_pos) { + lsb_pos -= bitwidth; + buffer |= word << lsb_pos; + if (lsb_pos == 0) { + // completed an output word + if (first_word && left_shared) { + Atomic::Or( + condensed(cur), buffer); + } else { + *condensed(cur) = buffer; + } + first_word = false; + cur++; + lsb_pos = bpw; + buffer = 0x0; + } + } else { + // code straddles two output words + H _1 = word >> (bitwidth - lsb_pos); + H _2 = word << (bpw - (bitwidth - lsb_pos)); + buffer |= _1; + if (first_word && left_shared) { + Atomic::Or( + condensed(cur), buffer); + } else { + *condensed(cur) = buffer; + } + first_word = false; + cur++; + buffer = _2; + lsb_pos = bpw - (bitwidth - lsb_pos); + } + } + // Trailing partial word is shared with the next group (or, for the last + // group in a chunk, lands in a word owned solely by this chunk). Either way + // it must be merged, not stored, since neighbours contribute the rest. + if (lsb_pos != bpw) { + Atomic::Or( + condensed(cur), buffer); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Q, DeviceType> data; + SubArray<1, H, DeviceType> codebook; + SubArray<1, size_t, DeviceType> group_offsets; + SubArray<1, size_t, DeviceType> chunk_word_offsets; + SubArray<1, H, DeviceType> condensed; + size_t primary_count; + SIZE chunk_size; + SIZE groups_per_chunk; + SIZE ngroups; + SIZE deflate_group_size; +}; + +template +class DeflatePackKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "deflate_pack"; + MGARDX_CONT + DeflatePackKernel(SubArray<1, Q, DeviceType> data, + SubArray<1, H, DeviceType> codebook, + SubArray<1, size_t, DeviceType> group_offsets, + SubArray<1, size_t, DeviceType> chunk_word_offsets, + SubArray<1, H, DeviceType> condensed, size_t primary_count, + SIZE chunk_size, SIZE groups_per_chunk, SIZE ngroups) + : data(data), codebook(codebook), group_offsets(group_offsets), + chunk_word_offsets(chunk_word_offsets), condensed(condensed), + primary_count(primary_count), chunk_size(chunk_size), + groups_per_chunk(groups_per_chunk), ngroups(ngroups) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = DeflatePackFunctor; + SIZE deflate_group_size = GetDeflateGroupSize(chunk_size); + FunctorType functor(data, codebook, group_offsets, chunk_word_offsets, + condensed, primary_count, chunk_size, groups_per_chunk, + ngroups, deflate_group_size); + // Same empirical sweep, reasoning, and HIP-only guard as + // DeflateGroupBitsKernel above -- 768 is the measured occupancy sweet + // spot on MI300 (256/512 ~113 GB/s, 768 ~120-124 GB/s, 960 back down to + // ~111 GB/s), untested on other backends. + SIZE tbx; + if constexpr (std::is_same::value) { + tbx = 768; + } else { + tbx = 256; + } + size_t sm_size = functor.shared_memory_size(); + SIZE gridx = (ngroups - 1) / tbx + 1; + return Task(functor, 1, 1, gridx, 1, 1, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Q, DeviceType> data; + SubArray<1, H, DeviceType> codebook; + SubArray<1, size_t, DeviceType> group_offsets; + SubArray<1, size_t, DeviceType> chunk_word_offsets; + SubArray<1, H, DeviceType> condensed; + size_t primary_count; + SIZE chunk_size; + SIZE groups_per_chunk; + SIZE ngroups; +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/ReorderByIndex.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/ReorderByIndex.hpp new file mode 100644 index 000000000..7d1dd0126 --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/ReorderByIndex.hpp @@ -0,0 +1,91 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_REORDER_BY_INDEX_TEMPLATE_HPP +#define MGARD_X_160_REORDER_BY_INDEX_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +// Jieyang: this kernel rely on whole grid sychronized execution +// For example, adding +// if (thread == 0) { +// __nanosleep(1e9); +// } +// will cause incorrect results + +template +class ReorderByIndexFunctor : public Functor { +public: + MGARDX_CONT ReorderByIndexFunctor() {} + MGARDX_CONT ReorderByIndexFunctor(SubArray<1, T, DeviceType> old_array, + SubArray<1, T, DeviceType> new_array, + SubArray<1, Q, DeviceType> index) + : old_array(old_array), new_array(new_array), index(index) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + unsigned int thread = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + T temp; + Q newIndex; + if (thread < old_array.shape(0)) { + temp = *old_array(thread); + newIndex = *index(thread); + *new_array(newIndex) = temp; + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T, DeviceType> old_array; + SubArray<1, T, DeviceType> new_array; + SubArray<1, Q, DeviceType> index; +}; + +template +class ReorderByIndexKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "reorder by index"; + + MGARDX_CONT + ReorderByIndexKernel(SubArray<1, T, DeviceType> old_array, + SubArray<1, T, DeviceType> new_array, + SubArray<1, Q, DeviceType> index) + : old_array(old_array), new_array(new_array), index(index) {} + + MGARDX_CONT + Task> GenTask(int queue_idx) { + using FunctorType = ReorderByIndexFunctor; + FunctorType functor(old_array, new_array, index); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = DeviceRuntime::GetMaxNumThreadsPerTB(); + gridz = 1; + gridy = 1; + gridx = (old_array.shape(0) / tbx) + 1; + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T, DeviceType> old_array; + SubArray<1, T, DeviceType> new_array; + SubArray<1, Q, DeviceType> index; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelHuffman/ReverseArray.hpp b/include/mgard-x-160/Lossless/ParallelHuffman/ReverseArray.hpp new file mode 100644 index 000000000..e66f928f3 --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelHuffman/ReverseArray.hpp @@ -0,0 +1,81 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_REVERSE_ARRAY_TEMPLATE_HPP +#define MGARD_X_160_REVERSE_ARRAY_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +template +class ReverseArrayFunctor : public Functor { +public: + MGARDX_CONT ReverseArrayFunctor() {} + MGARDX_CONT ReverseArrayFunctor(SubArray<1, T, DeviceType> array) + : array(array) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + unsigned int thread = (FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX()) + + FunctorBase::GetThreadIdX(); + if (thread < array.shape(0) / 2) { + T temp = *array(thread); + *array(thread) = *array(array.shape(0) - thread - 1); + *array(array.shape(0) - thread - 1) = temp; + } + } + + MGARDX_EXEC void Operation2() {} + + MGARDX_EXEC void Operation3() {} + + MGARDX_EXEC void Operation4() {} + + MGARDX_EXEC void Operation5() {} + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T, DeviceType> array; +}; + +template +class ReverseArrayKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "reverse array"; + MGARDX_CONT + ReverseArrayKernel(SubArray<1, T, DeviceType> array) : array(array) {} + + MGARDX_CONT + Task> GenTask(int queue_idx) { + using FunctorType = ReverseArrayFunctor; + FunctorType functor(array); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = DeviceRuntime::GetMaxNumThreadsPerTB(); + gridz = 1; + gridy = 1; + gridx = (array.shape(0) / tbx) + 1; + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T, DeviceType> array; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelRLE/Convert.hpp b/include/mgard-x-160/Lossless/ParallelRLE/Convert.hpp new file mode 100644 index 000000000..a2b57f783 --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelRLE/Convert.hpp @@ -0,0 +1,86 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_RLE_CONVERT_TEMPLATE_HPP +#define MGARD_X_160_RLE_CONVERT_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { +namespace parallel_rle { +template +class ConvertFunctor : public Functor { +public: + MGARDX_CONT ConvertFunctor() {} + MGARDX_CONT + ConvertFunctor(SubArray<1, C_run, DeviceType> counts_in, + SubArray<1, C_global, DeviceType> counts_out) + : counts_in(counts_in), counts_out(counts_out) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + IDX n = counts_in.shape(0); + + for (IDX i = start; i < n; i += grid_size) { + *counts_out(i) = (C_global)*counts_in(i); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, C_run, DeviceType> counts_in; + SubArray<1, C_global, DeviceType> counts_out; +}; + +template +class ConvertKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "convert"; + MGARDX_CONT + ConvertKernel(SubArray<1, C_run, DeviceType> counts_in, + SubArray<1, C_global, DeviceType> counts_out) + : counts_in(counts_in), counts_out(counts_out) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = ConvertFunctor; + FunctorType functor(counts_in, counts_out); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (counts_in.shape(0) - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, C_run, DeviceType> counts_in; + SubArray<1, C_global, DeviceType> counts_out; +}; +} // namespace parallel_rle +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelRLE/Decode.hpp b/include/mgard-x-160/Lossless/ParallelRLE/Decode.hpp new file mode 100644 index 000000000..45c3ecfec --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelRLE/Decode.hpp @@ -0,0 +1,109 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_RLE_DECODE_TEMPLATE_HPP +#define MGARD_X_160_RLE_DECODE_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { +namespace parallel_rle { +template +class DecodeFunctor : public Functor { +public: + MGARDX_CONT DecodeFunctor() {} + MGARDX_CONT DecodeFunctor(SubArray<1, C_run, DeviceType> counts, + SubArray<1, T_symbol, DeviceType> symbols, + SubArray<1, C_global, DeviceType> start_positions, + SubArray<1, T_symbol, DeviceType> data) + : counts(counts), symbols(symbols), start_positions(start_positions), + data(data) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + + IDX start = FunctorBase::GetBlockIdX(); + // * + // FunctorBase::GetBlockDimX() + + // FunctorBase::GetThreadIdX(); + + IDX n = counts.shape(0); + IDX grid_size = FunctorBase::GetGridDimX(); + // * + // FunctorBase::GetBlockDimX(); + + for (IDX i = start; i < n; i += grid_size) { + C_global curr_start_pos = i == 0 ? 0 : *start_positions(i - 1); + C_global next_start_pos = *start_positions(i); + T_symbol symbol = *symbols(i); + for (C_global j = + FunctorBase::GetThreadIdX() + curr_start_pos; + j < next_start_pos; j += FunctorBase::GetBlockDimX()) { + *data(j) = symbol; + } + + // for (SIZE j = curr_start_pos; j < curr_start_pos+1; j++) { + // *data(j) = symbol; + // } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, C_run, DeviceType> counts; + SubArray<1, T_symbol, DeviceType> symbols; + SubArray<1, C_global, DeviceType> start_positions; + SubArray<1, T_symbol, DeviceType> data; +}; + +template +class DecodeKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "decode"; + MGARDX_CONT + DecodeKernel(SubArray<1, C_run, DeviceType> counts, + SubArray<1, T_symbol, DeviceType> symbols, + SubArray<1, C_global, DeviceType> start_positions, + SubArray<1, T_symbol, DeviceType> data) + : counts(counts), symbols(symbols), start_positions(start_positions), + data(data) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = DecodeFunctor; + FunctorType functor(counts, symbols, start_positions, data); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 8; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = counts.shape(0); + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, C_run, DeviceType> counts; + SubArray<1, T_symbol, DeviceType> symbols; + SubArray<1, C_global, DeviceType> start_positions; + SubArray<1, T_symbol, DeviceType> data; +}; +} // namespace parallel_rle +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelRLE/Encode.hpp b/include/mgard-x-160/Lossless/ParallelRLE/Encode.hpp new file mode 100644 index 000000000..8c1bd02d7 --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelRLE/Encode.hpp @@ -0,0 +1,105 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_RLE_ENCODE_TEMPLATE_HPP +#define MGARD_X_160_RLE_ENCODE_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { +namespace parallel_rle { +template +class EncodeFunctor : public Functor { +public: + MGARDX_CONT EncodeFunctor() {} + MGARDX_CONT + EncodeFunctor(C_global total_run_length, + SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_positions, + SubArray<1, C_run, DeviceType> counts, + SubArray<1, T_symbol, DeviceType> symbols) + : total_run_length(total_run_length), data(data), + start_positions(start_positions), counts(counts), symbols(symbols) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + for (IDX i = start; i < total_run_length; i += grid_size) { + C_global curr_start_pos = *start_positions(i); + C_global next_start_pos = *start_positions(i + 1); + + *symbols(i) = *data(curr_start_pos); + *counts(i) = (C_run)(next_start_pos - curr_start_pos); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + C_global total_run_length; + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_positions; + SubArray<1, C_run, DeviceType> counts; + SubArray<1, T_symbol, DeviceType> symbols; +}; + +template +class EncodeKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "encode"; + MGARDX_CONT + EncodeKernel(C_global total_run_length, + SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_positions, + SubArray<1, C_run, DeviceType> counts, + SubArray<1, T_symbol, DeviceType> symbols) + : total_run_length(total_run_length), data(data), + start_positions(start_positions), counts(counts), symbols(symbols) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = EncodeFunctor; + FunctorType functor(total_run_length, data, start_positions, counts, + symbols); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (total_run_length - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + C_global total_run_length; + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_positions; + SubArray<1, C_run, DeviceType> counts; + SubArray<1, T_symbol, DeviceType> symbols; +}; +} // namespace parallel_rle +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelRLE/RunLengthEncoding.hpp b/include/mgard-x-160/Lossless/ParallelRLE/RunLengthEncoding.hpp new file mode 100644 index 000000000..e959c351f --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelRLE/RunLengthEncoding.hpp @@ -0,0 +1,361 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_RUN_LENGTH_ENCODING_TEMPLATE_HPP +#define MGARD_X_160_RUN_LENGTH_ENCODING_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" +#include "Convert.hpp" +#include "Decode.hpp" +#include "Encode.hpp" +#include "StartMarks.hpp" +#include "StartPositions.hpp" + +namespace mgard_x_160 { + +namespace parallel_rle { + +template +class RunLengthEncoding { +public: + RunLengthEncoding() : initialized(false) {} + + RunLengthEncoding(SIZE max_size) : initialized(true), max_size(max_size) {} + + void Resize(SIZE max_size, int queue_idx) { + this->initialized = true; + this->max_size = max_size; + start_marks.resize({max_size}, queue_idx); + scanned_start_marks.resize({max_size}, queue_idx); + start_positions.resize({max_size}, queue_idx); + MemoryManager::MallocHost(signature_verify, 7 * sizeof(char), + queue_idx); + DeviceCollective::ScanSumInclusive( + max_size, SubArray<1, C_global, DeviceType>(), + SubArray<1, C_global, DeviceType>(), this->scan_workspace, false, + queue_idx); + } + + static size_t EstimateMemoryFootprint(SIZE n) { + size_t memory_footprint = 0; + memory_footprint += n * sizeof(SIZE) * 3; + Array<1, Byte, DeviceType> tmp_workspace; + DeviceCollective::ScanSumInclusive( + n, SubArray<1, C_global, DeviceType>(), + SubArray<1, C_global, DeviceType>(), tmp_workspace, false, 0); + memory_footprint += tmp_workspace.shape(0); + return 0; + } + + double EstimateCR(Array<1, T_symbol, DeviceType> &original_data, + int queue_idx) { + Timer timer; + // Timer timer_each; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE original_length = original_data.shape(0); + + start_marks.resize({original_length}, queue_idx); + scanned_start_marks.resize({original_length}, queue_idx); + start_positions.resize({original_length}, queue_idx); + + // timer_each.start(); + + DeviceLauncher::Execute( + StartMarksKernel( + SubArray(original_data), SubArray(start_marks)), + queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // timer_each.end(); timer_each.print("StartMarksKernel", original_length * + // sizeof(T_symbol)); timer_each.clear(); timer_each.start(); + // DeviceRuntime::SyncQueue(queue_idx); + // PrintSubarray("StartMarksKernel", SubArray(start_marks)); + + DeviceCollective::ScanSumInclusive( + original_length, SubArray(start_marks), SubArray(scanned_start_marks), + scan_workspace, true, queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // timer_each.end(); timer_each.print("ScanSumInclusive", original_length * + // sizeof(T_symbol)); timer_each.clear(); timer_each.start(); + + C_global _total_run_length = 0; + MemoryManager::Copy1D( + &_total_run_length, scanned_start_marks.data() + original_length - 1, 1, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("RLE estimate CR", original_length * sizeof(T_symbol)); + timer.clear(); + } + + return (double)(original_length * sizeof(T_symbol)) / + (_total_run_length * (sizeof(T_symbol) + sizeof(C_run)) + 30); + } + + bool Compress(Array<1, T_symbol, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, float target_cr, + int queue_idx) { + Timer timer; + // Timer timer_each; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + // PrintSubarray("original_data", SubArray(original_data)); + + SIZE original_length = original_data.shape(0); + + start_marks.resize({original_length}, queue_idx); + scanned_start_marks.resize({original_length}, queue_idx); + start_positions.resize({original_length}, queue_idx); + + // timer_each.start(); + + DeviceLauncher::Execute( + StartMarksKernel( + SubArray(original_data), SubArray(start_marks)), + queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // timer_each.end(); timer_each.print("StartMarksKernel", original_length * + // sizeof(T_symbol)); timer_each.clear(); timer_each.start(); + // DeviceRuntime::SyncQueue(queue_idx); + // PrintSubarray("StartMarksKernel", SubArray(start_marks)); + + DeviceCollective::ScanSumInclusive( + original_length, SubArray(start_marks), SubArray(scanned_start_marks), + scan_workspace, true, queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // timer_each.end(); timer_each.print("ScanSumInclusive", original_length * + // sizeof(T_symbol)); timer_each.clear(); timer_each.start(); + + C_global _total_run_length = 0; + MemoryManager::Copy1D( + &_total_run_length, scanned_start_marks.data() + original_length - 1, 1, + queue_idx); + + if (target_cr > 0) { + double est_cr = + (double)(original_length * sizeof(T_symbol)) / + (_total_run_length * (sizeof(T_symbol) + sizeof(C_run)) + 30); + log::info("RLE estimated CR: " + std::to_string(est_cr) + + " (target: " + std::to_string(target_cr) + ")"); + if (est_cr < target_cr) { + return false; + } + } + + // DeviceRuntime::SyncQueue(queue_idx); + // PrintSubarray("scanned_start_marks", SubArray(scanned_start_marks)); + DeviceLauncher::Execute( + StartPositionsKernel( + SubArray(scanned_start_marks), SubArray(start_positions)), + queue_idx); + + // wait for total_run_length to be copied + total_run_length = _total_run_length; + DeviceRuntime::SyncQueue(queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // timer_each.end(); timer_each.print("StartPositionsKernel", + // original_length * sizeof(T_symbol)); timer_each.clear(); + // timer_each.start(); + // DeviceRuntime::SyncQueue(queue_idx); + // PrintSubarray("start_positions", SubArray(start_positions)); + + SIZE byte_offset = 0; + advance_with_align(byte_offset, 7); // signature + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, total_run_length); + advance_with_align(byte_offset, total_run_length); + + SIZE output_size = byte_offset; + compressed_data.resize({output_size}, queue_idx); + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + + byte_offset = 0; + SerializeArray(compressed_subarray, signature, 7, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &total_run_length, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &original_length, 1, byte_offset, + queue_idx); + + SubArray<1, C_run, DeviceType> counts( + {total_run_length}, (C_run *)(compressed_data.data() + byte_offset)); + advance_with_align(byte_offset, total_run_length); + SubArray<1, T_symbol, DeviceType> symbols( + {total_run_length}, (T_symbol *)(compressed_data.data() + byte_offset)); + advance_with_align(byte_offset, total_run_length); + + DeviceLauncher::Execute( + EncodeKernel( + total_run_length, SubArray(original_data), + SubArray(start_positions), counts, symbols), + queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // timer_each.end(); timer_each.print("EncodeKernel", original_length * + // sizeof(T_symbol)); timer_each.clear(); timer_each.start(); + + // PrintSubarray("counts", counts); + // PrintSubarray("symbols", symbols); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + log::info("RLE compression ratio: " + + std::to_string(original_length * sizeof(T_symbol)) + "/" + + std::to_string(compressed_data.shape(0)) + " (" + + std::to_string((double)original_length * sizeof(T_symbol) / + compressed_data.shape(0)) + + ")"); + timer.print("RLE compress", original_length * sizeof(T_symbol)); + timer.clear(); + } + + return true; + + // C_run * counts_host = new C_run[total_run_length]; + // MemoryManager::Copy1D(counts_host, counts.data(), + // total_run_length, queue_idx); + // DeviceRuntime::SyncQueue(queue_idx); + // C_run max_run = 0; + // for (int i = 0; i < total_run_length; i++) { + // if (counts_host[i] > max_run) { + // max_run = counts_host[i]; + // } + // } + + // std::cout << "total_run_length: " << total_run_length << std::endl; + // std::cout << "max_run: " << max_run << std::endl; + // IDX MAX_RUN = (IDX)1u << sizeof(C_run) * 8; + // std::cout << "max_run allowed: " << MAX_RUN << std::endl; + } + + void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) {} + + bool Verify(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + SubArray compressed_subarray(compressed_data); + SIZE byte_offset = 0; + DeserializeArray(compressed_subarray, signature_verify, 7, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + for (int i = 0; i < 7; i++) { + if (signature[i] != signature_verify[i]) { + return false; + } + } + return true; + } + + void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + if (!Verify(compressed_data, queue_idx)) { + throw std::runtime_error("RLE signature mismatch."); + } + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + Byte *signature_ptr = nullptr; + total_run_length_ptr = &total_run_length; + original_length_ptr = &original_length; + SIZE byte_offset = 0; + DeserializeArray(compressed_subarray, signature_ptr, 7, byte_offset, + true, queue_idx); + DeserializeArray(compressed_subarray, total_run_length_ptr, 1, + byte_offset, false, queue_idx); + DeserializeArray(compressed_subarray, original_length_ptr, 1, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + DeserializeArray(compressed_subarray, counts_ptr, total_run_length, + byte_offset, true, queue_idx); + DeserializeArray(compressed_subarray, symbols_ptr, + total_run_length, byte_offset, true, queue_idx); + + DeviceRuntime::SyncQueue(queue_idx); + // PrintSubarray("counts", SubArray<1, C_run, + // DeviceType>({total_run_length}, counts_ptr)); PrintSubarray("symbols", + // SubArray<1, T_symbol, DeviceType>({total_run_length}, symbols_ptr)); + // std::cout << "total_run_length: " << total_run_length << std::endl; + // std::cout << "original_length: " << original_length << std::endl; + } + + void Decompress(Array<1, Byte, DeviceType> &compressed_data, + Array<1, T_symbol, DeviceType> &decompressed_data, + int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + SubArray<1, C_run, DeviceType> counts({total_run_length}, counts_ptr); + SubArray<1, T_symbol, DeviceType> symbols({total_run_length}, symbols_ptr); + decompressed_data.resize({(SIZE)original_length}, queue_idx); + + // reuse array + SubArray counts_SIZE(start_marks); + DeviceLauncher::Execute( + ConvertKernel(counts, + counts_SIZE), + queue_idx); + + DeviceCollective::ScanSumInclusive( + total_run_length, counts_SIZE, SubArray(start_positions), + scan_workspace, true, queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // PrintSubarray("start_positions", SubArray(start_positions)); + + DeviceLauncher::Execute( + DecodeKernel( + counts, symbols, SubArray(start_positions), + SubArray(decompressed_data)), + queue_idx); + + // DeviceRuntime::SyncQueue(queue_idx); + // PrintSubarray("decompressed_data", SubArray(decompressed_data)); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("RLE decompress", original_length * sizeof(T_symbol)); + timer.clear(); + } + } + + bool initialized; + SIZE max_size; + SIZE original_length = 0; + SIZE total_run_length = 0; + SIZE *total_run_length_ptr = nullptr; + SIZE *original_length_ptr = nullptr; + C_run *counts_ptr = nullptr; + T_symbol *symbols_ptr = nullptr; + Byte signature[7] = {'M', 'G', 'X', 'R', 'L', 'E', 'C'}; + Byte *signature_verify; + + Array<1, C_global, DeviceType> start_marks; + Array<1, C_global, DeviceType> scanned_start_marks; + Array<1, C_global, DeviceType> start_positions; + Array<1, Byte, DeviceType> scan_workspace; +}; + +} // namespace parallel_rle +} // namespace mgard_x_160 +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelRLE/StartMarks.hpp b/include/mgard-x-160/Lossless/ParallelRLE/StartMarks.hpp new file mode 100644 index 000000000..0a3304aee --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelRLE/StartMarks.hpp @@ -0,0 +1,94 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_RLE_START_MARKS_TEMPLATE_HPP +#define MGARD_X_160_RLE_START_MARKS_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { +namespace parallel_rle { +template +class StartMarksFunctor : public Functor { +public: + MGARDX_CONT StartMarksFunctor() {} + MGARDX_CONT StartMarksFunctor(SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_marks) + : data(data), start_marks(start_marks) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX n = data.shape(0); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + // HIP will fail if making the following line a constexpr + IDX MAX_RUN = (IDX)1 << (sizeof(C_run) * 8); + for (IDX i = start; i < n; i += grid_size) { + if (i == 0) { + *start_marks(i) = 1; + } else { + if (i % MAX_RUN == 0) { + *start_marks(i) = 1; + } else { + *start_marks(i) = (*data(i) != *data(i - 1) ? 1 : 0); + } + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_marks; +}; + +template +class StartMarksKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "start marks"; + MGARDX_CONT + StartMarksKernel(SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_marks) + : data(data), start_marks(start_marks) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + StartMarksFunctor; + FunctorType functor(data, start_marks); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (data.shape(0) - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_marks; +}; +} // namespace parallel_rle +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelRLE/StartPositions.hpp b/include/mgard-x-160/Lossless/ParallelRLE/StartPositions.hpp new file mode 100644 index 000000000..66cee902a --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelRLE/StartPositions.hpp @@ -0,0 +1,99 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_RLE_START_POSITIONS_TEMPLATE_HPP +#define MGARD_X_160_RLE_START_POSITIONS_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { +namespace parallel_rle { +template +class StartPositionsFunctor : public Functor { +public: + MGARDX_CONT StartPositionsFunctor() {} + MGARDX_CONT + StartPositionsFunctor(SubArray<1, C_global, DeviceType> scanned_start_marks, + SubArray<1, C_global, DeviceType> start_positions) + : scanned_start_marks(scanned_start_marks), + start_positions(start_positions) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + IDX n = scanned_start_marks.shape(0); + + for (IDX i = start; i < n; i += grid_size) { + C_global curr_pos = *scanned_start_marks(i); + C_global prev_pos = i > 0 ? *scanned_start_marks(i - 1) : 0; + + if (i == n - 1) { + *start_positions(curr_pos) = n; + } + + if (i == 0) { + *start_positions((IDX)0) = 0; + } else if (curr_pos != prev_pos) { + *start_positions(curr_pos - 1) = i; + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, C_global, DeviceType> scanned_start_marks; + SubArray<1, C_global, DeviceType> start_positions; +}; + +template +class StartPositionsKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "start positions"; + MGARDX_CONT + StartPositionsKernel(SubArray<1, C_global, DeviceType> scanned_start_marks, + SubArray<1, C_global, DeviceType> start_positions) + : scanned_start_marks(scanned_start_marks), + start_positions(start_positions) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + StartPositionsFunctor; + FunctorType functor(scanned_start_marks, start_positions); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (scanned_start_marks.shape(0) - 1) / tbx + 1; + gridx = std::max((SIZE)1, gridx / repeat_factor); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, C_global, DeviceType> scanned_start_marks; + SubArray<1, C_global, DeviceType> start_positions; +}; +} // namespace parallel_rle +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Lossless/ParallelRLE/ZeroDecode.hpp b/include/mgard-x-160/Lossless/ParallelRLE/ZeroDecode.hpp new file mode 100644 index 000000000..339611d69 --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelRLE/ZeroDecode.hpp @@ -0,0 +1,172 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_ZERO_RLE_DECODE_TEMPLATE_HPP +#define MGARD_X_160_ZERO_RLE_DECODE_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { +namespace parallel_rle { + +// Step 1 of zero-RLE decode: widen each run count and add one, producing the +// per-symbol stride (preceding zeros + the symbol itself). The inclusive scan +// of these strides yields cum[s] = position[s] + 1, so position[s] = cum[s] - 1 +// recovers each nonzero's absolute index. +template +class ZeroStrideFunctor : public Functor { +public: + MGARDX_CONT ZeroStrideFunctor() {} + MGARDX_CONT + ZeroStrideFunctor(SubArray<1, C_run, DeviceType> counts, + SubArray<1, C_global, DeviceType> strides) + : counts(counts), strides(strides) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + IDX n = counts.shape(0); + + for (IDX i = start; i < n; i += grid_size) { + *strides(i) = (C_global)(*counts(i)) + 1; + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, C_run, DeviceType> counts; + SubArray<1, C_global, DeviceType> strides; +}; + +template +class ZeroStrideKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "zero rle stride"; + MGARDX_CONT + ZeroStrideKernel(SubArray<1, C_run, DeviceType> counts, + SubArray<1, C_global, DeviceType> strides) + : counts(counts), strides(strides) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + ZeroStrideFunctor; + FunctorType functor(counts, strides); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (counts.shape(0) - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, C_run, DeviceType> counts; + SubArray<1, C_global, DeviceType> strides; +}; + +// Step 2 of zero-RLE decode: scatter each nonzero symbol into its recovered +// absolute position. The output buffer is pre-zeroed by the caller, so only the +// nonzeros need to be written; runs of zeros (including any trailing zeros) are +// left untouched. +template +class ZeroScatterFunctor : public Functor { +public: + MGARDX_CONT ZeroScatterFunctor() {} + MGARDX_CONT + ZeroScatterFunctor(SubArray<1, T_symbol, DeviceType> symbols, + SubArray<1, C_global, DeviceType> strides, + SubArray<1, T_symbol, DeviceType> data) + : symbols(symbols), strides(strides), data(data) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + IDX n = symbols.shape(0); + + for (IDX i = start; i < n; i += grid_size) { + // strides has been inclusive-scanned: strides[i] == position[i] + 1. + C_global pos = *strides(i) - 1; + *data(pos) = *symbols(i); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T_symbol, DeviceType> symbols; + SubArray<1, C_global, DeviceType> strides; + SubArray<1, T_symbol, DeviceType> data; +}; + +template +class ZeroScatterKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "zero rle scatter"; + MGARDX_CONT + ZeroScatterKernel(SubArray<1, T_symbol, DeviceType> symbols, + SubArray<1, C_global, DeviceType> strides, + SubArray<1, T_symbol, DeviceType> data) + : symbols(symbols), strides(strides), data(data) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + ZeroScatterFunctor; + FunctorType functor(symbols, strides, data); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (symbols.shape(0) - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T_symbol, DeviceType> symbols; + SubArray<1, C_global, DeviceType> strides; + SubArray<1, T_symbol, DeviceType> data; +}; +} // namespace parallel_rle +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Lossless/ParallelRLE/ZeroEncode.hpp b/include/mgard-x-160/Lossless/ParallelRLE/ZeroEncode.hpp new file mode 100644 index 000000000..65a356a7c --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelRLE/ZeroEncode.hpp @@ -0,0 +1,114 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_ZERO_RLE_ENCODE_TEMPLATE_HPP +#define MGARD_X_160_ZERO_RLE_ENCODE_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { +namespace parallel_rle { + +// Zero-RLE encode. start_positions holds the absolute index of each of the +// num_symbols nonzeros (computed by the shared StartPositionsKernel), so for +// symbol s: +// symbols[s] = data[start_positions[s]] +// counts[s] = number of zeros immediately preceding it +// = start_positions[s] - start_positions[s-1] - 1 (s > 0) +// = start_positions[0] (s == 0) +// Trailing zeros after the last nonzero are not stored; they are recovered on +// decode from the known original length. +template +class ZeroEncodeFunctor : public Functor { +public: + MGARDX_CONT ZeroEncodeFunctor() {} + MGARDX_CONT + ZeroEncodeFunctor(C_global num_symbols, + SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_positions, + SubArray<1, C_run, DeviceType> counts, + SubArray<1, T_symbol, DeviceType> symbols) + : num_symbols(num_symbols), data(data), start_positions(start_positions), + counts(counts), symbols(symbols) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + for (IDX i = start; i < num_symbols; i += grid_size) { + C_global curr_pos = *start_positions(i); + C_global prev_pos = (i == 0) ? 0 : (*start_positions(i - 1) + 1); + + *symbols(i) = *data(curr_pos); + *counts(i) = (C_run)(curr_pos - prev_pos); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + C_global num_symbols; + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_positions; + SubArray<1, C_run, DeviceType> counts; + SubArray<1, T_symbol, DeviceType> symbols; +}; + +template +class ZeroEncodeKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "zero rle encode"; + MGARDX_CONT + ZeroEncodeKernel(C_global num_symbols, SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_positions, + SubArray<1, C_run, DeviceType> counts, + SubArray<1, T_symbol, DeviceType> symbols) + : num_symbols(num_symbols), data(data), start_positions(start_positions), + counts(counts), symbols(symbols) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + ZeroEncodeFunctor; + FunctorType functor(num_symbols, data, start_positions, counts, symbols); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (num_symbols - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + C_global num_symbols; + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_positions; + SubArray<1, C_run, DeviceType> counts; + SubArray<1, T_symbol, DeviceType> symbols; +}; +} // namespace parallel_rle +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Lossless/ParallelRLE/ZeroRunLengthEncoding.hpp b/include/mgard-x-160/Lossless/ParallelRLE/ZeroRunLengthEncoding.hpp new file mode 100644 index 000000000..624e355a8 --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelRLE/ZeroRunLengthEncoding.hpp @@ -0,0 +1,287 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_ZERO_RUN_LENGTH_ENCODING_TEMPLATE_HPP +#define MGARD_X_160_ZERO_RUN_LENGTH_ENCODING_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" +#include "ZeroDecode.hpp" +#include "ZeroEncode.hpp" +#include "ZeroStartMarks.hpp" +#include "ZeroStartPositions.hpp" + +namespace mgard_x_160 { + +namespace parallel_rle { + +// Zero-run-length encoding (RLE0). Only zeros are run-encoded; nonzero values +// are stored as literals together with the count of zeros that immediately +// precede each one. Trailing zeros are implied by the stored original length. +// This is the sparse-data front end intended to feed an entropy backend (e.g. +// Huffman/rANS) on the two output streams (counts, symbols) separately. +// +// Limitation: a single zero gap must fit in C_run (use uint32_t unless gaps can +// exceed 2^32). Unlike the full RLE, gaps are not split at C_run boundaries. +template +class ZeroRunLengthEncoding { +public: + ZeroRunLengthEncoding() : initialized(false) {} + + ZeroRunLengthEncoding(SIZE max_size) + : initialized(true), max_size(max_size) {} + + void Resize(SIZE max_size, int queue_idx) { + this->initialized = true; + this->max_size = max_size; + start_marks.resize({max_size}, queue_idx); + scanned_start_marks.resize({max_size}, queue_idx); + start_positions.resize({max_size}, queue_idx); + MemoryManager::MallocHost(signature_verify, 7 * sizeof(char), + queue_idx); + DeviceCollective::ScanSumInclusive( + max_size, SubArray<1, C_global, DeviceType>(), + SubArray<1, C_global, DeviceType>(), this->scan_workspace, false, + queue_idx); + } + + // Run the mark + scan stages and read back the number of nonzeros (= number + // of stored symbols). Leaves scanned_start_marks populated for the caller. + C_global CountSymbols(Array<1, T_symbol, DeviceType> &original_data, + int queue_idx) { + SIZE original_length = original_data.shape(0); + + start_marks.resize({original_length}, queue_idx); + scanned_start_marks.resize({original_length}, queue_idx); + start_positions.resize({original_length}, queue_idx); + + DeviceLauncher::Execute( + ZeroStartMarksKernel( + SubArray(original_data), SubArray(start_marks)), + queue_idx); + + DeviceCollective::ScanSumInclusive( + original_length, SubArray(start_marks), SubArray(scanned_start_marks), + scan_workspace, true, queue_idx); + + C_global _num_symbols = 0; + MemoryManager::Copy1D( + &_num_symbols, scanned_start_marks.data() + original_length - 1, 1, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + return _num_symbols; + } + + double EstimateCR(Array<1, T_symbol, DeviceType> &original_data, + int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE original_length = original_data.shape(0); + C_global _num_symbols = CountSymbols(original_data, queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Zero RLE estimate CR", original_length * sizeof(T_symbol)); + timer.clear(); + } + + return (double)(original_length * sizeof(T_symbol)) / + (_num_symbols * (sizeof(T_symbol) + sizeof(C_run)) + 30); + } + + bool Compress(Array<1, T_symbol, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, float target_cr, + int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE original_length = original_data.shape(0); + C_global _num_symbols = CountSymbols(original_data, queue_idx); + + if (target_cr > 0) { + double est_cr = (double)(original_length * sizeof(T_symbol)) / + (_num_symbols * (sizeof(T_symbol) + sizeof(C_run)) + 30); + log::info("Zero RLE estimated CR: " + std::to_string(est_cr) + + " (target: " + std::to_string(target_cr) + ")"); + if (est_cr < target_cr) { + return false; + } + } + + num_symbols = _num_symbols; + + // Compact nonzero positions, then derive (gap, value) pairs from them. + if (num_symbols > 0) { + DeviceLauncher::Execute( + ZeroStartPositionsKernel( + SubArray(scanned_start_marks), SubArray(start_positions)), + queue_idx); + } + + SIZE byte_offset = 0; + advance_with_align(byte_offset, 7); // signature + advance_with_align(byte_offset, 1); // num_symbols + advance_with_align(byte_offset, 1); // original_length + advance_with_align(byte_offset, num_symbols); + advance_with_align(byte_offset, num_symbols); + + SIZE output_size = byte_offset; + compressed_data.resize({output_size}, queue_idx); + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + + byte_offset = 0; + SerializeArray(compressed_subarray, signature, 7, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &num_symbols, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &original_length, 1, byte_offset, + queue_idx); + + SubArray<1, C_run, DeviceType> counts( + {num_symbols}, (C_run *)(compressed_data.data() + byte_offset)); + advance_with_align(byte_offset, num_symbols); + SubArray<1, T_symbol, DeviceType> symbols( + {num_symbols}, (T_symbol *)(compressed_data.data() + byte_offset)); + advance_with_align(byte_offset, num_symbols); + + if (num_symbols > 0) { + DeviceLauncher::Execute( + ZeroEncodeKernel( + num_symbols, SubArray(original_data), SubArray(start_positions), + counts, symbols), + queue_idx); + } + + DeviceRuntime::SyncQueue(queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + log::info("Zero RLE compression ratio: " + + std::to_string(original_length * sizeof(T_symbol)) + "/" + + std::to_string(compressed_data.shape(0)) + " (" + + std::to_string((double)original_length * sizeof(T_symbol) / + compressed_data.shape(0)) + + ")"); + timer.end(); + timer.print("Zero RLE compress", original_length * sizeof(T_symbol)); + timer.clear(); + } + + return true; + } + + void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) {} + + bool Verify(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + SubArray compressed_subarray(compressed_data); + SIZE byte_offset = 0; + DeserializeArray(compressed_subarray, signature_verify, 7, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + for (int i = 0; i < 7; i++) { + if (signature[i] != signature_verify[i]) { + return false; + } + } + return true; + } + + void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + if (!Verify(compressed_data, queue_idx)) { + throw std::runtime_error("Zero RLE signature mismatch."); + } + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + Byte *signature_ptr = nullptr; + num_symbols_ptr = &num_symbols; + original_length_ptr = &original_length; + SIZE byte_offset = 0; + DeserializeArray(compressed_subarray, signature_ptr, 7, byte_offset, + true, queue_idx); + DeserializeArray(compressed_subarray, num_symbols_ptr, 1, byte_offset, + false, queue_idx); + DeserializeArray(compressed_subarray, original_length_ptr, 1, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + DeserializeArray(compressed_subarray, counts_ptr, num_symbols, + byte_offset, true, queue_idx); + DeserializeArray(compressed_subarray, symbols_ptr, num_symbols, + byte_offset, true, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + } + + void Decompress(Array<1, Byte, DeviceType> &compressed_data, + Array<1, T_symbol, DeviceType> &decompressed_data, + int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SubArray<1, C_run, DeviceType> counts({(SIZE)num_symbols}, counts_ptr); + SubArray<1, T_symbol, DeviceType> symbols({(SIZE)num_symbols}, symbols_ptr); + decompressed_data.resize({(SIZE)original_length}, queue_idx); + + // Trailing/interior zeros are never written, so start from an all-zero + // buffer and scatter only the nonzeros. + MemoryManager::Memset1D(decompressed_data.data(), + original_length, 0, queue_idx); + + if (num_symbols > 0) { + // strides[i] = counts[i] + 1; inclusive scan -> position[i] + 1. + SubArray<1, C_global, DeviceType> strides(start_marks); + DeviceLauncher::Execute( + ZeroStrideKernel(counts, + strides), + queue_idx); + + DeviceCollective::ScanSumInclusive( + num_symbols, strides, SubArray(start_positions), scan_workspace, true, + queue_idx); + + DeviceLauncher::Execute( + ZeroScatterKernel( + symbols, SubArray(start_positions), SubArray(decompressed_data)), + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Zero RLE decompress", original_length * sizeof(T_symbol)); + timer.clear(); + } + } + + bool initialized; + SIZE max_size; + SIZE original_length = 0; + SIZE num_symbols = 0; + SIZE *num_symbols_ptr = nullptr; + SIZE *original_length_ptr = nullptr; + C_run *counts_ptr = nullptr; + T_symbol *symbols_ptr = nullptr; + Byte signature[7] = {'M', 'G', 'X', 'Z', 'R', 'L', '0'}; + Byte *signature_verify; + + Array<1, C_global, DeviceType> start_marks; + Array<1, C_global, DeviceType> scanned_start_marks; + Array<1, C_global, DeviceType> start_positions; + Array<1, Byte, DeviceType> scan_workspace; +}; + +} // namespace parallel_rle +} // namespace mgard_x_160 +#endif diff --git a/include/mgard-x-160/Lossless/ParallelRLE/ZeroStartMarks.hpp b/include/mgard-x-160/Lossless/ParallelRLE/ZeroStartMarks.hpp new file mode 100644 index 000000000..574036c4a --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelRLE/ZeroStartMarks.hpp @@ -0,0 +1,91 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_ZERO_RLE_START_MARKS_TEMPLATE_HPP +#define MGARD_X_160_ZERO_RLE_START_MARKS_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { +namespace parallel_rle { + +// Zero-RLE (RLE0) marks: a 1 at every *nonzero* element, 0 at zeros. The +// inclusive scan of these marks gives, for each nonzero, its rank among the +// nonzeros (and, in the last element, the total nonzero count = number of +// stored symbols). Unlike full RLE, zeros are never stored as symbols; they +// are recovered on decode from the gap (preceding-zero count) of each symbol. +template +class ZeroStartMarksFunctor : public Functor { +public: + MGARDX_CONT ZeroStartMarksFunctor() {} + MGARDX_CONT + ZeroStartMarksFunctor(SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_marks) + : data(data), start_marks(start_marks) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX n = data.shape(0); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + for (IDX i = start; i < n; i += grid_size) { + *start_marks(i) = (*data(i) != (T_symbol)0) ? 1 : 0; + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_marks; +}; + +template +class ZeroStartMarksKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "zero rle start marks"; + MGARDX_CONT + ZeroStartMarksKernel(SubArray<1, T_symbol, DeviceType> data, + SubArray<1, C_global, DeviceType> start_marks) + : data(data), start_marks(start_marks) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = + ZeroStartMarksFunctor; + FunctorType functor(data, start_marks); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (data.shape(0) - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + gridx / repeat_factor); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T_symbol, DeviceType> data; + SubArray<1, C_global, DeviceType> start_marks; +}; +} // namespace parallel_rle +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Lossless/ParallelRLE/ZeroStartPositions.hpp b/include/mgard-x-160/Lossless/ParallelRLE/ZeroStartPositions.hpp new file mode 100644 index 000000000..1e5177d38 --- /dev/null +++ b/include/mgard-x-160/Lossless/ParallelRLE/ZeroStartPositions.hpp @@ -0,0 +1,102 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_ZERO_RLE_START_POSITIONS_TEMPLATE_HPP +#define MGARD_X_160_ZERO_RLE_START_POSITIONS_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { +namespace parallel_rle { + +// Compact the absolute positions of the nonzeros. scanned_start_marks is the +// inclusive scan of the nonzero marks, so scanned[i] is the count of nonzeros +// in [0, i]. Element i is a nonzero exactly when its running count differs from +// the previous one; in that case its 0-based rank is scanned[i] - 1 and its +// position is i. Unlike the full-RLE StartPositions there is no i == 0 override +// (position 0 is only a symbol when data[0] is nonzero) and no terminal entry +// (ZeroEncode derives gaps from consecutive positions, not a sentinel). +template +class ZeroStartPositionsFunctor : public Functor { +public: + MGARDX_CONT ZeroStartPositionsFunctor() {} + MGARDX_CONT ZeroStartPositionsFunctor( + SubArray<1, C_global, DeviceType> scanned_start_marks, + SubArray<1, C_global, DeviceType> start_positions) + : scanned_start_marks(scanned_start_marks), + start_positions(start_positions) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + IDX n = scanned_start_marks.shape(0); + + for (IDX i = start; i < n; i += grid_size) { + C_global curr_pos = *scanned_start_marks(i); + C_global prev_pos = i > 0 ? *scanned_start_marks(i - 1) : 0; + if (curr_pos != prev_pos) { + *start_positions(curr_pos - 1) = i; + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, C_global, DeviceType> scanned_start_marks; + SubArray<1, C_global, DeviceType> start_positions; +}; + +template +class ZeroStartPositionsKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "zero rle start positions"; + MGARDX_CONT + ZeroStartPositionsKernel( + SubArray<1, C_global, DeviceType> scanned_start_marks, + SubArray<1, C_global, DeviceType> start_positions) + : scanned_start_marks(scanned_start_marks), + start_positions(start_positions) {} + + MGARDX_CONT + Task> + GenTask(int queue_idx) { + using FunctorType = + ZeroStartPositionsFunctor; + FunctorType functor(scanned_start_marks, start_positions); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + SIZE repeat_factor = 4; + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = (scanned_start_marks.shape(0) - 1) / tbx + 1; + gridx = std::max((SIZE)1, gridx / repeat_factor); + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, C_global, DeviceType> scanned_start_marks; + SubArray<1, C_global, DeviceType> start_positions; +}; +} // namespace parallel_rle +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Lossless/SymbolRans/SymbolRans.hpp b/include/mgard-x-160/Lossless/SymbolRans/SymbolRans.hpp new file mode 100644 index 000000000..51107e38f --- /dev/null +++ b/include/mgard-x-160/Lossless/SymbolRans/SymbolRans.hpp @@ -0,0 +1,186 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_SYMBOL_RANS_TEMPLATE_HPP +#define MGARD_X_160_SYMBOL_RANS_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" +#include "../ParallelHuffman/OutlierSeparator.hpp" +#include "../rANS/Rans.hpp" + +namespace mgard_x_160 { + +// Symbol-alphabet rANS lossless backend. Reuses Huffman's outlier front-end +// (values outside [0, dict_size) stored sparsely and zeroed in place) then +// entropy-codes the primary symbol stream with rANS over the dict_size +// alphabet. On the same symbol model rANS codes fractional bits, so its output +// is always <= Huffman's (no integer-code-length rounding loss) and uncapped -- +// the architecturally correct way to beat Huffman on ratio. Q = unsigned +// quantized type, S = signed. +template class SymbolRans { +public: + SymbolRans() : initialized(false) {} + + // SYMBOL_RANS_SEGMENT can be overridden at compile time to sweep the rANS + // segment size (smaller -> more parallel streams / occupancy, but more + // per-segment overhead -> lower CR). +#ifndef SYMBOL_RANS_SEGMENT +#define SYMBOL_RANS_SEGMENT 0 +#endif + + void Resize(SIZE n, int dict_size, double outlier_ratio, int queue_idx) { + initialized = true; + this->max_size = n; + this->dict_size = dict_size; + SIZE max_outliers = (SIZE)((double)n * outlier_ratio) + 1; + outlier_count_d.resize({1}, queue_idx); + outlier_idx_d.resize({max_outliers}, queue_idx); + outlier_val_d.resize({max_outliers}, queue_idx); + // NOTE: the shared-stream interleaved layout (interleaved=true) is the + // foundation for warp-cooperative coalesced encode, but its CUDA fast-path + // (warp kernels) is not implemented yet -- the sequential interleaved + // kernels would be a severe CUDA regression (one thread per block). So the + // symbol path uses the validated per-stream layout (block-interleaved + // symbol mapping already gives coalesced reads + a decode that beats + // Huffman). Flip to true once the warp kernels land. + rans.Resize(n, dict_size, queue_idx, (SIZE)SYMBOL_RANS_SEGMENT, + /*interleaved=*/false); + MemoryManager::MallocHost(signature_verify, 7 * sizeof(char), + queue_idx); + } + + void Compress(Array<1, S, DeviceType> &original_data, + Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + SIZE n = original_data.shape(0); + + // Separate out-of-dictionary outliers (zeroed in place in original_data). + ATOMIC_IDX zero = 0; + MemoryManager::Copy1D(outlier_count_d.data(), &zero, 1, + queue_idx); + DeviceLauncher::Execute( + OutlierSeparatorKernel( + SubArray(original_data), dict_size, SubArray(outlier_count_d), + SubArray(outlier_idx_d), SubArray(outlier_val_d)), + queue_idx); + MemoryManager::Copy1D(&outlier_count, outlier_count_d.data(), 1, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + if (outlier_count > outlier_val_d.shape(0)) { + throw std::runtime_error("SymbolRans: not enough outlier workspace."); + } + + // The in-range primary stream (outliers now zero) reinterpreted as + // unsigned. + Array<1, Q, DeviceType> primary({n}, (Q *)original_data.data()); + rans.Compress(primary, rans_stream, queue_idx); + SIZE rans_bytes = rans_stream.shape(0); + + // Layout: signature | n | dict_size | outlier_count | rans_bytes | + // outlier_idx[] | outlier_val[] | rANS stream + SIZE byte_offset = 0; + advance_with_align(byte_offset, 7); + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, 1); + advance_with_align(byte_offset, outlier_count); + advance_with_align(byte_offset, outlier_count); + advance_with_align(byte_offset, rans_bytes); + + compressed_data.resize({byte_offset}, queue_idx); + SubArray<1, Byte, DeviceType> cs(compressed_data); + + SIZE n_s = n, dict_s = dict_size, oc_s = outlier_count, rb_s = rans_bytes; + byte_offset = 0; + SerializeArray(cs, signature, 7, byte_offset, queue_idx); + SerializeArray(cs, &n_s, 1, byte_offset, queue_idx); + SerializeArray(cs, &dict_s, 1, byte_offset, queue_idx); + SerializeArray(cs, &oc_s, 1, byte_offset, queue_idx); + SerializeArray(cs, &rb_s, 1, byte_offset, queue_idx); + SerializeArray(cs, outlier_idx_d.data(), outlier_count, + byte_offset, queue_idx); + SerializeArray(cs, outlier_val_d.data(), outlier_count, byte_offset, + queue_idx); + align_byte_offset(byte_offset); + MemoryManager::Copy1D(compressed_data.data() + byte_offset, + rans_stream.data(), rans_bytes, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + } + + void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) {} + void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + } + + void Decompress(Array<1, Byte, DeviceType> &compressed_data, + Array<1, S, DeviceType> &decompressed_data, int queue_idx) { + SubArray<1, Byte, DeviceType> cs(compressed_data); + Byte *sig = nullptr; + SIZE n_s, dict_s, oc_s, rb_s; + SIZE *p_n = &n_s, *p_d = &dict_s, *p_oc = &oc_s, *p_rb = &rb_s; + SIZE byte_offset = 0; + DeserializeArray(cs, sig, 7, byte_offset, true, queue_idx); + DeserializeArray(cs, p_n, 1, byte_offset, false, queue_idx); + DeserializeArray(cs, p_d, 1, byte_offset, false, queue_idx); + DeserializeArray(cs, p_oc, 1, byte_offset, false, queue_idx); + DeserializeArray(cs, p_rb, 1, byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + SIZE n = n_s; + dict_size = dict_s; + outlier_count = oc_s; + SIZE rans_bytes = rb_s; + + ATOMIC_IDX *oidx_ptr = nullptr; + S *oval_ptr = nullptr; + DeserializeArray(cs, oidx_ptr, outlier_count, byte_offset, true, + queue_idx); + DeserializeArray(cs, oval_ptr, outlier_count, byte_offset, true, + queue_idx); + align_byte_offset(byte_offset); + Array<1, Byte, DeviceType> rans_alias({rans_bytes}, + compressed_data.data() + byte_offset); + + decompressed_data.resize({n}, queue_idx); + Array<1, Q, DeviceType> primary({n}, (Q *)decompressed_data.data()); + rans.Deserialize(rans_alias, queue_idx); + rans.Decompress(rans_alias, primary, queue_idx); + + // Scatter the outliers back into the decoded stream. + SubArray<1, ATOMIC_IDX, DeviceType> oidx({(SIZE)outlier_count}, oidx_ptr); + SubArray<1, S, DeviceType> oval({(SIZE)outlier_count}, oval_ptr); + DeviceLauncher::Execute( + OutlierSeparatorKernel( + SubArray(decompressed_data), dict_size, SubArray(outlier_count_d), + oidx, oval), + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + } + + static size_t EstimateMemoryFootprint(SIZE n, int dict_size, + double outlier_ratio) { + size_t size = 0; + size += (size_t)(n * outlier_ratio) * (sizeof(ATOMIC_IDX) + sizeof(S)); + size += (size_t)n * 2 + 16 * ((size_t)n / 2048 + 1); // rANS scratch ~2x + return size; + } + + bool initialized; + SIZE max_size = 0; + int dict_size = 0; + ATOMIC_IDX outlier_count = 0; + Byte signature[7] = {'M', 'G', 'X', 'S', 'R', 'A', 'N'}; + Byte *signature_verify; + + rans::Rans rans; + Array<1, Byte, DeviceType> rans_stream; + Array<1, ATOMIC_IDX, DeviceType> outlier_count_d; + Array<1, ATOMIC_IDX, DeviceType> outlier_idx_d; + Array<1, S, DeviceType> outlier_val_d; +}; + +} // namespace mgard_x_160 +#endif diff --git a/include/mgard-x-160/Lossless/Zstd.hpp b/include/mgard-x-160/Lossless/Zstd.hpp new file mode 100644 index 000000000..f0dd4ec74 --- /dev/null +++ b/include/mgard-x-160/Lossless/Zstd.hpp @@ -0,0 +1,165 @@ +#ifndef MGARD_X_160_ZSTD_TEMPLATE_HPP +#define MGARD_X_160_ZSTD_TEMPLATE_HPP + +#include +#include +#include + +#include "../RuntimeX/Utilities/Exceptions.h" + +namespace mgard_x_160 { + +#define CHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + char mgard_x_check_msg[256]; \ + std::snprintf(mgard_x_check_msg, sizeof(mgard_x_check_msg), \ + "CHECK(" #cond ") failed: " __VA_ARGS__); \ + throw mgard_x_160::ProcessingException(std::string(__FILE__) + ":" + \ + std::to_string(__LINE__) + " " + \ + mgard_x_check_msg); \ + } \ + } while (0) + +#define CHECK_ZSTD(fn, ...) \ + do { \ + size_t const err = (fn); \ + CHECK(!ZSTD_isError(err), "%s", ZSTD_getErrorName(err)); \ + } while (0) + +template class Zstd { +public: + Zstd() {} + Zstd(SIZE n, int compressionLevel) : compressionLevel(compressionLevel) { + Resize(n, compressionLevel, 0); + DeviceRuntime::SyncQueue(0); + } + + void Resize(SIZE buffer_size, int compressionLevel, int queue_idx) { + if (this->buffer_size < buffer_size) { + Release(queue_idx); + this->compressionLevel = compressionLevel; + size_t const estimated_out_size = ZSTD_compressBound(buffer_size); + MemoryManager::MallocHost( + out_data, estimated_out_size + sizeof(size_t), queue_idx); + MemoryManager::MallocHost(in_data, buffer_size, queue_idx); + this->buffer_size = buffer_size; + } + } + + void Release(int queue_idx) { + if (out_data != nullptr) { + MemoryManager::FreeHost(out_data, queue_idx); + out_data = nullptr; + } + if (in_data != nullptr) { + MemoryManager::FreeHost(in_data, queue_idx); + in_data = nullptr; + } + } + ~Zstd() { + Release(0); + DeviceRuntime::SyncQueue(0); + } + + static size_t EstimateMemoryFootprint(SIZE n) { + size_t size = 0; + return size; + } + + void Compress(Array<1, Byte, DeviceType> &data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) + timer.start(); + Array<1, Byte, DeviceType> &input_data = data; + Array<1, Byte, DeviceType> &output_data = data; + size_t input_count = input_data.shape(0); + Resize(input_count, compressionLevel, queue_idx); + size_t const estimated_out_size = ZSTD_compressBound(input_count); + MemoryManager::Copy1D(in_data, input_data.data(), input_count, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + size_t const actual_out_size = + ZSTD_compress(out_data + sizeof(size_t), estimated_out_size, in_data, + input_count, compressionLevel); + CHECK_ZSTD(actual_out_size); + + *(size_t *)out_data = (size_t)input_count; + output_data.resize({(SIZE)(actual_out_size + sizeof(size_t))}); + MemoryManager::Copy1D(output_data.data(), out_data, + actual_out_size + sizeof(size_t), + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + log::dbg("Zstd compression level: " + std::to_string(compressionLevel)); + log::dbg("Zstd compress ratio: " + + std::to_string((double)(input_count) / + (actual_out_size + sizeof(size_t)))); + if (log::level & log::TIME) { + timer.end(); + timer.print("Zstd compress", input_count); + timer.clear(); + } + } + + void Decompress(Array<1, Byte, DeviceType> &data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) + timer.start(); + Array<1, Byte, DeviceType> &input_data = data; + Array<1, Byte, DeviceType> &output_data = data; + size_t input_count = input_data.shape(0); + + Resize(input_count, compressionLevel, queue_idx); + MemoryManager::Copy1D(in_data, input_data.data(), input_count, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + // Written as a size_t by Compress; reading it into a narrower type would + // truncate streams of 4 GiB or more. + size_t actual_out_count = *reinterpret_cast(in_data); + + // FIX: Only resize out_data if needed, do not touch in_data which contains + // the compressed data we just copied. The original code called Resize() + // which would free and reallocate BOTH in_data and out_data, destroying + // the compressed data when actual_out_count > buffer_size. + if (this->buffer_size < actual_out_count) { + size_t const estimated_out_size = ZSTD_compressBound(actual_out_count); + if (out_data != nullptr) { + MemoryManager::FreeHost(out_data, queue_idx); + } + MemoryManager::MallocHost( + out_data, estimated_out_size + sizeof(size_t), queue_idx); + } + DeviceRuntime::SyncQueue(queue_idx); + + // A corrupt or truncated stream must fail here: an unchecked failure + // leaves out_data as garbage that the next stage misreads (MGARD 1.6.0 + // crashed in the Huffman decoder this way). + size_t const decompressed_size = + ZSTD_decompress(out_data, actual_out_count, in_data + sizeof(size_t), + input_count - sizeof(size_t)); + CHECK_ZSTD(decompressed_size); + CHECK(decompressed_size == actual_out_count, + "Zstd decompressed %zu bytes, header says %zu", decompressed_size, + actual_out_count); + + output_data.resize({(SIZE)actual_out_count}); + MemoryManager::Copy1D(output_data.data(), out_data, + actual_out_count, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + if (log::level & log::TIME) { + timer.end(); + timer.print("Zstd decompress", actual_out_count); + timer.clear(); + } + } + + int compressionLevel; + SIZE buffer_size = 0; + Byte *in_data = nullptr; + Byte *out_data = nullptr; +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Lossless/rANS/Rans.hpp b/include/mgard-x-160/Lossless/rANS/Rans.hpp new file mode 100644 index 000000000..15ce128e4 --- /dev/null +++ b/include/mgard-x-160/Lossless/rANS/Rans.hpp @@ -0,0 +1,510 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_RANS_TEMPLATE_HPP +#define MGARD_X_160_RANS_TEMPLATE_HPP + +#include + +#include "../../RuntimeX/RuntimeX.h" +#include "../ParallelHuffman/Histogram.hpp" +#include "RansCommon.hpp" +#include "RansDecode.hpp" +#include "RansEncode.hpp" +#include "RansInterleaved.hpp" + +namespace mgard_x_160 { +namespace rans { + +// Static rANS entropy coder over a symbol alphabet of arbitrary size (set at +// Resize): 256 for a byte stream, or dict_size for the Huffman-style primary +// quantized-symbol stream. Unlike Huffman it codes fractional bits, so the +// compressed size is the order-0 entropy of the symbol stream (no 1-bit/symbol +// floor and no integer-code-length rounding loss). Parallelism comes from +// splitting the input into independent segments, each its own rANS stream. +// +// Q is the symbol storage type (Byte, or the unsigned quantized type). Symbols +// must lie in [0, alphabet) and alphabet must be <= 65536 (slot table is +// uint16). The compressed stream is always bytes. +template class Rans { +public: + Rans() : initialized(false) {} + + // Choose M = 2^scale_bits with generous headroom over the alphabet so the + // normalized frequencies (each present symbol >= 1) keep enough precision: + // too small an M rounds the per-symbol probabilities coarsely and erases + // rANS's fractional-bit advantage over Huffman. Aim for ~16x the alphabet, + // capped at 2^16 (the max that keeps the 32-bit state valid with byte + // renormalization). + static uint32_t ScaleBitsFor(int alphabet) { + uint32_t sb = 12; + while (((uint32_t)1 << sb) < (uint32_t)alphabet * 16) { + sb++; + } + if (sb > 16) { + sb = 16; + } + return sb; + } + + void Resize(SIZE max_size, int alphabet, int queue_idx, + SIZE segment_size_override = 0, bool interleaved_mode = false) { + this->initialized = true; + this->max_size = max_size; + this->alphabet = alphabet; + this->scale_bits = ScaleBitsFor(alphabet); + this->interleaved = interleaved_mode; + this->segment_size = segment_size_override > 0 ? segment_size_override + : DEFAULT_SEGMENT_SIZE; + + // Two layouts share the same scratch/compact/offset machinery: + // - non-interleaved: one stream per lane (num_segments = blocks*NLANES), + // each stream's scratch holds up to segment_size symbols. + // - interleaved (warp-coalesced): one shared stream per block of NLANES + // lanes (num_segments = blocks), each block's scratch holds NLANES + // lanes' worth of bytes plus NLANES state flushes. + SIZE block_symbols = (SIZE)RANS_NLANES * segment_size; + SIZE max_blocks = (max_size + block_symbols - 1) / block_symbols; + SIZE max_segments; + if (interleaved) { + this->seg_capacity = + (IDX)RANS_NLANES * segment_size * 2 + (IDX)RANS_NLANES * 8 + 16; + max_segments = max_blocks; + } else { + this->seg_capacity = (IDX)segment_size * 2 + 16; + max_segments = max_blocks * (SIZE)RANS_NLANES; + } + if (max_segments == 0) { + max_segments = 1; + } + SIZE table_size = (SIZE)1 << scale_bits; + + freq32.resize({(SIZE)alphabet}, queue_idx); + freq_d.resize({(SIZE)alphabet}, queue_idx); + cum_d.resize({(SIZE)alphabet}, queue_idx); + esym_d.resize({(SIZE)alphabet}, queue_idx); + slot2sym_d.resize({table_size}, queue_idx); + seg_len_d.resize({max_segments}, queue_idx); + scratch.resize({(SIZE)(max_segments * seg_capacity)}, queue_idx); + + hcounts.resize(alphabet); + hfreq.resize(alphabet); + hcum.resize(alphabet); + hnorm.resize(alphabet); + hesym.resize(alphabet); + hslot.resize(table_size); + hseg_len.resize(max_segments); + hseg_off.resize(max_segments); + + MemoryManager::MallocHost(signature_verify, 7 * sizeof(char), + queue_idx); + } + + // Normalize the raw symbol counts so they sum to exactly M = 2^scale_bits, + // with every present symbol getting frequency >= 1, then build the cumulative + // table, the slot->symbol lookup, and the reciprocal-multiply encode tables. + void BuildTables(SIZE n, int queue_idx) { + uint32_t M = 1u << scale_bits; + + MemoryManager::Copy1D(hcounts.data(), freq32.data(), alphabet, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + uint32_t sum = 0; + int argmax = -1; + uint32_t max_freq = 0; + for (int s = 0; s < alphabet; s++) { + uint32_t f = 0; + if (hcounts[s] > 0) { + double scaled = (double)hcounts[s] * (double)M / (double)n; + f = (uint32_t)(scaled + 0.5); + if (f == 0) { + f = 1; + } + } + hnorm[s] = f; + sum += f; + if (f > max_freq) { + max_freq = f; + argmax = s; + } + } + + // Reconcile the rounded sum to exactly M by nudging the largest bins (never + // dropping a present symbol below 1). + while (sum > M) { + int best = -1; + uint32_t best_f = 1; + for (int s = 0; s < alphabet; s++) { + if (hnorm[s] > best_f) { + best_f = hnorm[s]; + best = s; + } + } + if (best < 0) { + break; + } + hnorm[best]--; + sum--; + } + while (sum < M) { + hnorm[argmax]++; + sum++; + } + + uint32_t c = 0; + for (int s = 0; s < alphabet; s++) { + hfreq[s] = hnorm[s]; + hcum[s] = c; + for (uint32_t k = 0; k < hnorm[s]; k++) { + hslot[c + k] = (uint16_t)s; + } + RansEncPacked &ep = hesym[s]; + if (hnorm[s] > 0) { + RansEncSymbol es; + RansEncSymbolInit(es, c, hnorm[s], scale_bits); + ep.x_max = es.x_max; + ep.rcp_freq = es.rcp_freq; + ep.bias = es.bias; + ep.cmpl_freq = (uint16_t)es.cmpl_freq; + ep.rcp_shift = (uint16_t)es.rcp_shift; + } else { + ep.x_max = 0; + ep.rcp_freq = 0; + ep.bias = 0; + ep.cmpl_freq = 0; + ep.rcp_shift = 0; + } + c += hnorm[s]; + } + + freq_d.load(hfreq.data(), 0, queue_idx); + cum_d.load(hcum.data(), 0, queue_idx); + slot2sym_d.load(hslot.data(), 0, queue_idx); + esym_d.load(hesym.data(), 0, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + } + + bool Compress(Array<1, Q, DeviceType> &input_data, + Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE n = input_data.shape(0); + SIZE block_symbols = (SIZE)RANS_NLANES * segment_size; + SIZE num_blocks = (n + block_symbols - 1) / block_symbols; + // The compaction unit ("segment") is one shared stream per block when + // interleaved, else one stream per lane. + SIZE num_segments = + interleaved ? num_blocks : num_blocks * (SIZE)RANS_NLANES; + if (n == 0) { + num_segments = 0; + } + + if (n > 0) { + MemoryManager::Memset1D(freq32.data(), alphabet, 0, + queue_idx); + Histogram(SubArray(input_data), SubArray(freq32), + n, alphabet, queue_idx); + BuildTables(n, queue_idx); + } + + if (num_segments > 0) { + seg_len_d.resize({num_segments}, queue_idx); + if (interleaved) { + if constexpr ((int)SubGroup::size() % (int)RANS_NLANES == + 0) { + // Warp-cooperative coalesced encode for subgroups that are a multiple + // of RANS_NLANES: a 32-lane CUDA warp handles one logical block, a + // 64-lane CDNA wavefront handles two. The bitstream is identical + // (always 32 logical lanes), so it stays cross-backend compatible. + DeviceLauncher::Execute( + InterleavedEncodeWarpKernel( + SubArray(input_data), n, segment_size, num_blocks, + SubArray(esym_d), seg_capacity, SubArray(scratch), + SubArray(seg_len_d)), + queue_idx); + } else { + // Portable sequential reference (size-1 subgroup backends). + DeviceLauncher::Execute( + InterleavedEncodeKernel( + SubArray(input_data), n, segment_size, num_blocks, + SubArray(esym_d), seg_capacity, SubArray(scratch), + SubArray(seg_len_d)), + queue_idx); + } + } else { + DeviceLauncher::Execute( + EncodeKernel(SubArray(input_data), n, segment_size, + num_segments, SubArray(esym_d), + seg_capacity, SubArray(scratch), + SubArray(seg_len_d)), + queue_idx); + } + + MemoryManager::Copy1D(hseg_len.data(), seg_len_d.data(), + num_segments, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + } + + uint64_t stream_bytes = 0; + for (SIZE p = 0; p < num_segments; p++) { + hseg_off[p] = (uint32_t)stream_bytes; + stream_bytes += hseg_len[p]; + } + + SIZE byte_offset = 0; + advance_with_align(byte_offset, 7); // signature + advance_with_align(byte_offset, 1); // scale_bits + advance_with_align(byte_offset, 1); // alphabet + advance_with_align(byte_offset, 1); // original_length + advance_with_align(byte_offset, 1); // segment_size + advance_with_align(byte_offset, 1); // interleaved + advance_with_align(byte_offset, 1); // num_segments + advance_with_align(byte_offset, 1); // stream_bytes + advance_with_align(byte_offset, alphabet); // norm freq + advance_with_align(byte_offset, num_segments); // seg offsets + advance_with_align(byte_offset, (SIZE)stream_bytes); // stream + + compressed_data.resize({byte_offset}, queue_idx); + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + + SIZE scale_bits_s = scale_bits, alphabet_s = alphabet, + original_length_s = n, segment_size_s = segment_size, + num_segments_s = num_segments, stream_bytes_s = stream_bytes, + interleaved_s = interleaved ? 1 : 0; + + byte_offset = 0; + SerializeArray(compressed_subarray, signature, 7, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &scale_bits_s, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &alphabet_s, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &original_length_s, 1, + byte_offset, queue_idx); + SerializeArray(compressed_subarray, &segment_size_s, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &interleaved_s, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &num_segments_s, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, &stream_bytes_s, 1, byte_offset, + queue_idx); + SerializeArray(compressed_subarray, hnorm.data(), alphabet, + byte_offset, queue_idx); + + align_byte_offset(byte_offset); + SubArray<1, uint32_t, DeviceType> seg_offset_sub( + {num_segments}, (uint32_t *)(compressed_data.data() + byte_offset)); + SerializeArray(compressed_subarray, hseg_off.data(), num_segments, + byte_offset, queue_idx); + + align_byte_offset(byte_offset); + SubArray<1, Byte, DeviceType> stream_sub( + {(SIZE)stream_bytes}, (Byte *)(compressed_data.data() + byte_offset)); + + if (num_segments > 0) { + DeviceLauncher::Execute( + CompactKernel(SubArray(scratch), SubArray(seg_len_d), + seg_offset_sub, num_segments, seg_capacity, + stream_sub), + queue_idx); + } + DeviceRuntime::SyncQueue(queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + log::info( + "rANS compression ratio: " + std::to_string(n * sizeof(Q)) + "/" + + std::to_string(compressed_data.shape(0)) + " (" + + std::to_string((double)n * sizeof(Q) / compressed_data.shape(0)) + + ")"); + timer.end(); + timer.print("rANS compress", n * sizeof(Q)); + timer.clear(); + } + return true; + } + + void Serialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) {} + + bool Verify(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + SubArray compressed_subarray(compressed_data); + SIZE byte_offset = 0; + DeserializeArray(compressed_subarray, signature_verify, 7, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + for (int i = 0; i < 7; i++) { + if (signature[i] != signature_verify[i]) { + return false; + } + } + return true; + } + + void Deserialize(Array<1, Byte, DeviceType> &compressed_data, int queue_idx) { + if (!Verify(compressed_data, queue_idx)) { + throw std::runtime_error("rANS signature mismatch."); + } + SubArray<1, Byte, DeviceType> compressed_subarray(compressed_data); + Byte *signature_ptr = nullptr; + SIZE scale_bits_s, alphabet_s, original_length_s, segment_size_s, + num_segments_s, stream_bytes_s, interleaved_s; + SIZE *p_scale = &scale_bits_s, *p_alpha = &alphabet_s, + *p_len = &original_length_s, *p_seg = &segment_size_s, + *p_num = &num_segments_s, *p_stream = &stream_bytes_s, + *p_il = &interleaved_s; + + SIZE byte_offset = 0; + DeserializeArray(compressed_subarray, signature_ptr, 7, byte_offset, + true, queue_idx); + DeserializeArray(compressed_subarray, p_scale, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, p_alpha, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, p_len, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, p_seg, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, p_il, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, p_num, 1, byte_offset, false, + queue_idx); + DeserializeArray(compressed_subarray, p_stream, 1, byte_offset, false, + queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + scale_bits = scale_bits_s; + alphabet = alphabet_s; + original_length = original_length_s; + segment_size = segment_size_s; + num_segments = num_segments_s; + stream_bytes = stream_bytes_s; + interleaved = interleaved_s != 0; + + if ((int)hnorm.size() < alphabet) { + hnorm.resize(alphabet); + hfreq.resize(alphabet); + hcum.resize(alphabet); + hslot.resize((size_t)1 << scale_bits); + } + uint16_t *hnorm_ptr = hnorm.data(); + DeserializeArray(compressed_subarray, hnorm_ptr, alphabet, + byte_offset, false, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + seg_offset_ptr = nullptr; + DeserializeArray(compressed_subarray, seg_offset_ptr, + num_segments, byte_offset, true, queue_idx); + stream_ptr = nullptr; + DeserializeArray(compressed_subarray, stream_ptr, stream_bytes, + byte_offset, true, queue_idx); + } + + void Decompress(Array<1, Byte, DeviceType> &compressed_data, + Array<1, Q, DeviceType> &decompressed_data, int queue_idx) { + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + decompressed_data.resize({(SIZE)original_length}, queue_idx); + + if (num_segments > 0) { + uint32_t c = 0; + for (int s = 0; s < alphabet; s++) { + hfreq[s] = hnorm[s]; + hcum[s] = c; + for (uint32_t k = 0; k < hnorm[s]; k++) { + hslot[c + k] = (uint16_t)s; + } + c += hnorm[s]; + } + if ((int)freq_d.shape(0) < alphabet) { + freq_d.resize({(SIZE)alphabet}, queue_idx); + cum_d.resize({(SIZE)alphabet}, queue_idx); + slot2sym_d.resize({(SIZE)1 << scale_bits}, queue_idx); + } + freq_d.load(hfreq.data(), 0, queue_idx); + cum_d.load(hcum.data(), 0, queue_idx); + slot2sym_d.load(hslot.data(), 0, queue_idx); + + SubArray<1, uint32_t, DeviceType> seg_offset_sub({(SIZE)num_segments}, + seg_offset_ptr); + SubArray<1, Byte, DeviceType> stream_sub({(SIZE)stream_bytes}, + stream_ptr); + + if (interleaved) { + // num_segments == num_blocks here. + DeviceLauncher::Execute( + InterleavedDecodeKernel( + stream_sub, seg_offset_sub, original_length, segment_size, + num_segments, SubArray(freq_d), SubArray(cum_d), + SubArray(slot2sym_d), scale_bits, SubArray(decompressed_data)), + queue_idx); + } else { + DeviceLauncher::Execute( + DecodeKernel( + stream_sub, seg_offset_sub, original_length, segment_size, + num_segments, SubArray(freq_d), SubArray(cum_d), + SubArray(slot2sym_d), scale_bits, SubArray(decompressed_data)), + queue_idx); + } + DeviceRuntime::SyncQueue(queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("rANS decompress", original_length * sizeof(Q)); + timer.clear(); + } + } + + static constexpr SIZE DEFAULT_SEGMENT_SIZE = 2048; + + bool initialized; + bool interleaved = false; + SIZE max_size = 0; + int alphabet = 256; + uint32_t scale_bits = 12; + SIZE segment_size = DEFAULT_SEGMENT_SIZE; + IDX seg_capacity = 0; + SIZE original_length = 0; + SIZE num_segments = 0; + SIZE stream_bytes = 0; + + uint32_t *seg_offset_ptr = nullptr; + Byte *stream_ptr = nullptr; + Byte signature[7] = {'M', 'G', 'X', 'R', 'A', 'N', 'S'}; + Byte *signature_verify; + + Array<1, uint32_t, DeviceType> freq32; + Array<1, uint32_t, DeviceType> freq_d; + Array<1, uint32_t, DeviceType> cum_d; + Array<1, RansEncPacked, DeviceType> esym_d; + Array<1, uint16_t, DeviceType> slot2sym_d; + Array<1, uint32_t, DeviceType> seg_len_d; + Array<1, Byte, DeviceType> scratch; + + std::vector hcounts; + std::vector hfreq; + std::vector hcum; + std::vector hnorm; + std::vector hesym; + std::vector hslot; + std::vector hseg_len; + std::vector hseg_off; +}; + +} // namespace rans +} // namespace mgard_x_160 +#endif diff --git a/include/mgard-x-160/Lossless/rANS/RansCommon.hpp b/include/mgard-x-160/Lossless/rANS/RansCommon.hpp new file mode 100644 index 000000000..2fe45d3ed --- /dev/null +++ b/include/mgard-x-160/Lossless/rANS/RansCommon.hpp @@ -0,0 +1,179 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_RANS_COMMON_TEMPLATE_HPP +#define MGARD_X_160_RANS_COMMON_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { +namespace rans { + +// Static range-ANS (rANS) over a byte alphabet (256 symbols), 32-bit state, +// byte-wise renormalization. The normalization interval is [RANS_L, RANS_L*256) +// so the state always fits in uint32 (RANS_L*256 == 2^31). Encoding writes +// bytes back-to-front (decreasing pointer); decoding reads them front-to-back. +// These primitives follow the well-known ryg_rans construction. + +static constexpr uint32_t RANS_L = 1u << 23; // lower bound of normalization +static constexpr int RANS_ALPHABET = 256; + +// Warp-interleaving factor. Streams are grouped into blocks of RANS_NLANES +// consecutive streams that interleave over a contiguous super-segment, so that +// at each step the RANS_NLANES streams (= consecutive threads in a warp) touch +// RANS_NLANES consecutive input/output positions -> coalesced memory access, +// with no extra per-stream metadata (same stream/flush/offset count as the +// non-interleaved layout). Stream p covers symbol positions +// (p / RANS_NLANES) * (RANS_NLANES * S) + (p % RANS_NLANES) + j * RANS_NLANES +// for j = 0..count_p-1, where S = symbols-per-stream (segment_size). +static constexpr IDX RANS_NLANES = 32; + +// Number of symbol positions assigned to interleaved stream p, given total n +// and S symbols-per-stream. Clamped to S so a stream never steals the next +// block's symbols. +MGARDX_CONT_EXEC IDX RansStreamBase(IDX p, IDX S) { + return (p / RANS_NLANES) * (RANS_NLANES * S) + (p % RANS_NLANES); +} +MGARDX_CONT_EXEC IDX RansStreamCount(IDX p, IDX S, IDX n) { + IDX base = RansStreamBase(p, S); + if (base >= n) { + return 0; + } + IDX count = (n - 1 - base) / RANS_NLANES + 1; + return count > S ? S : count; +} + +// Per-symbol precomputed encode constants (Alverson reciprocal, ryg_rans). They +// turn the encode step's integer division (x / freq, x % freq) into a +// multiply-high + shift, which is far cheaper on the GPU and is the encode +// bottleneck fix. Built once per frequency table on the host (see Rans.hpp). +// x_max = ((RANS_L >> scale_bits) << 8) * freq (renorm threshold) +// rcp_freq = fixed-point reciprocal of freq +// rcp_shift = reciprocal shift +// bias = additive bias +// cmpl_freq = (1 << scale_bits) - freq +struct RansEncSymbol { + uint32_t x_max; + uint32_t rcp_freq; + uint32_t bias; + uint32_t cmpl_freq; + uint32_t rcp_shift; +}; + +// Packed 16-byte encode-table entry so the hot encode loop fetches all +// per-symbol constants in a single (uint4) load instead of 5 separate array +// reads. The encode loop is compute/latency-bound, so cutting table loads +// 5x->1x is the main encode speedup. Internal to encode only (not part of the +// bitstream). +struct alignas(16) RansEncPacked { + uint32_t x_max; + uint32_t rcp_freq; + uint32_t bias; + uint16_t cmpl_freq; + uint16_t rcp_shift; +}; + +// Host-side: fill the precomputed encode constants for one symbol. Mirrors +// ryg_rans RansEncSymbolInit; produces a bitstream identical to the division +// form, so the decoder is unchanged. +MGARDX_CONT void RansEncSymbolInit(RansEncSymbol &s, uint32_t cum, + uint32_t freq, uint32_t scale_bits) { + s.x_max = ((RANS_L >> scale_bits) << 8) * freq; + s.cmpl_freq = (1u << scale_bits) - freq; + if (freq < 2) { + s.rcp_freq = ~0u; + s.rcp_shift = 0; + s.bias = cum + (1u << scale_bits) - 1; + } else { + uint32_t shift = 0; + while (freq > (1u << shift)) { + shift++; + } + s.rcp_freq = (uint32_t)(((1ull << (shift + 31)) + freq - 1) / freq); + s.rcp_shift = shift - 1; + s.bias = cum; + } +} + +// Split form of the encode step for warp-cooperative encoding, where the +// renorm bytes of all lanes in a step must be gathered and written coalesced +// (so they cannot be written inline). RansEncCollect reduces x and returns its +// renorm bytes (LSB first) in buf; RansEncApply then folds in the symbol. +MGARDX_CONT_EXEC int RansEncCollect(uint32_t &x, uint32_t x_max, Byte buf[4]) { + int nb = 0; + while (x >= x_max) { + buf[nb] = (Byte)(x & 0xFFu); + nb++; + x >>= 8; + } + return nb; +} +MGARDX_CONT_EXEC uint32_t RansEncApply(uint32_t x, uint32_t rcp_freq, + uint32_t bias, uint32_t cmpl_freq, + uint32_t rcp_shift) { + uint32_t q = (uint32_t)(((uint64_t)x * rcp_freq) >> 32); + return x + bias + (q >> rcp_shift) * cmpl_freq; +} + +// Encode one symbol with the precomputed constants: renormalize by flushing low +// bytes (back-to-front via the decreasing cursor ptr), then advance the state +// via reciprocal multiply instead of division. +template +MGARDX_EXEC uint32_t RansEncPut(uint32_t x, uint32_t x_max, uint32_t rcp_freq, + uint32_t bias, uint32_t cmpl_freq, + uint32_t rcp_shift, + SubArray<1, Byte, DeviceType> &out, IDX &ptr) { + if (x >= x_max) { + do { + --ptr; + *out(ptr) = (Byte)(x & 0xFFu); + x >>= 8; + } while (x >= x_max); + } + uint32_t q = (uint32_t)(((uint64_t)x * rcp_freq) >> 32); + return x + bias + (q >> rcp_shift) * cmpl_freq; +} + +// Flush the final 32-bit state as 4 little-endian bytes at the front of the +// (back-to-front) segment buffer. +template +MGARDX_EXEC void RansEncFlush(uint32_t x, SubArray<1, Byte, DeviceType> &out, + IDX &ptr) { + ptr -= 4; + *out(ptr + 0) = (Byte)(x >> 0); + *out(ptr + 1) = (Byte)(x >> 8); + *out(ptr + 2) = (Byte)(x >> 16); + *out(ptr + 3) = (Byte)(x >> 24); +} + +// Read the 4-byte little-endian state that RansEncFlush wrote. +template +MGARDX_EXEC uint32_t RansDecInit(SubArray<1, Byte, DeviceType> &in, IDX &rp) { + uint32_t x = (uint32_t)(*in(rp + 0)) | ((uint32_t)(*in(rp + 1)) << 8) | + ((uint32_t)(*in(rp + 2)) << 16) | + ((uint32_t)(*in(rp + 3)) << 24); + rp += 4; + return x; +} + +// Given the current slot (= x & mask) and the decoded symbol's (freq, cum), +// advance the state and renormalize by pulling bytes front-to-back. +template +MGARDX_EXEC uint32_t RansDecAdvance(uint32_t x, uint32_t freq, uint32_t cum, + uint32_t slot, uint32_t scale_bits, + SubArray<1, Byte, DeviceType> &in, + IDX &rp) { + x = freq * (x >> scale_bits) + slot - cum; + while (x < RANS_L) { + x = (x << 8) | (uint32_t)(*in(rp)); + ++rp; + } + return x; +} + +} // namespace rans +} // namespace mgard_x_160 +#endif diff --git a/include/mgard-x-160/Lossless/rANS/RansDecode.hpp b/include/mgard-x-160/Lossless/rANS/RansDecode.hpp new file mode 100644 index 000000000..4139b6128 --- /dev/null +++ b/include/mgard-x-160/Lossless/rANS/RansDecode.hpp @@ -0,0 +1,133 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_RANS_DECODE_TEMPLATE_HPP +#define MGARD_X_160_RANS_DECODE_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" +#include "RansCommon.hpp" + +namespace mgard_x_160 { +namespace rans { + +// Decode each segment independently: thread p reads its byte stream starting at +// seg_offset[p], reconstructs the state, then walks symbols forward (the mirror +// of the reverse encode), looking each up via the slot->symbol table and +// scattering into output[p*seg_size, ...). +template +class DecodeFunctor : public Functor { +public: + MGARDX_CONT DecodeFunctor() {} + MGARDX_CONT DecodeFunctor(SubArray<1, Byte, DeviceType> stream, + SubArray<1, uint32_t, DeviceType> seg_offset, + SIZE n, SIZE segment_size, SIZE num_segments, + SubArray<1, uint32_t, DeviceType> freq, + SubArray<1, uint32_t, DeviceType> cum, + SubArray<1, uint16_t, DeviceType> slot2sym, + uint32_t scale_bits, + SubArray<1, Q, DeviceType> output) + : stream(stream), seg_offset(seg_offset), n(n), + segment_size(segment_size), num_segments(num_segments), freq(freq), + cum(cum), slot2sym(slot2sym), scale_bits(scale_bits), output(output) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + uint32_t mask = (1u << scale_bits) - 1; + + for (IDX p = start; p < num_segments; p += grid_size) { + // Mirror the interleaved encode mapping: stream p produces symbols for + // positions base_pos + j*RANS_NLANES (consecutive streams -> consecutive + // output positions -> coalesced stores). + IDX base_pos = RansStreamBase(p, segment_size); + IDX count = RansStreamCount(p, segment_size, n); + + IDX rp = (IDX)(*seg_offset(p)); + uint32_t x = RansDecInit(stream, rp); + + for (IDX j = 0; j < count; j++) { + uint32_t slot = x & mask; + uint32_t s = (uint32_t)(*slot2sym(slot)); + *output(base_pos + j * RANS_NLANES) = (Q)s; + x = RansDecAdvance(x, *freq(s), *cum(s), slot, scale_bits, + stream, rp); + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Byte, DeviceType> stream; + SubArray<1, uint32_t, DeviceType> seg_offset; + SIZE n; + SIZE segment_size; + SIZE num_segments; + SubArray<1, uint32_t, DeviceType> freq; + SubArray<1, uint32_t, DeviceType> cum; + SubArray<1, uint16_t, DeviceType> slot2sym; + uint32_t scale_bits; + SubArray<1, Q, DeviceType> output; +}; + +template class DecodeKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "rans decode"; + MGARDX_CONT + DecodeKernel(SubArray<1, Byte, DeviceType> stream, + SubArray<1, uint32_t, DeviceType> seg_offset, SIZE n, + SIZE segment_size, SIZE num_segments, + SubArray<1, uint32_t, DeviceType> freq, + SubArray<1, uint32_t, DeviceType> cum, + SubArray<1, uint16_t, DeviceType> slot2sym, uint32_t scale_bits, + SubArray<1, Q, DeviceType> output) + : stream(stream), seg_offset(seg_offset), n(n), + segment_size(segment_size), num_segments(num_segments), freq(freq), + cum(cum), slot2sym(slot2sym), scale_bits(scale_bits), output(output) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = DecodeFunctor; + FunctorType functor(stream, seg_offset, n, segment_size, num_segments, freq, + cum, slot2sym, scale_bits, output); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + // One thread per segment, full grid (no repeat_factor cap) to maximize + // occupancy for the serial per-segment decode chain. + gridx = (num_segments - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), gridx); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Byte, DeviceType> stream; + SubArray<1, uint32_t, DeviceType> seg_offset; + SIZE n; + SIZE segment_size; + SIZE num_segments; + SubArray<1, uint32_t, DeviceType> freq; + SubArray<1, uint32_t, DeviceType> cum; + SubArray<1, uint16_t, DeviceType> slot2sym; + uint32_t scale_bits; + SubArray<1, Q, DeviceType> output; +}; + +} // namespace rans +} // namespace mgard_x_160 +#endif diff --git a/include/mgard-x-160/Lossless/rANS/RansEncode.hpp b/include/mgard-x-160/Lossless/rANS/RansEncode.hpp new file mode 100644 index 000000000..de54329a2 --- /dev/null +++ b/include/mgard-x-160/Lossless/rANS/RansEncode.hpp @@ -0,0 +1,207 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_RANS_ENCODE_TEMPLATE_HPP +#define MGARD_X_160_RANS_ENCODE_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" +#include "RansCommon.hpp" + +namespace mgard_x_160 { +namespace rans { + +// One independent rANS stream per segment (interleaving across threads gives +// the parallelism). Thread p encodes input[p*seg_size, ...) in reverse into a +// private scratch region [p*seg_capacity, (p+1)*seg_capacity), filling it +// back-to-front, then records the produced byte length. Bytes live at the high +// end of the region: [seg_capacity - seg_len, seg_capacity). +template +class EncodeFunctor : public Functor { +public: + MGARDX_CONT EncodeFunctor() {} + MGARDX_CONT + EncodeFunctor(SubArray<1, Q, DeviceType> input, SIZE n, SIZE segment_size, + SIZE num_segments, SubArray<1, RansEncPacked, DeviceType> esym, + IDX seg_capacity, SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len) + : input(input), n(n), segment_size(segment_size), + num_segments(num_segments), esym(esym), seg_capacity(seg_capacity), + scratch(scratch), seg_len(seg_len) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + for (IDX p = start; p < num_segments; p += grid_size) { + // Interleaved symbol mapping: consecutive streams (= consecutive warp + // lanes) read consecutive positions, so the input loads coalesce. + IDX base_pos = RansStreamBase(p, segment_size); + IDX count = RansStreamCount(p, segment_size, n); + + IDX base = p * seg_capacity; + IDX ptr = base + seg_capacity; // cursor, fills downward + + uint32_t x = RANS_L; + for (IDX j = count; j > 0; j--) { + IDX pos = base_pos + (j - 1) * RANS_NLANES; + uint32_t s = (uint32_t)(*input(pos)); + RansEncPacked e = *esym(s); // single 16-byte load of all constants + x = RansEncPut(x, e.x_max, e.rcp_freq, e.bias, + (uint32_t)e.cmpl_freq, (uint32_t)e.rcp_shift, + scratch, ptr); + } + RansEncFlush(x, scratch, ptr); + + *seg_len(p) = (uint32_t)(base + seg_capacity - ptr); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Q, DeviceType> input; + SIZE n; + SIZE segment_size; + SIZE num_segments; + SubArray<1, RansEncPacked, DeviceType> esym; + IDX seg_capacity; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; +}; + +template class EncodeKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "rans encode"; + MGARDX_CONT + EncodeKernel(SubArray<1, Q, DeviceType> input, SIZE n, SIZE segment_size, + SIZE num_segments, SubArray<1, RansEncPacked, DeviceType> esym, + IDX seg_capacity, SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len) + : input(input), n(n), segment_size(segment_size), + num_segments(num_segments), esym(esym), seg_capacity(seg_capacity), + scratch(scratch), seg_len(seg_len) {} + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = EncodeFunctor; + FunctorType functor(input, n, segment_size, num_segments, esym, + seg_capacity, scratch, seg_len); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + // One thread per segment, full grid (no repeat_factor cap): the per-thread + // rANS loop is a long serial dependent chain, so we want maximum occupancy + // to hide its latency rather than fewer threads each doing more segments. + gridx = (num_segments - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), gridx); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Q, DeviceType> input; + SIZE n; + SIZE segment_size; + SIZE num_segments; + SubArray<1, RansEncPacked, DeviceType> esym; + IDX seg_capacity; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; +}; + +// Copy each segment's bytes out of its scratch region (back-to-front layout) +// into a single contiguous stream at seg_offset[p]. One block per segment. +template +class CompactFunctor : public Functor { +public: + MGARDX_CONT CompactFunctor() {} + MGARDX_CONT CompactFunctor(SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len, + SubArray<1, uint32_t, DeviceType> seg_offset, + SIZE num_segments, IDX seg_capacity, + SubArray<1, Byte, DeviceType> stream) + : scratch(scratch), seg_len(seg_len), seg_offset(seg_offset), + num_segments(num_segments), seg_capacity(seg_capacity), stream(stream) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + for (IDX p = FunctorBase::GetBlockIdX(); p < num_segments; + p += FunctorBase::GetGridDimX()) { + uint32_t len = *seg_len(p); + IDX src = p * seg_capacity + (seg_capacity - (IDX)len); + IDX dst = (IDX)(*seg_offset(p)); + for (IDX j = FunctorBase::GetThreadIdX(); j < len; + j += FunctorBase::GetBlockDimX()) { + *stream(dst + j) = *scratch(src + j); + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; + SubArray<1, uint32_t, DeviceType> seg_offset; + SIZE num_segments; + IDX seg_capacity; + SubArray<1, Byte, DeviceType> stream; +}; + +template class CompactKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "rans compact"; + MGARDX_CONT + CompactKernel(SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len, + SubArray<1, uint32_t, DeviceType> seg_offset, SIZE num_segments, + IDX seg_capacity, SubArray<1, Byte, DeviceType> stream) + : scratch(scratch), seg_len(seg_len), seg_offset(seg_offset), + num_segments(num_segments), seg_capacity(seg_capacity), stream(stream) { + } + + MGARDX_CONT Task> GenTask(int queue_idx) { + using FunctorType = CompactFunctor; + FunctorType functor(scratch, seg_len, seg_offset, num_segments, + seg_capacity, stream); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = 1; + tby = 1; + tbx = 256; + gridz = 1; + gridy = 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), + (SIZE)num_segments); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; + SubArray<1, uint32_t, DeviceType> seg_offset; + SIZE num_segments; + IDX seg_capacity; + SubArray<1, Byte, DeviceType> stream; +}; + +} // namespace rans +} // namespace mgard_x_160 +#endif diff --git a/include/mgard-x-160/Lossless/rANS/RansInterleaved.hpp b/include/mgard-x-160/Lossless/rANS/RansInterleaved.hpp new file mode 100644 index 000000000..49d8c34bd --- /dev/null +++ b/include/mgard-x-160/Lossless/rANS/RansInterleaved.hpp @@ -0,0 +1,448 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + */ + +#ifndef MGARD_X_160_RANS_INTERLEAVED_TEMPLATE_HPP +#define MGARD_X_160_RANS_INTERLEAVED_TEMPLATE_HPP + +#include "../../RuntimeX/RuntimeX.h" +#include "RansCommon.hpp" + +namespace mgard_x_160 { +namespace rans { + +// Shared-stream interleaved rANS: a "block" of RANS_NLANES independent rANS +// states shares ONE byte stream, with the states' renorm bytes intermixed in +// LIFO emission order (standard interleaved rANS, generalized to NLANES +// states). This is the layout required to coalesce the encode byte writes (the +// CUDA warp-cooperative kernels below process one block per warp and write the +// per-step bytes coalesced). The SEQUENTIAL reference functors here (one thread +// per block) define the canonical byte order and are the correctness oracle / +// portable fallback; they produce a bitstream identical to the warp kernels. +// +// Block b, lane L owns symbol positions b*NLANES*S + L + j*NLANES, j < count, +// where S = symbols-per-lane (segment_size). Encode order: j high->low, then +// L = 0..NLANES-1; states flushed L = 0..NLANES-1 at the end (lowest address). +// Decode mirrors: init states L = NLANES-1..0, then j low->high, L = +// NLANES-1..0. + +template +MGARDX_EXEC IDX RansBlockLaneBase(IDX b, IDX L, IDX S) { + return b * (RANS_NLANES * S) + L; +} +template +MGARDX_EXEC IDX RansBlockLaneCount(IDX b, IDX L, IDX S, IDX n) { + IDX base = b * (RANS_NLANES * S) + L; + if (base >= n) { + return 0; + } + IDX count = (n - 1 - base) / RANS_NLANES + 1; + return count > S ? S : count; +} + +// --------------------------------------------------------------------------- +// Sequential reference encode: one thread per block. +// --------------------------------------------------------------------------- +template +class InterleavedEncodeFunctor : public Functor { +public: + MGARDX_CONT InterleavedEncodeFunctor() {} + MGARDX_CONT InterleavedEncodeFunctor( + SubArray<1, Q, DeviceType> input, SIZE n, SIZE segment_size, + SIZE num_blocks, SubArray<1, RansEncPacked, DeviceType> esym, + IDX block_capacity, SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len) + : input(input), n(n), segment_size(segment_size), num_blocks(num_blocks), + esym(esym), block_capacity(block_capacity), scratch(scratch), + seg_len(seg_len) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + + for (IDX b = start; b < num_blocks; b += grid_size) { + uint32_t x[RANS_NLANES]; + for (IDX L = 0; L < RANS_NLANES; L++) { + x[L] = RANS_L; + } + // Lane 0 starts earliest, so it has the most symbols => maxcount. + IDX maxcount = RansBlockLaneCount(b, 0, segment_size, n); + + IDX ptr = b * block_capacity + block_capacity; // cursor, fills downward + + for (IDX jj = maxcount; jj > 0; jj--) { + IDX j = jj - 1; + for (IDX L = 0; L < RANS_NLANES; L++) { + IDX base = b * (RANS_NLANES * segment_size) + L; + IDX count = RansBlockLaneCount(b, L, segment_size, n); + if (j < count) { + IDX pos = base + j * RANS_NLANES; + uint32_t s = (uint32_t)(*input(pos)); + RansEncPacked e = *esym(s); + x[L] = RansEncPut(x[L], e.x_max, e.rcp_freq, e.bias, + (uint32_t)e.cmpl_freq, + (uint32_t)e.rcp_shift, scratch, ptr); + } + } + } + for (IDX L = 0; L < RANS_NLANES; L++) { + RansEncFlush(x[L], scratch, ptr); + } + *seg_len(b) = (uint32_t)(b * block_capacity + block_capacity - ptr); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Q, DeviceType> input; + SIZE n; + SIZE segment_size; + SIZE num_blocks; + SubArray<1, RansEncPacked, DeviceType> esym; + IDX block_capacity; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; +}; + +template +class InterleavedEncodeKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "rans interleaved encode"; + MGARDX_CONT InterleavedEncodeKernel( + SubArray<1, Q, DeviceType> input, SIZE n, SIZE segment_size, + SIZE num_blocks, SubArray<1, RansEncPacked, DeviceType> esym, + IDX block_capacity, SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len) + : input(input), n(n), segment_size(segment_size), num_blocks(num_blocks), + esym(esym), block_capacity(block_capacity), scratch(scratch), + seg_len(seg_len) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = InterleavedEncodeFunctor; + FunctorType functor(input, n, segment_size, num_blocks, esym, + block_capacity, scratch, seg_len); + SIZE tbx = 256, gridx; + gridx = (num_blocks - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), gridx); + return Task(functor, 1, 1, gridx, 1, 1, tbx, 0, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Q, DeviceType> input; + SIZE n; + SIZE segment_size; + SIZE num_blocks; + SubArray<1, RansEncPacked, DeviceType> esym; + IDX block_capacity; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; +}; + +// --------------------------------------------------------------------------- +// Sequential reference decode: one thread per block. +// --------------------------------------------------------------------------- +template +class InterleavedDecodeFunctor : public Functor { +public: + MGARDX_CONT InterleavedDecodeFunctor() {} + MGARDX_CONT InterleavedDecodeFunctor( + SubArray<1, Byte, DeviceType> stream, + SubArray<1, uint32_t, DeviceType> seg_offset, SIZE n, SIZE segment_size, + SIZE num_blocks, SubArray<1, uint32_t, DeviceType> freq, + SubArray<1, uint32_t, DeviceType> cum, + SubArray<1, uint16_t, DeviceType> slot2sym, uint32_t scale_bits, + SubArray<1, Q, DeviceType> output) + : stream(stream), seg_offset(seg_offset), n(n), + segment_size(segment_size), num_blocks(num_blocks), freq(freq), + cum(cum), slot2sym(slot2sym), scale_bits(scale_bits), output(output) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + IDX start = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX grid_size = FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX(); + uint32_t mask = (1u << scale_bits) - 1; + + for (IDX b = start; b < num_blocks; b += grid_size) { + IDX rp = (IDX)(*seg_offset(b)); + uint32_t x[RANS_NLANES]; + // Init states in reverse of the flush order (flush wrote L=0..NLANES-1, + // so the lowest address is L=NLANES-1, read first going forward). + for (IDX LL = RANS_NLANES; LL > 0; LL--) { + x[LL - 1] = RansDecInit(stream, rp); + } + + IDX maxcount = RansBlockLaneCount(b, 0, segment_size, n); + for (IDX j = 0; j < maxcount; j++) { + for (IDX LL = RANS_NLANES; LL > 0; LL--) { + IDX L = LL - 1; + IDX count = RansBlockLaneCount(b, L, segment_size, n); + if (j < count) { + uint32_t slot = x[L] & mask; + uint32_t s = (uint32_t)(*slot2sym(slot)); + IDX pos = b * (RANS_NLANES * segment_size) + L + j * RANS_NLANES; + *output(pos) = (Q)s; + x[L] = RansDecAdvance(x[L], *freq(s), *cum(s), slot, + scale_bits, stream, rp); + } + } + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Byte, DeviceType> stream; + SubArray<1, uint32_t, DeviceType> seg_offset; + SIZE n; + SIZE segment_size; + SIZE num_blocks; + SubArray<1, uint32_t, DeviceType> freq; + SubArray<1, uint32_t, DeviceType> cum; + SubArray<1, uint16_t, DeviceType> slot2sym; + uint32_t scale_bits; + SubArray<1, Q, DeviceType> output; +}; + +template +class InterleavedDecodeKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "rans interleaved decode"; + MGARDX_CONT InterleavedDecodeKernel( + SubArray<1, Byte, DeviceType> stream, + SubArray<1, uint32_t, DeviceType> seg_offset, SIZE n, SIZE segment_size, + SIZE num_blocks, SubArray<1, uint32_t, DeviceType> freq, + SubArray<1, uint32_t, DeviceType> cum, + SubArray<1, uint16_t, DeviceType> slot2sym, uint32_t scale_bits, + SubArray<1, Q, DeviceType> output) + : stream(stream), seg_offset(seg_offset), n(n), + segment_size(segment_size), num_blocks(num_blocks), freq(freq), + cum(cum), slot2sym(slot2sym), scale_bits(scale_bits), output(output) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = InterleavedDecodeFunctor; + FunctorType functor(stream, seg_offset, n, segment_size, num_blocks, freq, + cum, slot2sym, scale_bits, output); + SIZE tbx = 256, gridx; + gridx = (num_blocks - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), gridx); + return Task(functor, 1, 1, gridx, 1, 1, tbx, 0, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Byte, DeviceType> stream; + SubArray<1, uint32_t, DeviceType> seg_offset; + SIZE n; + SIZE segment_size; + SIZE num_blocks; + SubArray<1, uint32_t, DeviceType> freq; + SubArray<1, uint32_t, DeviceType> cum; + SubArray<1, uint16_t, DeviceType> slot2sym; + uint32_t scale_bits; + SubArray<1, Q, DeviceType> output; +}; + +// --------------------------------------------------------------------------- +// Warp-cooperative encode: one SubGroup (= RANS_NLANES lanes) per block, lane L +// owns rANS state L. Written against the portable SubGroup abstraction, so the +// per-step renorm bytes of all lanes are gathered with a subgroup prefix-sum +// and written COALESCED, in the exact byte order of the sequential reference +// above (lane 0 at the high end of each step's range). Used only where the +// subgroup size equals RANS_NLANES (CUDA warp); other backends use the +// sequential kernel. +template +class InterleavedEncodeWarpFunctor : public Functor { +public: + MGARDX_CONT InterleavedEncodeWarpFunctor() {} + MGARDX_CONT InterleavedEncodeWarpFunctor( + SubArray<1, Q, DeviceType> input, SIZE n, SIZE segment_size, + SIZE num_blocks, SubArray<1, RansEncPacked, DeviceType> esym, + IDX block_capacity, SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len) + : input(input), n(n), segment_size(segment_size), num_blocks(num_blocks), + esym(esym), block_capacity(block_capacity), scratch(scratch), + seg_len(seg_len) { + Functor(); + } + + // Exclusive prefix sum of v across ONE RANS_NLANES-lane segment of the + // subgroup (lanes [seg_base, seg_base+RANS_NLANES)); returns the segment + // total via the reference parameter. logical_lane is the 0..RANS_NLANES-1 + // position within the segment. A wavefront wider than RANS_NLANES holds + // several independent segments, each scanning only its own lanes. + MGARDX_EXEC int SubgroupExclScan(SubGroup &sg, int logical_lane, + int seg_base, int v, int &total) { + int incl = v; + for (int d = 1; d < (int)RANS_NLANES; d <<= 1) { + int src = (logical_lane - d < 0) ? (seg_base + logical_lane) + : (seg_base + logical_lane - d); + int t = sg.shfl(incl, src); + if (logical_lane >= d) { + incl += t; + } + } + total = sg.shfl(incl, seg_base + (int)RANS_NLANES - 1); + return incl - v; + } + + MGARDX_EXEC void Operation1() { + SubGroup sg; + constexpr int W = + SubGroup::size(); // 32 (warp) or 64 (wavefront) + constexpr int SUBBLOCKS = + W / (int)RANS_NLANES; // logical blocks per subgroup + int lane_in_sg = sg.lane(); // 0..W-1 + int seg = lane_in_sg / (int)RANS_NLANES; // which logical block + int logical_lane = lane_in_sg % (int)RANS_NLANES; + int seg_base = seg * (int)RANS_NLANES; + + IDX gtid = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + IDX subgroup_idx = gtid / (IDX)W; + IDX num_subgroups = (FunctorBase::GetGridDimX() * + FunctorBase::GetBlockDimX()) / + (IDX)W; + + // The loop bound is uniform across the whole subgroup so every lane + // iterates together (required for the subgroup shuffles), even when the + // wavefront's several blocks have different lengths. + for (IDX sgi = subgroup_idx; sgi * (IDX)SUBBLOCKS < num_blocks; + sgi += num_subgroups) { + IDX b = sgi * (IDX)SUBBLOCKS + (IDX)seg; + bool block_active = (b < num_blocks); + + IDX base = b * (RANS_NLANES * segment_size) + (IDX)logical_lane; + int count = 0; + if (block_active && base < n) { + IDX c = (n - 1 - base) / RANS_NLANES + 1; + count = (int)(c > segment_size ? segment_size : c); + } + // Subgroup-wide max symbol count -> uniform inner-loop bound. + int maxcount = count; + for (int d = 1; d < W; d <<= 1) { + int t = sg.shfl(maxcount, lane_in_sg ^ d); + maxcount = t > maxcount ? t : maxcount; + } + + uint32_t x = RANS_L; + IDX ptr = b * block_capacity + block_capacity; // per-block shared cursor + + for (int jj = maxcount; jj > 0; jj--) { + int j = jj - 1; + Byte buf[4]; + int nb = 0; + RansEncPacked e; + bool active = (j < count); // false for inactive-block lanes + if (active) { + IDX pos = base + (IDX)j * RANS_NLANES; + uint32_t s = (uint32_t)(*input(pos)); + e = *esym(s); + nb = RansEncCollect(x, e.x_max, buf); + } + int total; + int excl = SubgroupExclScan(sg, logical_lane, seg_base, nb, total); + for (int k = 0; k < nb; k++) { + *scratch(ptr - (IDX)excl - 1 - (IDX)k) = buf[k]; + } + ptr -= (IDX)total; + if (active) { + x = RansEncApply(x, e.rcp_freq, e.bias, (uint32_t)e.cmpl_freq, + (uint32_t)e.rcp_shift); + } + sg.sync(); + } + + // Flush NLANES states (4 LE bytes each), logical lane 0 at the high end. + // Inactive-block lanes contribute 0 to keep the scan in lockstep, but + // write nothing. + int flush_nb = block_active ? 4 : 0; + int total; + int excl = SubgroupExclScan(sg, logical_lane, seg_base, flush_nb, total); + if (block_active) { + IDX p0 = ptr - (IDX)excl - 4; + *scratch(p0 + 0) = (Byte)(x >> 0); + *scratch(p0 + 1) = (Byte)(x >> 8); + *scratch(p0 + 2) = (Byte)(x >> 16); + *scratch(p0 + 3) = (Byte)(x >> 24); + ptr -= (IDX)total; + if (logical_lane == 0) { + *seg_len(b) = (uint32_t)(b * block_capacity + block_capacity - ptr); + } + } + sg.sync(); + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SubArray<1, Q, DeviceType> input; + SIZE n; + SIZE segment_size; + SIZE num_blocks; + SubArray<1, RansEncPacked, DeviceType> esym; + IDX block_capacity; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; +}; + +template +class InterleavedEncodeWarpKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "rans interleaved warp encode"; + MGARDX_CONT InterleavedEncodeWarpKernel( + SubArray<1, Q, DeviceType> input, SIZE n, SIZE segment_size, + SIZE num_blocks, SubArray<1, RansEncPacked, DeviceType> esym, + IDX block_capacity, SubArray<1, Byte, DeviceType> scratch, + SubArray<1, uint32_t, DeviceType> seg_len) + : input(input), n(n), segment_size(segment_size), num_blocks(num_blocks), + esym(esym), block_capacity(block_capacity), scratch(scratch), + seg_len(seg_len) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = InterleavedEncodeWarpFunctor; + FunctorType functor(input, n, segment_size, num_blocks, esym, + block_capacity, scratch, seg_len); + SIZE tbx = 256; // 8 subgroups (warps) per block + SIZE total_threads = num_blocks * RANS_NLANES; + SIZE gridx = (total_threads - 1) / tbx + 1; + gridx = std::max((SIZE)DeviceRuntime::GetNumSMs(), gridx); + return Task(functor, 1, 1, gridx, 1, 1, tbx, 0, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, Q, DeviceType> input; + SIZE n; + SIZE segment_size; + SIZE num_blocks; + SubArray<1, RansEncPacked, DeviceType> esym; + IDX block_capacity; + SubArray<1, Byte, DeviceType> scratch; + SubArray<1, uint32_t, DeviceType> seg_len; +}; + +} // namespace rans +} // namespace mgard_x_160 +#endif diff --git a/include/mgard-x-160/Metadata/Metadata.hpp b/include/mgard-x-160/Metadata/Metadata.hpp new file mode 100644 index 000000000..4edcb50ba --- /dev/null +++ b/include/mgard-x-160/Metadata/Metadata.hpp @@ -0,0 +1,288 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../Config/Config.h" +#include "../RuntimeX/RuntimeX.h" +#include "../Utilities/Types.h" +#include "MGARDConfig.hpp" +#include +#include + +#ifndef MGARD_X_160_METADATA +#define MGARD_X_160_METADATA + +namespace mgard_x_160 { + +struct MetadataBase { + // about MGARD software + uint8_t software_version[3]; + uint8_t file_version[3]; + uint32_t metadata_size = 0; + uint32_t metadata_crc32 = 0; + + enum endiness_type etype; + + // about data + enum data_type dtype; + enum data_structure_type dstype; + uint64_t total_dims = 0; + std::vector shape; + char *nonuniform_coords_file; + std::vector> coords; + + enum decomposition_type decomposition; + uint32_t l_target = 0; + // MGARD 1.6.0 Config::reorder, recorded as the SHUFFLE preprocessor. + uint32_t reorder = 0; + // Config::num_global_refactoring_level the data was decomposed with. Not + // present in streams written before it was recorded (see InitializeConfig). + bool has_num_global_refactoring_level = false; + uint64_t num_global_refactoring_level = 0; + + // Hybrid (block-local + global) hierarchy, a.k.a. BlockMGARD. Only + // meaningful when `decomposition == decomposition_type::Hybrid`; these are + // the parameters the decompressor must reproduce exactly, so they travel in + // the header rather than being re-supplied by the caller. + uint64_t hybrid_num_local_levels = 0; + uint64_t hybrid_num_global_levels = 0; + // Edge length of the block-local transform block (compile-time 8 today). + uint64_t hybrid_local_block_size = 0; + bool hybrid_enable_roi = false; + // Resolved basis stored in a stream. Auto is a compression-side policy and + // is never serialized. + compression_projection_mode_type hybrid_projection_mode = + compression_projection_mode_type::Orthogonal; + // Level-0 per-block tolerances; deeper levels are re-derived on read. + std::vector hybrid_roi_tolerance_map; + // Number of level-0 blocks per dimension, used to validate the map above. + std::vector hybrid_roi_block_dimensions; + + bool domain_decomposed = false; + // Not in the header when the domain is not decomposed; it is still copied + // into the Config (InitializeConfig), so default it to the Config default + // rather than leaving it uninitialized. + enum domain_decomposition_type ddtype = domain_decomposition_type::MaxDim; + uint8_t domain_decomposed_dim; + uint64_t domain_decomposed_size; + + enum operation_type otype; + + // about MDR + enum bitplane_encoding_type betype; + uint64_t number_bitplanes; + + // about compression + enum error_bound_type ebtype; + double norm = 0; // optional + double tol = 0; + enum norm_type ntype; + double s = 0; // optional + + enum lossless_type ltype; + uint32_t huff_dict_size = 0; // optional (for Huffman) + uint32_t huff_block_size = 0; // optional (for Huffman) + uint32_t block_delta_block_size = 0; // optional (for BlockDelta) + + enum processor_type ptype; + + void InitializeConfig(Config &config); + void PrintSummary(); + + std::vector Serialize(); + void Deserialize(const std::vector &serialized_data); + +protected: + static uint64_t SerializePreambleSize(); + uint64_t DeserializeSize(std::vector::const_iterator &iter); +}; + +template struct Metadata : MetadataBase { + using Mem = MemoryManager; + template + void FillForCompression(enum error_bound_type ebtype, T tol, T s, T norm, + enum decomposition_type decomposition, + enum lossless_type ltype, uint32_t huff_dict_size, + uint32_t huff_block_size, std::vector shape, + bool domain_decomposed, + domain_decomposition_type ddtype, + uint8_t domain_decomposed_dim, + uint64_t domain_decomposed_size) { + + otype = operation_type::Compression; + if (std::is_same::value) { + this->ptype = processor_type::X_SERIAL; + } else if (std::is_same::value) { + this->ptype = processor_type::X_OPENMP; + } else if (std::is_same::value) { + this->ptype = processor_type::X_CUDA; + } else if (std::is_same::value) { + this->ptype = processor_type::X_HIP; + } else if (std::is_same::value) { + this->ptype = processor_type::X_SYCL; + } + this->ebtype = ebtype; + this->tol = (double)tol; + if (s == std::numeric_limits::infinity()) { + this->ntype = norm_type::L_Inf; + this->s = (double)s; + } else { + this->ntype = norm_type::L_2; + this->s = (double)s; + } + this->norm = norm; + this->decomposition = decomposition; + this->ltype = ltype; + this->huff_dict_size = huff_dict_size; + this->huff_block_size = huff_block_size; + this->dtype = + std::is_same::value ? data_type::Double : data_type::Float; + this->dstype = data_structure_type::Cartesian_Grid_Uniform; + this->total_dims = shape.size(); + this->shape = std::vector(this->total_dims); + for (int d = 0; d < this->total_dims; d++) { + this->shape[d] = (uint64_t)shape[d]; + } + this->domain_decomposed = domain_decomposed; + this->ddtype = ddtype; + this->domain_decomposed_dim = domain_decomposed_dim; + this->domain_decomposed_size = domain_decomposed_size; + } + + template + void + FillForCompression(enum error_bound_type ebtype, T tol, T s, T norm, + enum decomposition_type decomposition, + enum lossless_type ltype, uint32_t huff_dict_size, + uint32_t huff_block_size, std::vector shape, + bool domain_decomposed, domain_decomposition_type ddtype, + uint8_t domain_decomposed_dim, + uint64_t domain_decomposed_size, std::vector coords) { + FillForCompression(ebtype, tol, s, norm, decomposition, ltype, + huff_dict_size, huff_block_size, shape, + domain_decomposed, ddtype, domain_decomposed_dim, + domain_decomposed_size); + for (int d = 0; d < this->total_dims; d++) { + std::vector coord(shape[d]); + T *coord_h = new T[shape[d]]; + MemoryManager::Copy1D(coord_h, coords[d], shape[d]); + for (SIZE i = 0; i < shape[d]; i++) { + coord[i] = (double)coord_h[i]; + } + this->coords.push_back(coord); + delete[] coord_h; + } + this->dstype = data_structure_type::Cartesian_Grid_Non_Uniform; + } + + template + void FillForMDR(T norm, enum decomposition_type decomposition, + enum lossless_type ltype, uint32_t huff_dict_size, + uint32_t huff_block_size, std::vector shape, + bool domain_decomposed, domain_decomposition_type ddtype, + uint8_t domain_decomposed_dim, + uint64_t domain_decomposed_size, uint64_t number_bitplanes) { + otype = operation_type::MDR; + if (std::is_same::value) { + this->ptype = processor_type::X_SERIAL; + } else if (std::is_same::value) { + this->ptype = processor_type::X_OPENMP; + } else if (std::is_same::value) { + this->ptype = processor_type::X_CUDA; + } else if (std::is_same::value) { + this->ptype = processor_type::X_HIP; + } else if (std::is_same::value) { + this->ptype = processor_type::X_SYCL; + } + this->norm = norm; + this->decomposition = decomposition; + this->ltype = ltype; + this->huff_dict_size = huff_dict_size; + this->huff_block_size = huff_block_size; + this->dtype = + std::is_same::value ? data_type::Double : data_type::Float; + this->dstype = data_structure_type::Cartesian_Grid_Uniform; + this->total_dims = shape.size(); + this->shape = std::vector(this->total_dims); + for (int d = 0; d < this->total_dims; d++) { + this->shape[d] = (uint64_t)shape[d]; + } + this->domain_decomposed = domain_decomposed; + this->ddtype = ddtype; + this->domain_decomposed_dim = domain_decomposed_dim; + this->domain_decomposed_size = domain_decomposed_size; + this->betype = bitplane_encoding_type::GroupedBitplaneEncoding; + this->number_bitplanes = number_bitplanes; + } + + template + void FillForMDR(T norm, enum decomposition_type decomposition, + enum lossless_type ltype, uint32_t huff_dict_size, + uint32_t huff_block_size, std::vector shape, + bool domain_decomposed, domain_decomposition_type ddtype, + uint8_t domain_decomposed_dim, + uint64_t domain_decomposed_size, uint64_t number_bitplanes, + std::vector coords) { + FillForMDR(norm, decomposition, ltype, huff_dict_size, huff_block_size, + shape, domain_decomposed, ddtype, domain_decomposed_dim, + domain_decomposed_size, number_bitplanes); + for (int d = 0; d < this->total_dims; d++) { + std::vector coord(shape[d]); + T *coord_h = new T[shape[d]]; + MemoryManager::Copy1D(coord_h, coords[d], shape[d]); + for (SIZE i = 0; i < shape[d]; i++) { + coord[i] = (double)coord_h[i]; + } + this->coords.push_back(coord); + delete[] coord_h; + } + this->dstype = data_structure_type::Cartesian_Grid_Non_Uniform; + } + + SERIALIZED_TYPE *Serialize(uint32_t &total_size) { + std::vector data_h = MetadataBase::Serialize(); + total_size = data_h.size(); + SERIALIZED_TYPE *data_d; + Mem::Malloc1D(data_d, total_size, 0); + Mem::Copy1D(data_d, data_h.data(), total_size, 0); + DeviceRuntime::SyncQueue(0); + return data_d; + } + void Deserialize(const SERIALIZED_TYPE *serialized_data) { + // Do a partial deserialize to get the size of the buffer. + std::vector data_h(MetadataBase::SerializePreambleSize()); + Mem::Copy1D(data_h.data(), serialized_data, data_h.size(), 0); + DeviceRuntime::SyncQueue(0); + auto data_h_iter = data_h.cbegin(); + std::uint64_t total_size = MetadataBase::DeserializeSize(data_h_iter); + + data_h.resize(total_size); + Mem::Copy1D(data_h.data(), serialized_data, total_size, 0); + DeviceRuntime::SyncQueue(0); + MetadataBase::Deserialize(data_h); + } +}; + +bool verify(const void *compressed_data, size_t compressed_size); +enum data_type infer_data_type(const void *compressed_data, + size_t compressed_size); +std::vector infer_shape(const void *compressed_data, + size_t compressed_size); +enum data_structure_type infer_data_structure(const void *compressed_data, + size_t compressed_size); +template +std::vector infer_coords(const void *compressed_data, + size_t compressed_size); + +std::string infer_nonuniform_coords_file(const void *compressed_data, + size_t compressed_size); + +bool infer_domain_decomposed(const void *compressed_data, + size_t compressed_size); + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Quantization/HybridHierarchyLinearQuantization.hpp b/include/mgard-x-160/Quantization/HybridHierarchyLinearQuantization.hpp new file mode 100644 index 000000000..e522a45ce --- /dev/null +++ b/include/mgard-x-160/Quantization/HybridHierarchyLinearQuantization.hpp @@ -0,0 +1,830 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_HYBRID_HIERARCHY_LINEAR_QUANTIZATION_TEMPLATE +#define MGARD_X_160_HYBRID_HIERARCHY_LINEAR_QUANTIZATION_TEMPLATE + +#include + +#include "../RuntimeX/RuntimeX.h" +#include "LinearQuantization.hpp" +#include "LocalQuantization.hpp" +#include "QuantizationInterface.hpp" + +namespace mgard_x_160 { + +#define MGARDX_QUANTIZE 1 +#define MGARDX_DEQUANTIZE 2 + +template +class HybridHierarchyQuantizer + : public QuantizationInterface { +public: + HybridHierarchyQuantizer() : initialized(false) {} + + HybridHierarchyQuantizer(HybridHierarchy &hierarchy, + Config config) + : initialized(true), hierarchy(&hierarchy), config(config) { + hierarchy.check_config(config); + this->L = config.num_local_refactoring_level; + this->M = config.num_global_refactoring_level; + + if (this->L == 0 && this->M == 0) { + throw ProcessingException("Both L and M cannot be zero"); + } + } + + void Adapt(HybridHierarchy &hierarchy, Config config, + int queue_idx) { + hierarchy.check_config(config); + this->initialized = true; + this->hierarchy = &hierarchy; + this->config = config; + this->L = config.num_local_refactoring_level; + this->M = config.num_global_refactoring_level; + + if (this->L == 0 && this->M == 0) { + throw ProcessingException("Both L and M cannot be zero"); + } + + if (this->L > 0) { + local_quantizer.Adapt(hierarchy, config, queue_idx); + if (config.enable_roi) { + this->initial_block_tolerances = config.roi_tolerance_map; + ComputeLocalShapes(); + SetBlockTolerances(this->initial_block_tolerances, queue_idx); + } + } + + if (this->M > 0) { + global_hierarchy = &hierarchy.global_hierarchy(); + global_quantizer.Adapt(*global_hierarchy, config, queue_idx); + } else { + global_hierarchy = nullptr; + } + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + size_t size = 0; + return size; + } + + void SetOrthogonalProjection(bool enabled) { + orthogonal_projection = enabled; + local_quantizer.SetOrthogonalProjection(enabled); + } + + // Set block-level tolerances according to ROI table + void SetBlockTolerances(const std::vector &initial_block_tolerances, + int queue_idx) { + BuildROIToleranceMap(initial_block_tolerances); + // Upload once here so Quantize/Dequantize don't have to re-upload the + // tolerance map (which barely changes) on every call. + device_roi_tolerance_map.resize({(SIZE)roi_tolerance_map.size()}, + queue_idx); + device_roi_tolerance_map.load(roi_tolerance_map.data(), 0, queue_idx); + } + + // An s-norm bound is only defensible when the block-local stage is a single + // level with no global stage: only then is every block transformed + // independently, so the total squared error is a sum over blocks and the + // budget divides evenly in quadrature (see restrict_hybrid_config_for_s_norm, + // which makes the public compress path fall back to exactly this). + // + // Here we can only refuse: L and M fixed the buffer layout back at Adapt(), + // so a low-level caller cannot be silently re-configured mid-flight. + void CheckSNormConfiguration(T s) { + if (s == std::numeric_limits::infinity()) { + return; + } + if (this->L != 1 || this->M != 0) { + throw ProcessingException( + "hybrid with an s-norm bound requires exactly one block-local level " + "and no global stage (got L=" + + std::to_string(this->L) + ", M=" + std::to_string(this->M) + + "); the public compress path falls back to this automatically"); + } + if (config.enable_roi) { + throw ProcessingException("ROI mode is L-inf only"); + } + } + + // Tolerance for the coarsest layer when it has to be quantized on its own + // (M == 0, so there is no global stage to cover it). In ROI mode the + // coarsest layer has no tolerance of its own -- the map only describes the + // coefficient layers' block grids -- so use the tightest tolerance among + // the coarsest blocks, which is exactly what the global stage is given when + // M > 0 (see ErrorBudgetAllocation). + T CoarsestTolerance(T tol) { + return config.enable_roi ? (T)GetMinToleranceForGlobal() : tol; + } + + // Called only when this->M > 0 + T ErrorBudgetAllocation(T tol) { + T global_tol = tol; + + if (this->L > 0) { + if (this->config.enable_roi) { + global_tol = GetMinToleranceForGlobal(); + } + global_tol = global_tol / (1 << this->L); + } + return global_tol; + } + + void Quantize(SubArray original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray quantized_data, int queue_idx) {} + + void Dequantize(SubArray original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray quantized_data, int queue_idx) {} + + // Quantize the global (coarsest) region at the front of the decomposed + // array with the global quantizer. Shared between the unfused Quantize() + // path and the fused decompose+quantize path. Only valid when M > 0. + template + void QuantizeGlobalPart(SubArray<1, T, DeviceType> original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray<1, Q, DeviceType> quantized_data, + LosslessCompressorType &lossless, int queue_idx) { + T global_tol = ErrorBudgetAllocation(tol); + + std::vector global_shape = + global_hierarchy->level_shape(global_hierarchy->l_target()); + SubArray global_data_v(global_shape, + original_data.data()); + SubArray global_data_q(global_shape, + quantized_data.data()); + for (DIM d = 0; d < D; d++) { + global_data_v.setLd(d, global_shape[d]); + global_data_q.setLd(d, global_shape[d]); + } + global_data_v.project(D - 3, D - 2, D - 1); + global_data_q.project(D - 3, D - 2, D - 1); + global_quantizer.Quantize(global_data_v, ebtype, global_tol, s, norm, + global_data_q, lossless, queue_idx, + orthogonal_projection); + } + + // Dequantize the global (coarsest) region at the front of the decomposed + // array with the global quantizer. Shared between the unfused Dequantize() + // path and the fused dequantize+recompose path. Only valid when M > 0. + template + void DequantizeGlobalPart(SubArray<1, T, DeviceType> original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray<1, Q, DeviceType> quantized_data, + LosslessCompressorType &lossless, int queue_idx) { + T global_tol = ErrorBudgetAllocation(tol); + + std::vector global_shape = + global_hierarchy->level_shape(global_hierarchy->l_target()); + SubArray global_data_v(global_shape, + original_data.data()); + SubArray global_data_q(global_shape, + quantized_data.data()); + for (DIM d = 0; d < D; d++) { + global_data_v.setLd(d, global_shape[d]); + global_data_q.setLd(d, global_shape[d]); + } + global_data_v.project(D - 3, D - 2, D - 1); + global_data_q.project(D - 3, D - 2, D - 1); + global_quantizer.Dequantize(global_data_v, ebtype, global_tol, s, norm, + global_data_q, lossless, queue_idx, + orthogonal_projection); + } + + template + void Quantize(SubArray<1, T, DeviceType> original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray<1, Q, DeviceType> quantized_data, + LosslessCompressorType &lossless, int queue_idx) { + if (this->L == 0 && this->M == 0) { + throw ProcessingException("Both L and M cannot be zero"); + } + CheckSNormConfiguration(s); + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE global_q_size = 0; + + // Global quantization + if (this->M > 0) { + global_q_size = global_hierarchy->total_num_elems(); + QuantizeGlobalPart(original_data, ebtype, tol, s, norm, quantized_data, + lossless, queue_idx); + } + + // Local quantization + if (this->L > 0) { + SIZE local_length = original_data.shape(0) - global_q_size; + + SubArray<1, T, DeviceType> local_data_v({local_length}, + original_data(global_q_size)); + SubArray<1, Q, DeviceType> local_data_q({local_length}, + quantized_data(global_q_size)); + + // Switch between ROI and Non-ROI + if (config.enable_roi) { + local_quantizer.Quantize( + local_data_v, ebtype, 0.0, s, norm, local_data_q, + SubArray<1, double, DeviceType>(device_roi_tolerance_map), + level_offsets, level_block_counts, lossless, queue_idx); + } else { + local_quantizer.Quantize(local_data_v, ebtype, tol, s, norm, + local_data_q, lossless, queue_idx); + } + } + + // Coarsest layer, at the front of the decomposed array. When M > 0 the + // global stage above quantized it; when M == 0 the non-ROI local + // quantizer covers it as its layer 0, but the ROI local quantizer's + // per-block quantizers only span the L coefficient layers. Without this + // the coarsest layer -- (5/8)^D of the array -- would never be written, + // and only a same-process decompress would appear to work, by reading the + // values left in the cached decomposed buffer. + if (this->L > 0 && this->M == 0 && config.enable_roi) { + SIZE coarsest_size = local_quantizer.layer_len[0]; + SubArray<1, T, DeviceType> coarsest_v({coarsest_size}, + original_data.data()); + SubArray<1, Q, DeviceType> coarsest_q({coarsest_size}, + quantized_data.data()); + local_quantizer.QuantizeCoarsest(coarsest_v, coarsest_q, ebtype, + CoarsestTolerance(tol), s, norm, + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Hybrid Quantization", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + // Whether the fused decompose+quantize path can be used: it covers the + // local stage only (L > 0), relies on the 3D in-cache block kernel, and has + // the same L-inf-only constraint as the local quantizer. + bool CanFuseQuantize(T s) { + // s != inf is fine here: its weighting is a per-level scalar (the node + // volume is constant within a level), which is exactly what the fused + // kernels already take. ROI is not -- its per-block quantizers have no + // s-norm formulation, so it stays L-inf only. + if (config.enable_roi && s != std::numeric_limits::infinity()) { + return false; + } + return this->L > 0 && D >= 1 && D <= 3; + } + + // Which of the conditions above ruled the fused path out, for logging. Kept + // next to CanFuseQuantize so the two cannot drift apart. Returns an empty + // string when fusing is possible. + std::string WhyCannotFuseQuantize(T s) { + if (this->L == 0) { + return "no block-local levels"; + } + if (D > 3) { + return "fused kernel supports 1D, 2D and 3D only"; + } + if (config.enable_roi && s != std::numeric_limits::infinity()) { + return "ROI mode is L-inf only"; + } + return ""; + } + + // Fused decompose+quantization driver: the local levels are decomposed and + // quantized in one kernel per level (coefficients never round-trip through + // global memory as T), writing symbols directly to their final location in + // quantized_data. The coarsest region is then handled as in the unfused + // path: global decompose + global quantize when M > 0, otherwise a single + // coarsest-layer quantization -- in ROI mode too, since the per-block + // quantizers only span the coefficient layers. + template + void DecomposeQuantize(RefactorType &refactor, + SubArray data, + SubArray<1, T, DeviceType> decomposed_data, + SubArray<1, Q, DeviceType> quantized_data, + enum error_bound_type ebtype, T tol, T s, T norm, + LosslessCompressorType &lossless, int queue_idx) { + if (!CanFuseQuantize(s)) { + throw ProcessingException("DecomposeQuantize requires L > 0, D <= 3, and " + "(non-ROI or s == inf)"); + } + CheckSNormConfiguration(s); + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + SIZE huff_dict_size = config.huff_dict_size; + + if (config.enable_roi) { + // Per-level per-block reciprocal quantizers from the device-resident + // tolerance map, same math and block ordering as the ROI Quantize path + // (the fused kernel indexes them by block id, which matches the + // idx / hybrid_local_coeff_per_block(D) mapping of the unfused ROI + // kernel). + double C = orthogonal_projection ? (1 + std::pow(3, D)) : 1.0; + double norm_factor = + (ebtype == error_bound_type::REL) ? (double)norm : 1.0; + std::vector> device_quantizers(this->L); + std::vector> block_quantizers(this->L); + for (SIZE l = 0; l < this->L; ++l) { + SIZE level_offset = level_offsets[l]; + SIZE num_blocks = level_block_counts[l]; + double denom = std::pow(2, this->L - l + 1) * C; + device_quantizers[l] = Array<1, T, DeviceType>({num_blocks}, queue_idx); + DeviceLauncher::Execute( + ComputeROIQuantizersKernel( + SubArray<1, double, DeviceType>(device_roi_tolerance_map), + level_offset, num_blocks, norm_factor, denom, + /*reciprocal=*/true, + SubArray<1, T, DeviceType>(device_quantizers[l])), + queue_idx); + block_quantizers[l] = SubArray<1, T, DeviceType>(device_quantizers[l]); + } + refactor.local_refactor.DecomposeQuantize( + data, decomposed_data, quantized_data, std::vector(), + block_quantizers, prep_huffman, huff_dict_size, queue_idx, + orthogonal_projection); + } else { + std::vector level_quantizers = + local_quantizer.DecomposeLevelQuantizers(ebtype, tol, s, norm); + refactor.local_refactor.DecomposeQuantize( + data, decomposed_data, quantized_data, level_quantizers, + std::vector>(), prep_huffman, + huff_dict_size, queue_idx, orthogonal_projection); + } + + // Coarsest region (compacted at the front of decomposed_data by the + // fused local stage). + if (this->M > 0) { + refactor.DecomposeGlobal(decomposed_data, queue_idx, + orthogonal_projection); + QuantizeGlobalPart(decomposed_data, ebtype, tol, s, norm, quantized_data, + lossless, queue_idx); + } else { + // Also in ROI mode: the per-block quantizers only span the coefficient + // layers, so the coarsest layer needs one of its own here. + SIZE coarsest_size = local_quantizer.layer_len[0]; + SubArray<1, T, DeviceType> coarsest_v({coarsest_size}, + decomposed_data.data()); + SubArray<1, Q, DeviceType> coarsest_q({coarsest_size}, + quantized_data.data()); + local_quantizer.QuantizeCoarsest(coarsest_v, coarsest_q, ebtype, + CoarsestTolerance(tol), s, norm, + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Hybrid Decomposition+Quantization (fused)", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + // Fused dequantize+recomposition driver (inverse of DecomposeQuantize): + // the coarsest region is first reconstructed as in the unfused path + // (global dequantize + global recompose when M > 0, otherwise a single + // coarsest-layer dequantization, in ROI mode too), + // then the local levels are dequantized and recomposed in one kernel per + // level (coefficients never round-trip through global memory as T), + // writing the final level directly into the unpadded output. + template + void DequantizeRecompose(RefactorType &refactor, + SubArray data, + SubArray<1, T, DeviceType> decomposed_data, + SubArray<1, Q, DeviceType> quantized_data, + enum error_bound_type ebtype, T tol, T s, T norm, + LosslessCompressorType &lossless, int queue_idx) { + if (!CanFuseQuantize(s)) { + throw ProcessingException("DequantizeRecompose requires L > 0, D <= 3, " + "and (non-ROI or s == inf)"); + } + CheckSNormConfiguration(s); + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + SIZE huff_dict_size = config.huff_dict_size; + + // Coarsest region first: it is the input of the local recomposition. + if (this->M > 0) { + DequantizeGlobalPart(decomposed_data, ebtype, tol, s, norm, + quantized_data, lossless, queue_idx); + refactor.RecomposeGlobal(decomposed_data, queue_idx, + orthogonal_projection); + } else { + // Also in ROI mode -- see the matching branch in DecomposeQuantize. + SIZE coarsest_size = local_quantizer.layer_len[0]; + SubArray<1, T, DeviceType> coarsest_v({coarsest_size}, + decomposed_data.data()); + SubArray<1, Q, DeviceType> coarsest_q({coarsest_size}, + quantized_data.data()); + local_quantizer.DequantizeCoarsest(coarsest_v, coarsest_q, ebtype, + CoarsestTolerance(tol), s, norm, + queue_idx); + } + + if (config.enable_roi) { + // Per-level per-block dequantizers from the device-resident tolerance + // map, same math and block ordering as the ROI Dequantize path (the + // fused kernel indexes them by block id, which matches the + // idx / hybrid_local_coeff_per_block(D) mapping of the unfused ROI + // kernel). + double C = orthogonal_projection ? (1 + std::pow(3, D)) : 1.0; + double norm_factor = + (ebtype == error_bound_type::REL) ? (double)norm : 1.0; + std::vector> device_dequantizers(this->L); + std::vector> block_dequantizers(this->L); + for (SIZE l = 0; l < this->L; ++l) { + SIZE level_offset = level_offsets[l]; + SIZE num_blocks = level_block_counts[l]; + double denom = std::pow(2, this->L - l + 1) * C; + device_dequantizers[l] = + Array<1, T, DeviceType>({num_blocks}, queue_idx); + DeviceLauncher::Execute( + ComputeROIQuantizersKernel( + SubArray<1, double, DeviceType>(device_roi_tolerance_map), + level_offset, num_blocks, norm_factor, denom, + /*reciprocal=*/false, + SubArray<1, T, DeviceType>(device_dequantizers[l])), + queue_idx); + block_dequantizers[l] = + SubArray<1, T, DeviceType>(device_dequantizers[l]); + } + refactor.local_refactor.RecomposeDequantize( + data, decomposed_data, quantized_data, std::vector(), + block_dequantizers, prep_huffman, huff_dict_size, queue_idx, + orthogonal_projection); + } else { + std::vector level_dequantizers = + local_quantizer.RecomposeLevelDequantizers(ebtype, tol, s, norm); + refactor.local_refactor.RecomposeDequantize( + data, decomposed_data, quantized_data, level_dequantizers, + std::vector>(), prep_huffman, + huff_dict_size, queue_idx, orthogonal_projection); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Hybrid Dequantization+Recomposition (fused)", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + template + void Dequantize(SubArray<1, T, DeviceType> original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray<1, Q, DeviceType> quantized_data, + LosslessCompressorType &lossless, int queue_idx) { + if (this->L == 0 && this->M == 0) { + throw ProcessingException("Both L and M cannot be zero"); + } + CheckSNormConfiguration(s); + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE global_q_size = 0; + if (this->M > 0) { + global_q_size = global_hierarchy->total_num_elems(); + // log::info("Total Elems: " + std::to_string(global_q_size)); + } + + // Global dequantization + if (this->M > 0) { + DequantizeGlobalPart(original_data, ebtype, tol, s, norm, quantized_data, + lossless, queue_idx); + } + + // Local dequantization + if (this->L > 0) { + SIZE local_length = original_data.shape(0) - global_q_size; + SubArray<1, T, DeviceType> local_data_v({local_length}, + original_data(global_q_size)); + SubArray<1, Q, DeviceType> local_data_q({local_length}, + quantized_data(global_q_size)); + // Switch between ROI and Non-ROI + if (config.enable_roi) { + local_quantizer.Dequantize( + local_data_v, ebtype, 0.0, s, norm, local_data_q, + SubArray<1, double, DeviceType>(device_roi_tolerance_map), + level_offsets, level_block_counts, lossless, queue_idx); + } else { + local_quantizer.Dequantize(local_data_v, ebtype, tol, s, norm, + local_data_q, lossless, queue_idx); + } + } + + // Coarsest layer -- inverse of the corresponding block in Quantize(). + if (this->L > 0 && this->M == 0 && config.enable_roi) { + SIZE coarsest_size = local_quantizer.layer_len[0]; + SubArray<1, T, DeviceType> coarsest_v({coarsest_size}, + original_data.data()); + SubArray<1, Q, DeviceType> coarsest_q({coarsest_size}, + quantized_data.data()); + local_quantizer.DequantizeCoarsest(coarsest_v, coarsest_q, ebtype, + CoarsestTolerance(tol), s, norm, + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Hybrid Dequantization", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + // Compute fine and coarse shapes for all local levels + void ComputeLocalShapes() { + fine_shapes.clear(); + coarse_shapes.clear(); + + // Only compute if L > 0 + if (this->L == 0) { + return; + } + + for (SIZE l = 0; l < this->L; ++l) { + fine_shapes.push_back(hierarchy->local_fine_shape(l)); + coarse_shapes.push_back(hierarchy->local_coarse_shape(l)); + } + } + + // Build ROI tolerance map for all local levels (stored as 1D array) + void BuildROIToleranceMap(const std::vector &initial_tolerances) { + roi_tolerance_map.clear(); + level_offsets.clear(); + level_block_counts.clear(); + + // Only build ROI map if L > 0 + if (this->L == 0) { + return; + } + + // Level 0: use initial tolerances directly + level_offsets.push_back(0); + level_block_counts.push_back(initial_tolerances.size()); + roi_tolerance_map.insert(roi_tolerance_map.end(), + initial_tolerances.begin(), + initial_tolerances.end()); + + // Propagate tolerances through local levels + for (SIZE level = 1; level < this->L; ++level) { + SIZE prev_offset = level_offsets[level - 1]; + SIZE prev_count = level_block_counts[level - 1]; + + std::vector prev_level_tolerances( + roi_tolerance_map.begin() + prev_offset, + roi_tolerance_map.begin() + prev_offset + prev_count); + + std::vector next_level_tolerances = + PropagateTolerances(prev_level_tolerances, level - 1, level); + + level_offsets.push_back(roi_tolerance_map.size()); + level_block_counts.push_back(next_level_tolerances.size()); + roi_tolerance_map.insert(roi_tolerance_map.end(), + next_level_tolerances.begin(), + next_level_tolerances.end()); + } + } + + // Propagate error to next level + std::vector + PropagateTolerances(const std::vector ¤t_tolerances, + SIZE curr_level, SIZE next_level) { + // Get current and next level block dimensions from computed shapes + std::vector curr_blocks = GetBlockDimensions(curr_level); + std::vector next_blocks = GetBlockDimensions(next_level); + + SIZE next_size = 1; + for (DIM d = 0; d < D; ++d) { + next_size *= next_blocks[d]; + } + + // Initialize tolerance list for next block + std::vector next_tolerances(next_size, + std::numeric_limits::max()); + + // For each block in next level, find minimum tolerance from contributing + // blocks + for (SIZE idx = 0; idx < next_size; ++idx) { + std::vector next_coord = LinearToCoord(idx, next_blocks); + double min_tol = std::numeric_limits::max(); + + // Find all contributing blocks from current level + std::vector contributing_blocks = + GetContributingBlocks(next_coord, curr_blocks); + + for (SIZE contrib_idx : contributing_blocks) { + if (contrib_idx < current_tolerances.size()) { + min_tol = std::min(min_tol, current_tolerances[contrib_idx]); + } + } + + next_tolerances[idx] = min_tol; + } + + return next_tolerances; + } + + // Contributing tables: + // NEXT BLOCK CUR BLOCK + // 0 0,1 + // 1 1,2,3 + // 2 3,4 + // 3 4,5,6 + // 4 6,7 + // Get contributing block indices from previous level for a given next-level + // block + std::vector + GetContributingBlocks(const std::vector &next_coord, + const std::vector &curr_blocks) { + std::vector> contrib_per_dim(D); + + // For each dimension, apply the 5->8 inverse mapping + for (DIM d = 0; d < D; ++d) { + SIZE next_idx = next_coord[d]; + SIZE group = next_idx / 5; + SIZE offset = next_idx % 5; + SIZE base = group * 8; + + // Apply the propagation pattern + switch (offset) { + case 0: // min(0, 1) + contrib_per_dim[d] = {base + 0, base + 1}; + break; + case 1: // min(1, 2, 3) + contrib_per_dim[d] = {base + 1, base + 2, base + 3}; + break; + case 2: // min(3, 4) + contrib_per_dim[d] = {base + 3, base + 4}; + break; + case 3: // min(4, 5, 6) + contrib_per_dim[d] = {base + 4, base + 5, base + 6}; + break; + case 4: // min(6, 7) + contrib_per_dim[d] = {base + 6, base + 7}; + break; + } + + // Filter out-of-bounds indices + std::vector valid; + for (SIZE idx : contrib_per_dim[d]) { + if (idx < curr_blocks[d]) { + valid.push_back(idx); + } + } + contrib_per_dim[d] = valid; + } + + // Generate all combinations (Cartesian product) + return CartesianProduct(contrib_per_dim, curr_blocks); + } + + // Cartesian product of contributing indices across dimensions + std::vector + CartesianProduct(const std::vector> &indices_per_dim, + const std::vector &blocks) { + std::vector result; + std::vector coord(D); + CartesianProductHelper(indices_per_dim, blocks, 0, coord, result); + return result; + } + + void + CartesianProductHelper(const std::vector> &indices_per_dim, + const std::vector &blocks, DIM dim, + std::vector &coord, std::vector &result) { + if (dim == D) { + result.push_back(CoordToLinear(coord, blocks)); + return; + } + + for (SIZE idx : indices_per_dim[dim]) { + coord[dim] = idx; + CartesianProductHelper(indices_per_dim, blocks, dim + 1, coord, result); + } + } + + // Convert linear index to coordinate + std::vector LinearToCoord(SIZE linear_idx, + const std::vector &dims) { + std::vector coord(D); + for (int d = D - 1; d >= 0; --d) { + coord[d] = linear_idx % dims[d]; + linear_idx /= dims[d]; + } + return coord; + } + + // Convert coordinate to linear index + SIZE CoordToLinear(const std::vector &coord, + const std::vector &dims) { + SIZE linear = 0; + SIZE stride = 1; + for (int d = D - 1; d >= 0; --d) { + linear += coord[d] * stride; + stride *= dims[d]; + } + return linear; + } + + // Get block dimensions at a specific level + std::vector GetBlockDimensions(SIZE level) { + // Use the fine shape for this level (before decomposition) + const std::vector &fine_shape = fine_shapes[level]; + + // Calculate block size (8x8x8 for local decomposition) + const SIZE BLOCK_SIZE = 8; + + // Calculate number of blocks in each dimension + std::vector block_dims(D); + for (DIM d = 0; d < D; ++d) { + block_dims[d] = (fine_shape[d] + BLOCK_SIZE - 1) / BLOCK_SIZE; + } + + return block_dims; + } + + // Get minimum tolerance for global quantization + T GetMinToleranceForGlobal() { + if (!this->config.enable_roi) { + return std::numeric_limits::max(); + } + + // Get the last level + SIZE last_level_idx = level_offsets.size() - 1; + SIZE last_offset = level_offsets[last_level_idx]; + SIZE last_count = level_block_counts[last_level_idx]; + + double min_tol = std::numeric_limits::max(); + for (SIZE i = 0; i < last_count; ++i) { + min_tol = std::min(min_tol, roi_tolerance_map[last_offset + i]); + } + + return min_tol; + } + + bool initialized; + SIZE L; // Number of local levels + SIZE M; // Number of global levels + + HybridHierarchy *hierarchy; + // hierarchy->global_hierarchy() when M > 0. + Hierarchy *global_hierarchy = nullptr; + Config config; + bool orthogonal_projection = true; + + LocalQuantizer local_quantizer; + LinearQuantizer global_quantizer; + + // 1D ROI tolerance map: all levels stored consecutively + std::vector roi_tolerance_map; + + // Device-resident copy of roi_tolerance_map, uploaded once in + // SetBlockTolerances and reused by every Quantize/Dequantize call. + Array<1, double, DeviceType> device_roi_tolerance_map; + + // Offset for each level in the 1D tolerance map + std::vector level_offsets; + + // Number of blocks at each level + std::vector level_block_counts; + + // Fine and coarse shapes for each local level + std::vector> fine_shapes; + std::vector> coarse_shapes; + + std::vector initial_block_tolerances; +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Quantization/LinearQuantization.hpp b/include/mgard-x-160/Quantization/LinearQuantization.hpp new file mode 100644 index 000000000..4c5a89e0b --- /dev/null +++ b/include/mgard-x-160/Quantization/LinearQuantization.hpp @@ -0,0 +1,618 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_LINEAR_QUANTIZATION_TEMPLATE +#define MGARD_X_160_LINEAR_QUANTIZATION_TEMPLATE + +#include "../RuntimeX/RuntimeX.h" +#include "QuantizationInterface.hpp" + +namespace mgard_x_160 { + +#define MGARDX_QUANTIZE 1 +#define MGARDX_DEQUANTIZE 2 + +template +class LevelwiseLinearQuantizerNDFunctor : public Functor { +public: + MGARDX_CONT LevelwiseLinearQuantizerNDFunctor() {} + MGARDX_CONT LevelwiseLinearQuantizerNDFunctor( + SubArray<2, SIZE, DeviceType> level_ranges, + SubArray<2, int, DeviceType> level_marks, SIZE l_target, + SubArray<1, T, DeviceType> quantizers, + SubArray<1, T, DeviceType> level_volumes, + SubArray<2, SIZE, DeviceType> level_volume_offsets, bool calc_vol, + SubArray v, + SubArray quantized_v, bool prep_huffman, + SIZE dict_size, bool level_linearize = false, + SubArray<1, QUANTIZED_INT, DeviceType> *quantized_linearized_v = nullptr) + : level_ranges(level_ranges), level_marks(level_marks), + l_target(l_target), quantizers(quantizers), + level_volumes(level_volumes), + level_volume_offsets(level_volume_offsets), calc_vol(calc_vol), v(v), + quantized_v(quantized_v), prep_huffman(prep_huffman), + dict_size(dict_size), level_linearize(level_linearize), + quantized_linearized_v(quantized_linearized_v) { + Functor(); + } + + // MGARD 1.6.0 "reorder" layout (Config::reorder, recorded as the SHUFFLE + // preprocessor): coefficients stored level by level. Offset of this thread's + // coefficient within its level. Verbatim from MGARD 1.6.0. + MGARDX_EXEC SIZE calc_level_offset() { + // Use curr_region to encode region id to distinguish different regions + // curr_region of current level is always >=1, + // since curr_region=0 refers to the next coarser level + // most significant bit --> fastest dim + // least signigiciant bit --> slowest dim + for (int d = D - 1; d >= 0; d--) { + level = Math::Max(level, *level_marks(d, idx[d])); + } + + SIZE curr_region = 0; + for (int d = D - 1; d >= 0; d--) { + SIZE bit = level == *level_marks(d, idx[d]); + curr_region += bit << d; + } + + // region size + SIZE coarse_level_size[D]; + SIZE diff_level_size[D]; + for (int d = D - 1; d >= 0; d--) { + coarse_level_size[d] = *level_ranges(level, d); + diff_level_size[d] = + *level_ranges(level + 1, d) - *level_ranges(level, d); + } + + SIZE curr_region_dims[D]; + for (int d = D - 1; d >= 0; d--) { + // Use region id to decode dimension of this region + SIZE bit = (curr_region >> d) & 1u; + curr_region_dims[d] = bit ? diff_level_size[d] : coarse_level_size[d]; + } + + SIZE curr_region_size = 1; + for (int d = D - 1; d >= 0; d--) { + curr_region_size *= curr_region_dims[d]; + } + + // region offset + SIZE curr_region_offset = 0; + // prev_region start with 1 since that is the region id of the first + // region of current level + for (SIZE prev_region = 1; prev_region < curr_region; prev_region++) { + SIZE prev_region_size = 1; + for (int d = D - 1; d >= 0; d--) { + // Use region id to decode dimension of a previous region + SIZE bit = (prev_region >> d) & 1u; + // Calculate the num of elements of the previous region + prev_region_size *= bit ? diff_level_size[d] : coarse_level_size[d]; + } + curr_region_offset += prev_region_size; + } + + // printf("(%u %u): level: %u, curr_region: %u, curr_region_offset: %u\n", + // idx[0], idx[1], level, curr_region, curr_region_offset); + + // thread offset + SIZE curr_region_thread_idx[D]; + SIZE curr_thread_offset = 0; + SIZE coarse_level_offset = 0; + for (int d = D - 1; d >= 0; d--) { + SIZE bit = (curr_region >> d) & 1u; + curr_region_thread_idx[d] = bit ? idx[d] - coarse_level_size[d] : idx[d]; + } + + SIZE global_data_idx[D]; + for (int d = D - 1; d >= 0; d--) { + SIZE bit = (curr_region >> d) & 1u; + if (level == 0) { + global_data_idx[d] = curr_region_thread_idx[d]; + } else if (*level_ranges(level + 1, d) % 2 == 0 && + curr_region_thread_idx[d] == *level_ranges(level + 1, d) / 2) { + global_data_idx[d] = *level_ranges(level + 1, d) - 1; + } else { + global_data_idx[d] = curr_region_thread_idx[d] * 2 + bit; + } + } + + SIZE stride = 1; + for (int d = D - 1; d >= 0; d--) { + curr_thread_offset += global_data_idx[d] * stride; + stride *= *level_ranges(level + 1, d); + } + + stride = 1; + for (int d = D - 1; d >= 0; d--) { + if (global_data_idx[d] % 2 != 0 && + global_data_idx[d] != *level_ranges(level + 1, d) - 1) { + coarse_level_offset = 0; + } + if (global_data_idx[d]) { + coarse_level_offset += ((global_data_idx[d] - 1) / 2 + 1) * stride; + } + stride *= (*level_ranges(level + 1, d)) / 2 + 1; + } + + if (level == 0) + coarse_level_offset = 0; + + SIZE level_offset = curr_thread_offset - coarse_level_offset; + return level_offset; + } + + MGARDX_EXEC void Operation1() { + // determine global idx + SIZE firstD = div_roundup(v.shape(D - 1), F); + + SIZE bidx = FunctorBase::GetBlockIdX(); + idx[D - 1] = (bidx % firstD) * F + FunctorBase::GetThreadIdX(); + + bidx /= firstD; + if (D >= 2) { + idx[D - 2] = FunctorBase::GetBlockIdY() * + FunctorBase::GetBlockDimY() + + FunctorBase::GetThreadIdY(); + } + if (D >= 3) { + idx[D - 3] = FunctorBase::GetBlockIdZ() * + FunctorBase::GetBlockDimZ() + + FunctorBase::GetThreadIdZ(); + } + + for (int d = D - 4; d >= 0; d--) { + idx[d] = bidx % v.shape(d); + // idx0[d] = idx[d]; + bidx /= v.shape(d); + } + + level = 0; + + bool in_range = true; + for (int d = D - 1; d >= 0; d--) { + if (idx[d] >= v.shape(d)) + in_range = false; + } + + if (level >= 0 && level <= l_target && in_range) { + T t = v[idx]; + T volume = 1; + if (calc_vol) { + // Determine level + for (int d = D - 1; d >= 0; d--) { + level = Math::Max(level, *level_marks(d, idx[d])); + } + + // Determine volume + for (int d = D - 1; d >= 0; d--) { + volume *= *level_volumes(*level_volume_offsets(level, d) + idx[d]); + } + if (sizeof(T) == sizeof(double)) + volume = sqrt(volume); + else if (sizeof(T) == sizeof(float)) + volume = sqrtf(volume); + } + + T quantizer = *quantizers(level); + QUANTIZED_INT quantized_data; + + if constexpr (OP == MGARDX_QUANTIZE) { + if constexpr (sizeof(T) == sizeof(double)) { + quantized_data = copysign((T)0.5 + fabs(t * quantizer * volume), t); + } else if constexpr (sizeof(T) == sizeof(float)) { + quantized_data = copysign((T)0.5 + fabsf(t * quantizer * volume), t); + } + // Shift quantized values into the non-negative dictionary range the + // Huffman coder expects. Folding the shift here makes it free (the + // element is already being written) and removes a standalone full-array + // pass. Gated by prep_huffman so an alternative lossless backend that + // does not need outlier handling keeps the raw signed values. The value + // stays signed (QUANTIZED_INT); out-of-range entries are detected later + // by the outlier separation pass. + if (prep_huffman) { + quantized_data += dict_size / 2; + } + if (!level_linearize) { + quantized_v[idx] = quantized_data; + } else { + SIZE level_offset = calc_level_offset(); + *(quantized_linearized_v[level](level_offset)) = quantized_data; + } + } else if constexpr (OP == MGARDX_DEQUANTIZE) { + if (!level_linearize) { + quantized_data = quantized_v[idx]; + } else { + SIZE level_offset = calc_level_offset(); + quantized_data = *(quantized_linearized_v[level](level_offset)); + } + if (prep_huffman) { + quantized_data -= dict_size / 2; + } + v[idx] = (quantizer * volume) * (T)quantized_data; + } + } + } + + MGARDX_CONT size_t shared_memory_size() { + size_t size = 0; + return size; + } + +private: + IDX threadId; + SubArray<2, SIZE, DeviceType> level_ranges; + SubArray<2, int, DeviceType> level_marks; + SIZE l_target; + SubArray<1, T, DeviceType> quantizers; + SubArray<1, T, DeviceType> level_volumes; + SubArray<2, SIZE, DeviceType> level_volume_offsets; + SubArray v; + SubArray quantized_v; + bool calc_vol; + bool prep_huffman; + SIZE dict_size; + bool level_linearize; + SubArray<1, QUANTIZED_INT, DeviceType> *quantized_linearized_v; + SubArray<1, SIZE, DeviceType> shape; + + SIZE idx[D]; // thread global idx + SIZE idx0[D]; // block global idx + + int level; +}; + +template +class LevelwiseLinearQuantizerKernel : public Kernel { +public: + constexpr static DIM NumDim = D; + using DataType = T; + constexpr static std::string_view Name = "lwqzk"; + MGARDX_CONT + LevelwiseLinearQuantizerKernel( + SubArray<2, SIZE, DeviceType> level_ranges, + SubArray<2, int, DeviceType> level_marks, SIZE l_target, + SubArray<1, T, DeviceType> quantizers, + SubArray<1, T, DeviceType> level_volumes, + SubArray<2, SIZE, DeviceType> level_volume_offsets, bool calc_vol, + SubArray v, + SubArray quantized_v, bool prep_huffman, + SIZE dict_size, bool level_linearize = false, + SubArray<1, QUANTIZED_INT, DeviceType> *quantized_linearized_v = nullptr) + : level_ranges(level_ranges), level_marks(level_marks), + l_target(l_target), quantizers(quantizers), + level_volumes(level_volumes), + level_volume_offsets(level_volume_offsets), calc_vol(calc_vol), v(v), + quantized_v(quantized_v), prep_huffman(prep_huffman), + dict_size(dict_size), level_linearize(level_linearize), + quantized_linearized_v(quantized_linearized_v) {} + + template + MGARDX_CONT + Task> + GenTask(int queue_idx) { + using FunctorType = + LevelwiseLinearQuantizerNDFunctor; + + FunctorType functor(level_ranges, level_marks, l_target, quantizers, + level_volumes, level_volume_offsets, calc_vol, v, + quantized_v, prep_huffman, dict_size, level_linearize, + quantized_linearized_v); + + SIZE total_thread_z = v.shape(D - 3); + SIZE total_thread_y = v.shape(D - 2); + SIZE total_thread_x = v.shape(D - 1); + + SIZE tbx, tby, tbz, gridx, gridy, gridz; + size_t sm_size = functor.shared_memory_size(); + tbz = R; + tby = C; + tbx = F; + gridz = ceil((double)total_thread_z / tbz); + gridy = ceil((double)total_thread_y / tby); + gridx = ceil((double)total_thread_x / tbx); + for (DIM d = 3; d < D; d++) { + gridx *= v.shape(D - (d + 1)); + } + + // printf("%u %u %u %u %u %u %u %u %u\n", total_thread_x, total_thread_y, + // total_thread_z, tbx, tby, tbz, gridx, gridy, gridz); + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, sm_size, queue_idx, + std::string(Name)); + } + +private: + SubArray<2, SIZE, DeviceType> level_ranges; + SubArray<2, int, DeviceType> level_marks; + SIZE l_target; + SubArray<1, T, DeviceType> quantizers; + SubArray<1, T, DeviceType> level_volumes; + SubArray<2, SIZE, DeviceType> level_volume_offsets; + bool calc_vol; + SubArray v; + SubArray quantized_v; + bool prep_huffman; + SIZE dict_size; + bool level_linearize; + SubArray<1, QUANTIZED_INT, DeviceType> *quantized_linearized_v; + SubArray<1, SIZE, DeviceType> shape; +}; + +template +class LinearQuantizer : public QuantizationInterface { +public: + LinearQuantizer() : initialized(false) {} + + LinearQuantizer(Hierarchy &hierarchy, Config config) + : initialized(true), hierarchy(&hierarchy), config(config) { + quantizers_array = Array<1, T, DeviceType>({hierarchy.l_target() + 1}); + } + + void Adapt(Hierarchy &hierarchy, Config config, + int queue_idx) { + this->initialized = true; + this->hierarchy = &hierarchy; + this->config = config; + quantizers_array.resize({hierarchy.l_target() + 1}, queue_idx); + } + + void CalcQuantizers(size_t dof, T *quantizers, enum error_bound_type type, + T tol, T s, T norm, SIZE l_target, + enum decomposition_type decomposition, + bool orthogonal_projection, bool reciprocal) { + + double abs_tol = tol; + if (type == error_bound_type::REL) { + abs_tol *= norm; + } + abs_tol *= 2; + if (s == std::numeric_limits::infinity()) { + + // printf("quantizers: "); + for (int l = 0; l < l_target + 1; l++) { + if (decomposition == decomposition_type::MultiDim || + decomposition == decomposition_type::Hybrid) { + if (!orthogonal_projection) { + // Hierarchical basis (no mass-matrix correction): reconstruction is + // multilinear prolongation, a partition of unity, so a level-l + // quantization error propagates to any node with amplification + // <= 1. The max error is then bounded by sum over (l_target+1) + // levels of (step/2), i.e. the (1+3^D) orthogonal amplification + // drops out. + quantizers[l] = (abs_tol) / (l_target + 1); + } else { + // ben + quantizers[l] = (abs_tol) / ((l_target + 1) * (1 + std::pow(3, D))); + } + // xin + // quantizers[l] = (tol) / ((l_target + 1) * (1 + 3 * std::sqrt(3) / + // 4)); + // quantizers[l] = (0.5 * abs_tol) / std::pow(2, l_target - 1); + + } else if (decomposition == decomposition_type::SingleDim) { + // ken + quantizers[l] = + (abs_tol) / ((l_target + 1) * D * (1 + std::pow(3, 1))); + } + if (reciprocal) + quantizers[l] = 1.0f / quantizers[l]; + } + + } else { // s != inf + // xin - uniform + // T C2 = 1 + 3 * std::sqrt(3) / 4; + // T c = std::sqrt(std::pow(2, D - 2 * s)); + // T cc = (1 - c) / (1 - std::pow(c, l_target + 1)); + // T level_eb = cc * tol / C2; + // for (int l = 0; l < l_target + 1; l++) { + // quantizers[l] = level_eb; + // // T c = std::sqrt(std::pow(2, 2*s*l + D * (l_target - l))); + // level_eb *= c; + // if (reciprocal) + // quantizers[l] = 1.0f / quantizers[l]; + // } + + // ben - uniform + for (int l = 0; l < l_target + 1; l++) { + quantizers[l] = (abs_tol) / (std::exp2(s * l) * std::sqrt(dof)); + if (reciprocal) + quantizers[l] = 1.0f / quantizers[l]; + } + } + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + Hierarchy hierarchy; + hierarchy.EstimateMemoryFootprint(shape); + size_t size = 0; + size += sizeof(T) * (hierarchy.l_target() + 1); + return size; + } + + void Quantize(SubArray original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray quantized_data, int queue_idx) {} + + void Dequantize(SubArray original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray quantized_data, int queue_idx) {} + + // MGARD 1.6.0 "reorder" layout: one view per level into the quantized + // buffer, which then holds the levels back to back. Returns nullptr when the + // data is in the plain layout. Release with FreeLinearizedViews. + SubArray<1, Q, DeviceType> * + MakeLinearizedViews(SubArray quantized_data, + SubArray<2, SIZE, DeviceType> level_ranges_subarray, + SubArray<1, Q, DeviceType> *&views_host, int queue_idx) { + views_host = nullptr; + if (!config.reorder) { + return nullptr; + } + views_host = new SubArray<1, Q, DeviceType>[hierarchy->l_target() + 1]; + SIZE *ranges_h = level_ranges_subarray.dataHost(); + SIZE last_level_size = 0; + for (SIZE l = 0; l < hierarchy->l_target() + 1; l++) { + SIZE level_size = 1; + for (DIM d = 0; d < D; d++) { + level_size *= ranges_h[(l + 1) * D + d]; + } + views_host[l] = SubArray<1, Q, DeviceType>( + {level_size - last_level_size}, quantized_data(last_level_size)); + last_level_size = level_size; + } + SubArray<1, Q, DeviceType> *views = nullptr; + MemoryManager::Malloc1D(views, hierarchy->l_target() + 1, + queue_idx); + DeviceRuntime::SyncDevice(); + MemoryManager::Copy1D(views, views_host, + hierarchy->l_target() + 1, queue_idx); + DeviceRuntime::SyncDevice(); + return views; + } + + void FreeLinearizedViews(SubArray<1, Q, DeviceType> *views, + SubArray<1, Q, DeviceType> *views_host, + int queue_idx) { + if (views) { + DeviceRuntime::SyncQueue(queue_idx); + MemoryManager::Free(views); + } + delete[] views_host; + } + + template + void Quantize(SubArray original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray quantized_data, + LosslessCompressorType &lossless, int queue_idx, + bool orthogonal_projection = true) { + + // Toggle controlled from outside via the configured lossless backend: when + // a backend needs a non-negative Huffman dictionary we fold the dictionary + // shift into quantization; alternative backends that handle signed values + // directly leave the quantized data untouched (no shift / no outliers). + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + SIZE total_elems = hierarchy->total_num_elems(); + SubArray<2, SIZE, DeviceType> level_ranges_subarray( + hierarchy->level_ranges()); + SubArray<2, int, DeviceType> level_marks_subarray(hierarchy->level_marks()); + SubArray<1, T, DeviceType> level_volumes_subarray( + hierarchy->level_volumes(false)); + SubArray<2, SIZE, DeviceType> level_volume_offsets_subarray( + hierarchy->level_volume_offsets()); + SubArray<1, T, DeviceType> quantizers_subarray(quantizers_array); + T *quantizers = new T[hierarchy->l_target() + 1]; + CalcQuantizers(total_elems, quantizers, ebtype, tol, s, norm, + hierarchy->l_target(), config.decomposition, + orthogonal_projection, true); + MemoryManager::Copy1D(quantizers_subarray.data(), quantizers, + hierarchy->l_target() + 1, queue_idx); + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SubArray<1, Q, DeviceType> *linearized_host = nullptr; + SubArray<1, Q, DeviceType> *linearized = MakeLinearizedViews( + quantized_data, level_ranges_subarray, linearized_host, queue_idx); + bool calc_vol = + s != std::numeric_limits::infinity(); // m.ntype == norm_type::L_2; + DeviceLauncher::Execute( + LevelwiseLinearQuantizerKernel( + level_ranges_subarray, level_marks_subarray, hierarchy->l_target(), + quantizers_subarray, level_volumes_subarray, + level_volume_offsets_subarray, calc_vol, original_data, + quantized_data, prep_huffman, config.huff_dict_size, + (bool)config.reorder, linearized), + queue_idx); + FreeLinearizedViews(linearized, linearized_host, queue_idx); + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Quantization", hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + + delete[] quantizers; + } + + template + void Dequantize(SubArray original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray quantized_data, + LosslessCompressorType &lossless_compressor, int queue_idx, + bool orthogonal_projection = true) { + + SIZE total_elems = hierarchy->total_num_elems(); + SubArray<2, SIZE, DeviceType> level_ranges_subarray( + hierarchy->level_ranges()); + SubArray<2, int, DeviceType> level_marks_subarray(hierarchy->level_marks()); + SubArray<1, T, DeviceType> level_volumes_subarray( + hierarchy->level_volumes(true)); + SubArray<2, SIZE, DeviceType> level_volume_offsets_subarray( + hierarchy->level_volume_offsets()); + + // Must mirror the toggle used during Quantize so the dictionary shift is + // undone exactly when it was applied. + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + + SubArray<1, T, DeviceType> quantizers_subarray(quantizers_array); + T *quantizers = new T[hierarchy->l_target() + 1]; + CalcQuantizers(total_elems, quantizers, ebtype, tol, s, norm, + hierarchy->l_target(), config.decomposition, + orthogonal_projection, false); + MemoryManager::Copy1D(quantizers_subarray.data(), quantizers, + hierarchy->l_target() + 1, queue_idx); + DeviceRuntime::SyncQueue(queue_idx); + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SubArray<1, Q, DeviceType> *linearized_host = nullptr; + SubArray<1, Q, DeviceType> *linearized = MakeLinearizedViews( + quantized_data, level_ranges_subarray, linearized_host, queue_idx); + bool calc_vol = + s != std::numeric_limits::infinity(); // m.ntype == norm_type::L_2; + DeviceLauncher::Execute( + LevelwiseLinearQuantizerKernel( + level_ranges_subarray, level_marks_subarray, hierarchy->l_target(), + quantizers_subarray, level_volumes_subarray, + level_volume_offsets_subarray, calc_vol, original_data, + quantized_data, prep_huffman, config.huff_dict_size, + (bool)config.reorder, linearized), + queue_idx); + FreeLinearizedViews(linearized, linearized_host, queue_idx); + + DeviceRuntime::SyncQueue(queue_idx); + if (log::level & log::TIME) { + timer.end(); + timer.print("Dequantization", hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + + delete[] quantizers; + } + + bool initialized; + Hierarchy *hierarchy; + Config config; + Array<1, T, DeviceType> quantizers_array; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Quantization/LocalQuantization.hpp b/include/mgard-x-160/Quantization/LocalQuantization.hpp new file mode 100644 index 000000000..957668799 --- /dev/null +++ b/include/mgard-x-160/Quantization/LocalQuantization.hpp @@ -0,0 +1,787 @@ +#ifndef MGARD_X_160_LOCAL_QUANTIZATION_TEMPLATE +#define MGARD_X_160_LOCAL_QUANTIZATION_TEMPLATE + +#include "../Hierarchy/HybridHierarchy.h" +#include "../RuntimeX/RuntimeX.h" + +#include "QuantizationInterface.hpp" +#include +#include + +namespace mgard_x_160 { + +#define MGARDX_QUANTIZE 1 +#define MGARDX_DEQUANTIZE 2 + +// Non-ROI Version +template +class QuantizeLocalLevelFunctor : public Functor { +public: + MGARDX_CONT QuantizeLocalLevelFunctor() {} + MGARDX_CONT QuantizeLocalLevelFunctor(T quantizer, + SubArray<1, T, DeviceType> v, + SubArray<1, Q, DeviceType> quantized_v, + bool prep_huffman, SIZE dict_size) + : quantizer(quantizer), v(v), quantized_v(quantized_v), + prep_huffman(prep_huffman), dict_size(dict_size) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + if (idx < v.shape(0)) { + T t = *v(idx); + Q quantized_data; + T volume = 1; + + if constexpr (OP == MGARDX_QUANTIZE) { + if constexpr (sizeof(T) == sizeof(double)) { + quantized_data = copysign((T)0.5 + fabs(t * quantizer * volume), t); + } else if constexpr (sizeof(T) == sizeof(float)) { + quantized_data = copysign((T)0.5 + fabsf(t * quantizer * volume), t); + } + // Fold the Huffman dictionary shift into quantization (mirrors + // LevelwiseLinearQuantizerNDFunctor): the lossless stage expects + // values in the non-negative dictionary range and separates + // out-of-range entries as outliers. + if (prep_huffman) { + quantized_data += dict_size / 2; + } + *quantized_v(idx) = quantized_data; + } else if constexpr (OP == MGARDX_DEQUANTIZE) { + quantized_data = *quantized_v(idx); + if (prep_huffman) { + quantized_data -= dict_size / 2; + } + *v(idx) = (quantizer * volume) * (T)quantized_data; + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SIZE idx; + T quantizer; + SubArray<1, T, DeviceType> v; + SubArray<1, Q, DeviceType> quantized_v; + bool prep_huffman; + SIZE dict_size; +}; + +template +class QuantizeLocalLevelKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lvl_qk"; + + MGARDX_CONT + QuantizeLocalLevelKernel(T quantizer, SubArray<1, T, DeviceType> v, + SubArray<1, Q, DeviceType> quantized_v, + bool prep_huffman, SIZE dict_size) + : quantizer(quantizer), v(v), quantized_v(quantized_v), + prep_huffman(prep_huffman), dict_size(dict_size) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = QuantizeLocalLevelFunctor; + FunctorType functor(quantizer, v, quantized_v, prep_huffman, dict_size); + + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (v.shape(0) + tbx - 1) / tbx; + SIZE gridy = 1, gridz = 1; + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, 0, queue_idx, + std::string(Name)); + } + +private: + T quantizer; + SubArray<1, T, DeviceType> v; + SubArray<1, Q, DeviceType> quantized_v; + bool prep_huffman; + SIZE dict_size; +}; + +// ROI Version +// +// The coefficient array is a row-major sequence of block-local blocks, each +// contributing hybrid_local_coeff_per_block(D) coefficients, so a +// coefficient's block index -- and therefore its ROI quantizer -- is +// idx / that count. The count is dimension dependent (3 in 1D, 39 in 2D, +// 387 in 3D), hence the D template parameter. +template +class QuantizeLocalLevelROIFunctor : public Functor { +public: + MGARDX_CONT QuantizeLocalLevelROIFunctor() {} + MGARDX_CONT QuantizeLocalLevelROIFunctor( + SubArray<1, T, DeviceType> quantizers, SubArray<1, T, DeviceType> v, + SubArray<1, Q, DeviceType> quantized_v, bool prep_huffman, SIZE dict_size) + : quantizers(quantizers), v(v), quantized_v(quantized_v), + prep_huffman(prep_huffman), dict_size(dict_size) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + if (idx < v.shape(0)) { + SIZE block_idx = idx / kCoeffPerBlock; + + T quantizer = *quantizers(block_idx); + + T t = *v(idx); + Q quantized_data; + T volume = 1; + + if constexpr (OP == MGARDX_QUANTIZE) { + if constexpr (sizeof(T) == sizeof(double)) { + quantized_data = copysign((T)0.5 + fabs(t * quantizer * volume), t); + } else if constexpr (sizeof(T) == sizeof(float)) { + quantized_data = copysign((T)0.5 + fabsf(t * quantizer * volume), t); + } + if (prep_huffman) { + quantized_data += dict_size / 2; + } + *quantized_v(idx) = quantized_data; + } else if constexpr (OP == MGARDX_DEQUANTIZE) { + quantized_data = *quantized_v(idx); + if (prep_huffman) { + quantized_data -= dict_size / 2; + } + *v(idx) = (quantizer * volume) * (T)quantized_data; + } + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + static constexpr SIZE kCoeffPerBlock = (SIZE)hybrid_local_coeff_per_block(D); + SIZE idx; + SubArray<1, T, DeviceType> quantizers; + SubArray<1, T, DeviceType> v; + SubArray<1, Q, DeviceType> quantized_v; + bool prep_huffman; + SIZE dict_size; +}; + +template +class QuantizeLocalLevelROIKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "lvl_qk_roi"; + + MGARDX_CONT + QuantizeLocalLevelROIKernel(SubArray<1, T, DeviceType> quantizers, + SubArray<1, T, DeviceType> v, + SubArray<1, Q, DeviceType> quantized_v, + bool prep_huffman, SIZE dict_size) + : quantizers(quantizers), v(v), quantized_v(quantized_v), + prep_huffman(prep_huffman), dict_size(dict_size) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = QuantizeLocalLevelROIFunctor; + FunctorType functor(quantizers, v, quantized_v, prep_huffman, dict_size); + + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (v.shape(0) + tbx - 1) / tbx; + SIZE gridy = 1, gridz = 1; + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, 0, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, T, DeviceType> quantizers; + SubArray<1, T, DeviceType> v; + SubArray<1, Q, DeviceType> quantized_v; + bool prep_huffman; + SIZE dict_size; +}; + +// Computes per-block ROI quantizers directly on device from a device-resident +// tolerance map, avoiding a host-side loop plus a per-call H2D transfer of the +// result (the tolerance map itself is uploaded once, not on every call). +template +class ComputeROIQuantizersFunctor : public Functor { +public: + MGARDX_CONT ComputeROIQuantizersFunctor() {} + MGARDX_CONT + ComputeROIQuantizersFunctor(SubArray<1, double, DeviceType> tolerance_map, + SIZE level_offset, SIZE num_blocks, + double norm_factor, double denom, bool reciprocal, + SubArray<1, T, DeviceType> quantizers) + : tolerance_map(tolerance_map), level_offset(level_offset), + num_blocks(num_blocks), norm_factor(norm_factor), denom(denom), + reciprocal(reciprocal), quantizers(quantizers) { + Functor(); + } + + MGARDX_EXEC void Operation1() { + idx = FunctorBase::GetBlockIdX() * + FunctorBase::GetBlockDimX() + + FunctorBase::GetThreadIdX(); + + if (idx < num_blocks) { + double block_tol = *tolerance_map(level_offset + idx) * norm_factor * 2; + double block_quantizer = block_tol / denom; + *quantizers(idx) = + reciprocal ? (T)(1.0 / block_quantizer) : (T)block_quantizer; + } + } + + MGARDX_CONT size_t shared_memory_size() { return 0; } + +private: + SIZE idx; + SubArray<1, double, DeviceType> tolerance_map; + SIZE level_offset; + SIZE num_blocks; + double norm_factor; + double denom; + bool reciprocal; + SubArray<1, T, DeviceType> quantizers; +}; + +template +class ComputeROIQuantizersKernel : public Kernel { +public: + constexpr static bool EnableAutoTuning() { return false; } + constexpr static std::string_view Name = "roi_qcalc"; + + MGARDX_CONT + ComputeROIQuantizersKernel(SubArray<1, double, DeviceType> tolerance_map, + SIZE level_offset, SIZE num_blocks, + double norm_factor, double denom, bool reciprocal, + SubArray<1, T, DeviceType> quantizers) + : tolerance_map(tolerance_map), level_offset(level_offset), + num_blocks(num_blocks), norm_factor(norm_factor), denom(denom), + reciprocal(reciprocal), quantizers(quantizers) {} + + MGARDX_CONT Task> + GenTask(int queue_idx) { + using FunctorType = ComputeROIQuantizersFunctor; + FunctorType functor(tolerance_map, level_offset, num_blocks, norm_factor, + denom, reciprocal, quantizers); + + SIZE tbx = 256, tby = 1, tbz = 1; + SIZE gridx = (num_blocks + tbx - 1) / tbx; + SIZE gridy = 1, gridz = 1; + + return Task(functor, gridz, gridy, gridx, tbz, tby, tbx, 0, queue_idx, + std::string(Name)); + } + +private: + SubArray<1, double, DeviceType> tolerance_map; + SIZE level_offset; + SIZE num_blocks; + double norm_factor; + double denom; + bool reciprocal; + SubArray<1, T, DeviceType> quantizers; +}; + +template +class LocalQuantizer : public QuantizationInterface { +public: + LocalQuantizer() : initialized(false) {} + LocalQuantizer(HybridHierarchy &hierarchy, Config config) + : initialized(true), hierarchy(&hierarchy), config(config) { + hierarchy.check_config(config); + this->L = config.num_local_refactoring_level; + this->M = config.num_global_refactoring_level; + compute_local_ranges(); + prepare_layers(); + } + + // Add logic to determine if roi or not + void Adapt(HybridHierarchy &hierarchy, Config config, + int queue_idx) { + hierarchy.check_config(config); + this->initialized = true; + this->hierarchy = &hierarchy; + this->config = config; + this->L = config.num_local_refactoring_level; + this->M = config.num_global_refactoring_level; + compute_local_ranges(); + prepare_layers(); + } + + static size_t EstimateMemoryFootprint(std::vector shape) { + size_t size = 0; + return size; + } + + void SetOrthogonalProjection(bool enabled) { + orthogonal_projection = enabled; + } + + void compute_local_ranges() { + coarse_shape = hierarchy->input_shape(); + + fine_num_elems.clear(); + coarse_num_elems.clear(); + local_coeff_size.clear(); + fine_shapes.clear(); + coarse_shapes.clear(); + + for (int l = 0; l < this->L; ++l) { + SIZE last_level_size = 1, curr_level_size = 1; + std::vector fine_shape = hierarchy->local_fine_shape(l); + coarse_shape = hierarchy->local_coarse_shape(l); + for (DIM d = 0; d < D; ++d) { + last_level_size *= fine_shape[d]; + curr_level_size *= coarse_shape[d]; + } + fine_num_elems.push_back(last_level_size); + coarse_num_elems.push_back(curr_level_size); + local_coeff_size.push_back(last_level_size - curr_level_size); + fine_shapes.push_back(fine_shape); + coarse_shapes.push_back(coarse_shape); + } + } + + void prepare_layers() { + if (this->L == 0) { + layer_len.clear(); + layer_off.clear(); + return; + } + + layer_len.assign(this->L + 1, 0); + layer_off.assign(this->L + 1, 0); + + // The length of coarsest layer (level 0) + layer_len[0] = coarse_num_elems[this->L - 1]; + layer_off[0] = 0; + + SIZE accum = layer_len[0]; + + for (SIZE l = 1; l <= this->L; ++l) { + layer_len[l] = local_coeff_size[this->L - l]; + layer_off[l] = accum; + accum += layer_len[l]; + } + } + + // Calculate quantizers between levels(Used in Non-ROI) + void CalcQuantizers(size_t dof, T *quantizers, enum error_bound_type type, + T tol, T s, T norm, SIZE l_target, + enum decomposition_type decomposition, bool reciprocal) { + double abs_tol = tol; + if (type == error_bound_type::REL) { + abs_tol *= norm; + } + abs_tol *= 2; + + if (s == std::numeric_limits::infinity()) { + double C = orthogonal_projection ? (1 + std::pow(3, D)) : 1.0; + + for (int l = 0; l <= l_target; l++) { + // Modified here + quantizers[l] = (abs_tol) / (std::pow(2, l + 1) * C); + + if (reciprocal) { + quantizers[l] = 1.0f / quantizers[l]; + } + } + } else { + // s-norm (L2 family), non-ROI only. + // + // Same per-level law as LinearQuantizer's s != inf branch, + // step_l = abs_tol / (2^(s*l) * sqrt(dof)), + // with l counted from the coarsest layer, matching layer_len/layer_off. + // + // LinearQuantizer additionally scales each value by its node volume, + // reading a per-node level_volumes array. We cannot: the block-local + // quantize kernels take one scalar per layer and have no volume input. + // We do not need to. Hierarchy::calc_volume fills that array with + // total_dist / (dof - 1) -- the *average* spacing, identical for every + // node of a level -- so the per-node volume is a per-level constant and + // folds into the scalar exactly. + // + // Sign convention follows the kernels: quantize multiplies by the + // reciprocal step and by the volume, dequantize multiplies by the step + // and by the reciprocal volume, so the pair round-trips. + for (int l = 0; l <= l_target; l++) { + double step = + abs_tol / (std::exp2((double)s * l) * std::sqrt((double)dof)); + double volume = LayerVolume(l); + quantizers[l] = reciprocal ? (volume / step) : (step / volume); + } + } + } + + // sqrt of the cell volume of the grid that layer `l` lives on, matching + // LevelwiseLinearQuantizerFunctor's `volume = sqrt(prod_d spacing_d)`. + // + // Layer 0 is the coarsest data (grid coarse_shapes[L-1]); layer l >= 1 holds + // the coefficients dropped by block-local level L - l, which live on that + // level's fine grid. Coordinates are treated as normalized to a unit extent + // per dimension, which is what the block-local transform already assumes -- + // its mass/tridiagonal weights are built from a fixed uniform spacing. + double LayerVolume(int l) { + if (this->L == 0 || fine_shapes.empty()) { + return 1.0; + } + const std::vector &shape = + (l == 0) ? coarse_shapes[this->L - 1] + : fine_shapes[this->L - std::min(l, this->L)]; + double volume = 1.0; + for (DIM d = 0; d < D; d++) { + SIZE n = shape[d]; + volume *= (n > 1) ? (1.0 / (double)(n - 1)) : 1.0; + } + return std::sqrt(volume); + } + + // Reciprocal quantizers indexed by decompose level (level 0 = finest + // coefficients) for the fused decompose+quantize path. Decompose level l + // corresponds to non-ROI layer L - l, i.e. quantizer index L - l. + std::vector DecomposeLevelQuantizers(enum error_bound_type ebtype, T tol, + T s, T norm) { + std::vector quantizers(this->L + 1); + CalcQuantizers(hierarchy->total_num_elems(), quantizers.data(), ebtype, tol, + s, norm, this->L, config.decomposition, true); + std::vector level_quantizers(this->L); + for (SIZE l = 0; l < this->L; l++) { + level_quantizers[l] = quantizers[this->L - l]; + } + return level_quantizers; + } + + // Non-reciprocal dequantizers indexed by decompose level (level 0 = finest + // coefficients) for the fused dequantize+recompose path. Decompose level l + // corresponds to non-ROI layer L - l, i.e. quantizer index L - l. + std::vector RecomposeLevelDequantizers(enum error_bound_type ebtype, T tol, + T s, T norm) { + std::vector quantizers(this->L + 1); + CalcQuantizers(hierarchy->total_num_elems(), quantizers.data(), ebtype, tol, + s, norm, this->L, config.decomposition, false); + std::vector level_dequantizers(this->L); + for (SIZE l = 0; l < this->L; l++) { + level_dequantizers[l] = quantizers[this->L - l]; + } + return level_dequantizers; + } + + // Dequantize only the coarsest layer (layer 0). Used by the fused + // dequantize+recompose path when there is no global stage; the coefficient + // layers are dequantized inside the recompose kernels. + void DequantizeCoarsest(SubArray<1, T, DeviceType> v, + SubArray<1, Q, DeviceType> quantized_v, + enum error_bound_type ebtype, T tol, T s, T norm, + int queue_idx) { + std::vector quantizers(this->L + 1); + CalcQuantizers(hierarchy->total_num_elems(), quantizers.data(), ebtype, tol, + s, norm, this->L, config.decomposition, false); + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + DeviceLauncher::Execute( + QuantizeLocalLevelKernel( + quantizers[0], v, quantized_v, prep_huffman, config.huff_dict_size), + queue_idx); + } + + // Quantize only the coarsest layer (layer 0). Used by the fused + // decompose+quantize path when there is no global stage; the coefficient + // layers have already been quantized inside the decompose kernels. + void QuantizeCoarsest(SubArray<1, T, DeviceType> v, + SubArray<1, Q, DeviceType> quantized_v, + enum error_bound_type ebtype, T tol, T s, T norm, + int queue_idx) { + std::vector quantizers(this->L + 1); + CalcQuantizers(hierarchy->total_num_elems(), quantizers.data(), ebtype, tol, + s, norm, this->L, config.decomposition, true); + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + DeviceLauncher::Execute( + QuantizeLocalLevelKernel( + quantizers[0], v, quantized_v, prep_huffman, config.huff_dict_size), + queue_idx); + } + + void Quantize(SubArray original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray quantized_data, int queue_idx) {} + + void Dequantize(SubArray original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray quantized_data, int queue_idx) {} + + // Non-ROI + template + void Quantize(SubArray<1, T, DeviceType> original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray<1, Q, DeviceType> quantized_data, + LosslessCompressorType &lossless, int queue_idx) { + T *host_quantizers = new T[this->L + 1]; + CalcQuantizers(hierarchy->total_num_elems(), host_quantizers, ebtype, tol, + s, norm, this->L, config.decomposition, true); + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + SIZE huff_dict_size = config.huff_dict_size; + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE start_level = (this->M > 0) ? 1 : 0; + SIZE offset_adjustment = (this->M > 0) ? layer_off[1] : 0; + + for (SIZE l = start_level; l <= this->L; ++l) { + SIZE adjusted_off = layer_off[l] - offset_adjustment; + SubArray<1, T, DeviceType> v_in({layer_len[l]}, + original_data((IDX)adjusted_off)); + SubArray<1, Q, DeviceType> qv({layer_len[l]}, + quantized_data((IDX)adjusted_off)); + // Launch + T quantizer = host_quantizers[l]; + DeviceLauncher::Execute( + QuantizeLocalLevelKernel( + quantizer, v_in, qv, prep_huffman, huff_dict_size), + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Quantization", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + + delete[] host_quantizers; + } + + // Non-ROI + template + void Dequantize(SubArray<1, T, DeviceType> original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray<1, Q, DeviceType> quantized_data, + LosslessCompressorType &lossless, int queue_idx) { + T *host_quantizers = new T[this->L + 1]; + CalcQuantizers(hierarchy->total_num_elems(), host_quantizers, ebtype, tol, + s, norm, this->L, config.decomposition, false); + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + SIZE huff_dict_size = config.huff_dict_size; + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE start_level = (this->M > 0) ? 1 : 0; + SIZE offset_adjustment = (this->M > 0) ? layer_off[1] : 0; + + for (SIZE l = start_level; l <= this->L; ++l) { + SIZE adjusted_off = layer_off[l] - offset_adjustment; + SubArray<1, T, DeviceType> v_in({layer_len[l]}, + original_data((IDX)adjusted_off)); + SubArray<1, Q, DeviceType> qv({layer_len[l]}, + quantized_data((IDX)adjusted_off)); + // Launch + T quantizer = host_quantizers[l]; + DeviceLauncher::Execute( + QuantizeLocalLevelKernel( + quantizer, v_in, qv, prep_huffman, huff_dict_size), + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Dequantization", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + + delete[] host_quantizers; + } + + // With ROI + template + void Quantize(SubArray<1, T, DeviceType> original_data, + enum error_bound_type ebtype, double tol, T s, T norm, + SubArray<1, Q, DeviceType> quantized_data, + SubArray<1, double, DeviceType> device_roi_tolerance_map, + const std::vector &level_offsets, + const std::vector &level_block_counts, + LosslessCompressorType &lossless, int queue_idx) { + if (s != std::numeric_limits::infinity()) { + throw ProcessingException("Only L-inf supported"); + } + + double C = orthogonal_projection ? (1 + std::pow(3, D)) : 1.0; + double norm_factor = (ebtype == error_bound_type::REL) ? (double)norm : 1.0; + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + SIZE huff_dict_size = config.huff_dict_size; + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE accumulated_coeff_size = 0; + // Process Layer 1 to Layer L with ROI tolerances + for (SIZE l = 0; l < this->L; ++l) { + SIZE roi_level = l; + SIZE level_offset = level_offsets[roi_level]; + SIZE num_blocks = level_block_counts[roi_level]; + + // l=0 is finest coefficients (laid out at the end of the data array), + // which maps to non-ROI layer L. The correct exponent is (L - l + 1). + double denom = std::pow(2, this->L - l + 1) * C; + + // Compute per-block quantizers directly on device from the + // already-uploaded tolerance map (reciprocal for quantization), instead + // of recomputing on host and re-uploading every call. + Array<1, T, DeviceType> device_quantizers({num_blocks}, queue_idx); + DeviceLauncher::Execute( + ComputeROIQuantizersKernel( + device_roi_tolerance_map, level_offset, num_blocks, norm_factor, + denom, /*reciprocal=*/true, + SubArray<1, T, DeviceType>(device_quantizers)), + queue_idx); + + accumulated_coeff_size += local_coeff_size[l]; + SubArray<1, T, DeviceType> v_in( + {local_coeff_size[l]}, + original_data(original_data.shape(0) - accumulated_coeff_size)); + SubArray<1, Q, DeviceType> qv( + {local_coeff_size[l]}, + quantized_data(quantized_data.shape(0) - accumulated_coeff_size)); + + DeviceLauncher::Execute( + QuantizeLocalLevelROIKernel( + SubArray<1, T, DeviceType>(device_quantizers), v_in, qv, + prep_huffman, huff_dict_size), + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Quantization with ROI", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + // With ROI + template + void Dequantize(SubArray<1, T, DeviceType> original_data, + enum error_bound_type ebtype, double tol, T s, T norm, + SubArray<1, Q, DeviceType> quantized_data, + SubArray<1, double, DeviceType> device_roi_tolerance_map, + const std::vector &level_offsets, + const std::vector &level_block_counts, + LosslessCompressorType &lossless, int queue_idx) { + if (s != std::numeric_limits::infinity()) { + throw ProcessingException("Only L-inf supported"); + } + + double C = orthogonal_projection ? (1 + std::pow(3, D)) : 1.0; + double norm_factor = (ebtype == error_bound_type::REL) ? (double)norm : 1.0; + bool prep_huffman = config.lossless != lossless_type::CPU_Lossless && + config.lossless != lossless_type::BlockDelta && + config.lossless != lossless_type::LZ4; + SIZE huff_dict_size = config.huff_dict_size; + + Timer timer; + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.start(); + } + + SIZE accumulated_coeff_size = 0; + // Process Layer 1 to Layer L with ROI tolerances + for (SIZE l = 0; l < this->L; ++l) { + SIZE roi_level = l; + SIZE level_offset = level_offsets[roi_level]; + SIZE num_blocks = level_block_counts[roi_level]; + + // l=0 is finest coefficients (laid out at the end of the data array), + // which maps to non-ROI layer L. The correct exponent is (L - l + 1). + double denom = std::pow(2, this->L - l + 1) * C; + + // Compute per-block quantizers directly on device from the + // already-uploaded tolerance map (no reciprocal for dequantization). + Array<1, T, DeviceType> device_quantizers({num_blocks}, queue_idx); + DeviceLauncher::Execute( + ComputeROIQuantizersKernel( + device_roi_tolerance_map, level_offset, num_blocks, norm_factor, + denom, /*reciprocal=*/false, + SubArray<1, T, DeviceType>(device_quantizers)), + queue_idx); + + accumulated_coeff_size += local_coeff_size[l]; + SubArray<1, T, DeviceType> v_in( + {local_coeff_size[l]}, + original_data(original_data.shape(0) - accumulated_coeff_size)); + SubArray<1, Q, DeviceType> qv( + {local_coeff_size[l]}, + quantized_data(quantized_data.shape(0) - accumulated_coeff_size)); + + DeviceLauncher::Execute( + QuantizeLocalLevelROIKernel( + SubArray<1, T, DeviceType>(device_quantizers), v_in, qv, + prep_huffman, huff_dict_size), + queue_idx); + } + + if (log::level & log::TIME) { + DeviceRuntime::SyncQueue(queue_idx); + timer.end(); + timer.print("Local Dequantization with ROI", + hierarchy->total_num_elems() * sizeof(T)); + timer.clear(); + } + } + + bool initialized; + SIZE L; + SIZE M; + HybridHierarchy *hierarchy; + Config config; + bool orthogonal_projection = true; + + // For Non-ROI + std::vector layer_len; + std::vector layer_off; + + // For ROI + std::vector tol_table; + + std::vector fine_num_elems; + std::vector coarse_num_elems; + std::vector local_coeff_size; + std::vector coarse_shape; + // Grid each block-local level lives on, needed by the s != inf quantizer to + // weight a layer by its node spacing. + std::vector> fine_shapes; + std::vector> coarse_shapes; +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Quantization/QuantizationInterface.hpp b/include/mgard-x-160/Quantization/QuantizationInterface.hpp new file mode 100644 index 000000000..52aef9b16 --- /dev/null +++ b/include/mgard-x-160/Quantization/QuantizationInterface.hpp @@ -0,0 +1,25 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_QUANTIZATION_INTERFACE_HPP +#define MGARD_X_160_QUANTIZATION_INTERFACE_HPP +namespace mgard_x_160 { +template +class QuantizationInterface { + virtual void Quantize(SubArray original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray quantized_data, + int queue_idx) = 0; + + virtual void Dequantize(SubArray original_data, + enum error_bound_type ebtype, T tol, T s, T norm, + SubArray quantized_data, + int queue_idx) = 0; +}; +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/AutoTuners/AutoTuner.h b/include/mgard-x-160/RuntimeX/AutoTuners/AutoTuner.h new file mode 100644 index 000000000..becdc9723 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/AutoTuners/AutoTuner.h @@ -0,0 +1,270 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_AUTOTUNER_H +#define MGARD_X_160_AUTOTUNER_H + +#include +#include +#include +#include + +namespace mgard_x_160 { + +const int tBLK_ENCODE = 256; +const int tBLK_DEFLATE = 256; +const int tBLK_CANONICAL = 128; + +template MGARDX_CONT constexpr int TypeToIdx() { + if constexpr (std::is_same::value) { + return 0; + } else if constexpr (std::is_same::value) { + return 1; + } else { + return 0; + } +} + +template +std::string format(const std::string &format, Args... args) { + int size_s = std::snprintf(nullptr, 0, format.c_str(), args...) + + 1; // Extra space for '\0' + if (size_s <= 0) { + throw std::runtime_error("Error during formatting."); + } + auto size = static_cast(size_s); + auto buf = std::make_unique(size); + std::snprintf(buf.get(), size, format.c_str(), args...); + return std::string(buf.get(), + buf.get() + size - 1); // We don't want the '\0' inside +} + +template +MGARDX_CONT void FillAutoTunerTable(std::string kernel_name, int config) { + + int precision_idx = TypeToIdx(); + DIM dim_idx = D - 1; + + std::string device_type_string = ""; + if (std::is_same::value) { + device_type_string = "Serial"; + } else if (std::is_same::value) { + device_type_string = "Openmp"; + } else if (std::is_same::value) { + device_type_string = "Cuda"; + } else if (std::is_same::value) { + device_type_string = "Hip"; + } else if (std::is_same::value) { + device_type_string = "Sycl"; + } else { + throw std::runtime_error("invalid device_type in FillAutoTunerTable."); + } + + string curr_file_path = __FILE__; + string curr_dir_path = curr_file_path.substr(0, curr_file_path.rfind("/")); + + // std::cout << "********************curr_dir_path: " + curr_dir_path << "\n"; + + string intput_dir_path = curr_dir_path; + + // std::cout << "********************intput_dir_path: " + intput_dir_path << + // "\n"; + + std::string extension = ".h"; + + std::string input_file = + intput_dir_path + "/AutoTuner" + device_type_string + extension; + + // std::cout << "********************intput_file: " + input_file << "\n"; + + std::string regex1_string = "(static constexpr int " + kernel_name + + ".*\\{)((.*\\{.*\\}.*\n){" + + std::to_string(precision_idx) + "})(.*\\{(., ){" + + std::to_string(dim_idx) + "})(.)"; + + std::string replace1_string = "$1$2$4 " + std::to_string(config); + std::string regex2_string = ", (.)"; + std::string replace2_string = ", $1"; + std::string regex3_string = "\\{ "; + std::string replace3_string = "{"; + + std::ifstream t(input_file); + std::stringstream buffer; + buffer << t.rdbuf(); + // std::cout << "********************read file: " << buffer.str() << "\n"; + + std::regex e1(regex1_string, std::regex_constants::ECMAScript); + std::string new_string1 = + std::regex_replace(buffer.str(), e1, replace1_string); + // std::cout << "********************new file: " << new_string1 << "\n"; + + std::regex e2(regex2_string, std::regex_constants::ECMAScript); + std::string new_string2 = + std::regex_replace(new_string1, e2, replace2_string); + // std::cout << "********************new file2: " << new_string2 << "\n"; + + std::regex e3(regex3_string, std::regex_constants::ECMAScript); + std::string new_string3 = + std::regex_replace(new_string2, e3, replace3_string); + // std::cout << "********************new file3: " << new_string3 << "\n"; + + std::ofstream ofs(input_file, std::ofstream::trunc); + ofs << new_string3; + ofs.close(); +} + +template class AutoTuningTable { +public: + MGARDX_CONT + AutoTuningTable(){}; +}; + +template class AutoTuner { +public: + MGARDX_CONT + AutoTuner(){}; + + static AutoTuningTable autoTuningTable; + static bool ProfileKenrles; + static bool WriteToTable; +}; + +template void BeginAutoTuning() { + AutoTuner::ProfileKernels = true; + AutoTuner::WriteToTable = true; +} + +template void EndAutoTuning() { + AutoTuner::ProfileKernels = false; + AutoTuner::WriteToTable = false; +} + +struct ExecutionConfig { + constexpr ExecutionConfig(SIZE z, SIZE y, SIZE x) : z(z), y(y), x(x) {} + SIZE x, y, z; +}; + +template +MGARDX_CONT constexpr ExecutionConfig GetExecutionConfig(int config_idx) { + + constexpr int CONFIG_CANDIDATE[5][7][3] = {{{1, 1, 16}, + {1, 1, 32}, + {1, 1, 64}, + {1, 1, 128}, + {1, 1, 256}, + {1, 1, 512}, + {1, 1, 1024}}, + {{1, 2, 4}, + {1, 4, 4}, + {1, 4, 8}, + {1, 4, 16}, + {1, 4, 32}, + {1, 2, 64}, + {1, 2, 128}}, + {{2, 2, 2}, + {4, 4, 4}, + {4, 4, 8}, + {4, 4, 16}, + {4, 4, 32}, + {2, 2, 64}, + {2, 2, 128}}, + {{2, 2, 2}, + {4, 4, 4}, + {4, 4, 8}, + {4, 4, 16}, + {4, 4, 32}, + {2, 2, 64}, + {2, 2, 128}}, + {{2, 2, 2}, + {4, 4, 4}, + {4, 4, 8}, + {4, 4, 16}, + {4, 4, 32}, + {2, 2, 64}, + {2, 2, 128}}}; + ExecutionConfig config(CONFIG_CANDIDATE[D - 1][config_idx][0], + CONFIG_CANDIDATE[D - 1][config_idx][1], + CONFIG_CANDIDATE[D - 1][config_idx][2]); + return config; +} + +template +MGARDX_CONT constexpr ExecutionConfig +GetExecutionConfig(std::string_view functor_name) { + int precision_idx = TypeToIdx(); + DIM dim_idx = D - 1; + int config_idx = 0; + if (functor_name == "gpk_reo_3d") { + config_idx = + AutoTuningTable::gpk_reo_3d[precision_idx][dim_idx]; + } else if (functor_name == "gpk_rev_3d") { + config_idx = + AutoTuningTable::gpk_rev_3d[precision_idx][dim_idx]; + } else if (functor_name == "gpk_reo_nd") { + config_idx = + AutoTuningTable::gpk_reo_nd[precision_idx][dim_idx]; + } else if (functor_name == "gpk_rev_nd") { + config_idx = + AutoTuningTable::gpk_rev_nd[precision_idx][dim_idx]; + } else if (functor_name == "lpk1_3d") { + config_idx = AutoTuningTable::lpk1_3d[precision_idx][dim_idx]; + } else if (functor_name == "lpk2_3d") { + config_idx = AutoTuningTable::lpk2_3d[precision_idx][dim_idx]; + } else if (functor_name == "lpk3_3d") { + config_idx = AutoTuningTable::lpk3_3d[precision_idx][dim_idx]; + } else if (functor_name == "lpk1_nd") { + config_idx = AutoTuningTable::lpk1_nd[precision_idx][dim_idx]; + } else if (functor_name == "lpk2_nd") { + config_idx = AutoTuningTable::lpk2_nd[precision_idx][dim_idx]; + } else if (functor_name == "lpk3_nd") { + config_idx = AutoTuningTable::lpk3_nd[precision_idx][dim_idx]; + } else if (functor_name == "ipk1_3d") { + config_idx = AutoTuningTable::ipk1_3d[precision_idx][dim_idx]; + } else if (functor_name == "ipk2_3d") { + config_idx = AutoTuningTable::ipk2_3d[precision_idx][dim_idx]; + } else if (functor_name == "ipk3_3d") { + config_idx = AutoTuningTable::ipk3_3d[precision_idx][dim_idx]; + } else if (functor_name == "ipk1_nd") { + config_idx = AutoTuningTable::ipk1_nd[precision_idx][dim_idx]; + } else if (functor_name == "ipk2_nd") { + config_idx = AutoTuningTable::ipk2_nd[precision_idx][dim_idx]; + } else if (functor_name == "ipk3_nd") { + config_idx = AutoTuningTable::ipk3_nd[precision_idx][dim_idx]; + } else if (functor_name == "lwpk") { + config_idx = AutoTuningTable::lwpk[precision_idx][dim_idx]; + } else if (functor_name == "lwqzk") { + config_idx = AutoTuningTable::lwqzk[precision_idx][dim_idx]; + } else if (functor_name == "lwdqzk") { + config_idx = AutoTuningTable::lwdqzk[precision_idx][dim_idx]; + } else if (functor_name == "llk") { + config_idx = AutoTuningTable::llk[precision_idx][dim_idx]; + } else if (functor_name == "sdck") { + config_idx = AutoTuningTable::sdck[precision_idx][dim_idx]; + } else if (functor_name == "sdmtk") { + config_idx = AutoTuningTable::sdmtk[precision_idx][dim_idx]; + } else if (functor_name == "encode") { + config_idx = AutoTuningTable::encode[precision_idx][dim_idx]; + } else if (functor_name == "deflate") { + config_idx = AutoTuningTable::deflate[precision_idx][dim_idx]; + } else if (functor_name == "decode") { + config_idx = AutoTuningTable::decode[precision_idx][dim_idx]; + } else { + log::err("Wrong functor_name"); + config_idx = 0; + } + return GetExecutionConfig(config_idx); +} + +} // namespace mgard_x_160 + +#include "AutoTunerCuda.h" +#include "AutoTunerHip.h" +#include "AutoTunerOpenmp.h" +#include "AutoTunerSerial.h" +#include "AutoTunerSycl.h" + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerCuda.h b/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerCuda.h new file mode 100644 index 000000000..d18caa080 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerCuda.h @@ -0,0 +1,106 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_AUTOTUNER_CUDA_H +#define MGARD_X_160_AUTOTUNER_CUDA_H +// clang-format off +namespace mgard_x_160 { + +template <> class AutoTuningTable { +public: + static constexpr int num_types = 2; + static constexpr int num_dims = 9; + + static constexpr int gpk_reo_3d[num_types][num_dims] = {{2, 3, 5, 0, 0, 0, 2, 0, 0}, + {2, 3, 2, 0, 0, 0, 3, 0, 0}}; + + static constexpr int gpk_rev_3d[num_types][num_dims] = {{3, 5, 6, 3, 3, 5, 6, 0, 0}, + {2, 5, 3, 3, 4, 5, 6, 0, 0}}; + + static constexpr int gpk_reo_nd[num_types][num_dims] = {{5, 3, 5, 5, 5, 5, 5, 5, 5}, + {5, 3, 5, 5, 5, 5, 5, 5, 5}}; + + static constexpr int gpk_rev_nd[num_types][num_dims] = {{0, 5, 3, 4, 3, 0, 0, 0, 0}, + {0, 5, 3, 4, 5, 0, 0, 0, 0}}; + + static constexpr int lpk1_3d[num_types][num_dims] = {{3, 5, 3, 0, 1, 1, 4, 0, 0}, + {4, 3, 1, 1, 1, 1, 1, 0, 0}}; + + static constexpr int lpk2_3d[num_types][num_dims] = {{4, 4, 3, 4, 1, 3, 4, 0, 0}, + {0, 4, 2, 1, 1, 1, 1, 0, 0}}; + + static constexpr int lpk3_3d[num_types][num_dims] = {{1, 4, 2, 1, 1, 1, 4, 0, 0}, + {1, 0, 1, 1, 1, 1, 1, 0, 0}}; + + static constexpr int lpk1_nd[num_types][num_dims] = {{2, 0, 1, 1, 1, 0, 0, 0, 0}, + {0, 0, 1, 1, 1, 0, 0, 0, 0}}; + + static constexpr int lpk2_nd[num_types][num_dims] = {{2, 1, 3, 1, 1, 0, 0, 0, 0}, + {0, 2, 1, 1, 1, 0, 0, 0, 0}}; + + static constexpr int lpk3_nd[num_types][num_dims] = {{2, 3, 1, 1, 1, 0, 0, 0, 0}, + {0, 2, 1, 1, 2, 0, 0, 0, 0}}; + + static constexpr int ipk1_3d[num_types][num_dims] = {{6, 4, 4, 4, 5, 3, 4, 0, 0}, + {5, 3, 2, 3, 3, 1, 1, 0, 0}}; + + static constexpr int ipk2_3d[num_types][num_dims] = {{2, 3, 6, 2, 2, 2, 6, 0, 0}, + {3, 4, 5, 2, 2, 3, 4, 0, 0}}; + + static constexpr int ipk3_3d[num_types][num_dims] = {{2, 2, 6, 2, 2, 2, 6, 0, 0}, + {2, 2, 3, 2, 2, 2, 4, 0, 0}}; + + static constexpr int ipk1_nd[num_types][num_dims] = {{0, 2, 3, 2, 1, 0, 0, 0, 0}, + {0, 3, 3, 2, 1, 0, 0, 0, 0}}; + + static constexpr int ipk2_nd[num_types][num_dims] = {{0, 1, 2, 2, 2, 0, 0, 0, 0}, + {0, 2, 2, 2, 2, 0, 0, 0, 0}}; + + static constexpr int ipk3_nd[num_types][num_dims] = {{0, 2, 3, 2, 2, 0, 0, 0, 0}, + {0, 3, 4, 2, 4, 0, 0, 0, 0}}; + + static constexpr int lwpk[num_types][num_dims] = {{3, 5, 5, 5, 2, 1, 2, 0, 0}, + {3, 6, 4, 1, 2, 1, 1, 0, 0}}; + + static constexpr int lwqzk[num_types][num_dims] = {{4, 4, 3, 3, 3, 0, 2, 0, 0}, + {2, 3, 3, 3, 3, 0, 2, 0, 0}}; + + static constexpr int lwdqzk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int llk[num_types][num_dims] = {{3, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; + + static constexpr int sdck[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int sdmtk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int encode[num_types][num_dims] = {{4, 4, 4, 4, 4, 4, 4, 4, 4}, + {4, 4, 4, 4, 4, 4, 4, 4, 4}}; + + static constexpr int deflate[num_types][num_dims] = {{0, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; + + static constexpr int decode[num_types][num_dims] = {{0, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; + +}; + +template <> class AutoTuner { +public: + MGARDX_CONT + AutoTuner(){}; + static AutoTuningTable autoTuningTable; + static bool ProfileKernels; + static bool WriteToTable; +}; + +} // namespace mgard_x_160 +// clang-format on +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerHip.h b/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerHip.h new file mode 100644 index 000000000..09726b3ad --- /dev/null +++ b/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerHip.h @@ -0,0 +1,106 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_AUTOTUNER_HIP_H +#define MGARD_X_160_AUTOTUNER_HIP_H +// clang-format off +namespace mgard_x_160 { + +template <> class AutoTuningTable { +public: + static constexpr int num_types = 2; + static constexpr int num_dims = 9; + + static constexpr int gpk_reo_3d[num_types][num_dims] = {{2, 3, 5, 0, 0, 0, 2, 0, 0}, + {2, 3, 2, 0, 0, 0, 3, 0, 0}}; + + static constexpr int gpk_rev_3d[num_types][num_dims] = {{3, 5, 6, 3, 3, 5, 6, 0, 0}, + {2, 5, 3, 3, 4, 5, 6, 0, 0}}; + + static constexpr int gpk_reo_nd[num_types][num_dims] = {{5, 3, 5, 5, 5, 5, 5, 5, 5}, + {5, 3, 5, 5, 5, 5, 5, 5, 5}}; + + static constexpr int gpk_rev_nd[num_types][num_dims] = {{0, 5, 3, 4, 3, 0, 0, 0, 0}, + {0, 5, 3, 4, 5, 0, 0, 0, 0}}; + + static constexpr int lpk1_3d[num_types][num_dims] = {{3, 5, 3, 0, 1, 1, 4, 0, 0}, + {4, 3, 1, 1, 1, 1, 1, 0, 0}}; + + static constexpr int lpk2_3d[num_types][num_dims] = {{4, 4, 3, 4, 1, 3, 4, 0, 0}, + {0, 4, 2, 1, 1, 1, 1, 0, 0}}; + + static constexpr int lpk3_3d[num_types][num_dims] = {{1, 4, 2, 1, 1, 1, 4, 0, 0}, + {1, 0, 1, 1, 1, 1, 1, 0, 0}}; + + static constexpr int lpk1_nd[num_types][num_dims] = {{2, 0, 1, 1, 1, 0, 0, 0, 0}, + {0, 0, 1, 1, 1, 0, 0, 0, 0}}; + + static constexpr int lpk2_nd[num_types][num_dims] = {{2, 1, 3, 1, 1, 0, 0, 0, 0}, + {0, 2, 1, 1, 1, 0, 0, 0, 0}}; + + static constexpr int lpk3_nd[num_types][num_dims] = {{2, 3, 1, 1, 1, 0, 0, 0, 0}, + {0, 2, 1, 1, 2, 0, 0, 0, 0}}; + + static constexpr int ipk1_3d[num_types][num_dims] = {{6, 4, 4, 4, 5, 3, 4, 0, 0}, + {5, 3, 2, 3, 3, 1, 1, 0, 0}}; + + static constexpr int ipk2_3d[num_types][num_dims] = {{2, 3, 6, 2, 2, 2, 6, 0, 0}, + {3, 4, 5, 2, 2, 3, 4, 0, 0}}; + + static constexpr int ipk3_3d[num_types][num_dims] = {{2, 2, 6, 2, 2, 2, 6, 0, 0}, + {2, 2, 3, 2, 2, 2, 4, 0, 0}}; + + static constexpr int ipk1_nd[num_types][num_dims] = {{0, 2, 3, 2, 1, 0, 0, 0, 0}, + {0, 3, 3, 2, 1, 0, 0, 0, 0}}; + + static constexpr int ipk2_nd[num_types][num_dims] = {{0, 1, 2, 2, 2, 0, 0, 0, 0}, + {0, 2, 2, 2, 2, 0, 0, 0, 0}}; + + static constexpr int ipk3_nd[num_types][num_dims] = {{0, 2, 3, 2, 2, 0, 0, 0, 0}, + {0, 3, 4, 2, 4, 0, 0, 0, 0}}; + + static constexpr int lwpk[num_types][num_dims] = {{3, 5, 5, 5, 2, 1, 2, 0, 0}, + {3, 6, 4, 1, 2, 1, 1, 0, 0}}; + + static constexpr int lwqzk[num_types][num_dims] = {{4, 6, 5, 5, 4, 0, 2, 0, 0}, + {3, 4, 6, 5, 4, 0, 2, 0, 0}}; + + static constexpr int lwdqzk[num_types][num_dims] = {{4, 4, 3, 3, 3, 0, 2, 0, 0}, + {2, 3, 3, 3, 3, 0, 2, 0, 0}}; + + static constexpr int llk[num_types][num_dims] = {{3, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; + + static constexpr int sdck[num_types][num_dims] = {{3, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; + + static constexpr int sdmtk[num_types][num_dims] = {{3, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; + + static constexpr int encode[num_types][num_dims] = {{4, 4, 4, 4, 4, 4, 4, 4, 4}, + {4, 4, 4, 4, 4, 4, 4, 4, 4}}; + + static constexpr int deflate[num_types][num_dims] = {{1, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; + + static constexpr int decode[num_types][num_dims] = {{5, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; + +}; + +template <> class AutoTuner { +public: + MGARDX_CONT + AutoTuner(){}; + static AutoTuningTable autoTuningTable; + static bool ProfileKernels; + static bool WriteToTable; +}; + +} // namespace mgard_x_160 +// clang-format on +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerKokkos.h b/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerKokkos.h new file mode 100644 index 000000000..58d67c9cd --- /dev/null +++ b/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerKokkos.h @@ -0,0 +1,384 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_AUTOTUNER_KOKKOS_H +#define MGARD_X_160_AUTOTUNER_KOKKOS_H + +namespace mgard_x_160 { + +template <> class KernelConfigs { +public: + MGARDX_CONT + KernelConfigs(){}; +}; + +template <> class AutoTuningTable { +public: + MGARDX_CONT + AutoTuningTable() { + this->auto_tuning_cc = new int **[num_arch]; + this->auto_tuning_mr1 = new int **[num_arch]; + this->auto_tuning_mr2 = new int **[num_arch]; + this->auto_tuning_mr3 = new int **[num_arch]; + this->auto_tuning_ts1 = new int **[num_arch]; + this->auto_tuning_ts2 = new int **[num_arch]; + this->auto_tuning_ts3 = new int **[num_arch]; + for (int i = 0; i < num_arch; i++) { + this->auto_tuning_cc[i] = new int *[num_precision]; + this->auto_tuning_mr1[i] = new int *[num_precision]; + this->auto_tuning_mr2[i] = new int *[num_precision]; + this->auto_tuning_mr3[i] = new int *[num_precision]; + this->auto_tuning_ts1[i] = new int *[num_precision]; + this->auto_tuning_ts2[i] = new int *[num_precision]; + this->auto_tuning_ts3[i] = new int *[num_precision]; + for (int j = 0; j < num_precision; j++) { + this->auto_tuning_cc[i][j] = new int[num_range]; + this->auto_tuning_mr1[i][j] = new int[num_range]; + this->auto_tuning_mr2[i][j] = new int[num_range]; + this->auto_tuning_mr3[i][j] = new int[num_range]; + this->auto_tuning_ts1[i][j] = new int[num_range]; + this->auto_tuning_ts2[i][j] = new int[num_range]; + this->auto_tuning_ts3[i][j] = new int[num_range]; + } + } + + // Default + for (int i = 0; i < num_arch; i++) { + for (int j = 0; j < num_precision; j++) { + for (int k = 0; k < num_range; k++) { + this->auto_tuning_cc[i][j][k] = 0; + this->auto_tuning_mr1[i][j][k] = 0; + this->auto_tuning_mr2[i][j][k] = 0; + this->auto_tuning_mr3[i][j][k] = 0; + this->auto_tuning_ts1[i][j][k] = 0; + this->auto_tuning_ts2[i][j][k] = 0; + this->auto_tuning_ts3[i][j][k] = 0; + } + } + } + + // Volta-Single + this->auto_tuning_cc[1][0][0] = 1; + this->auto_tuning_cc[1][0][1] = 1; + this->auto_tuning_cc[1][0][2] = 1; + this->auto_tuning_cc[1][0][3] = 1; + this->auto_tuning_cc[1][0][4] = 1; + this->auto_tuning_cc[1][0][5] = 5; + this->auto_tuning_cc[1][0][6] = 5; + this->auto_tuning_cc[1][0][7] = 5; + this->auto_tuning_cc[1][0][8] = 5; + + this->auto_tuning_mr1[1][0][0] = 1; + this->auto_tuning_mr2[1][0][0] = 1; + this->auto_tuning_mr3[1][0][0] = 1; + this->auto_tuning_mr1[1][0][1] = 1; + this->auto_tuning_mr2[1][0][1] = 1; + this->auto_tuning_mr3[1][0][1] = 1; + this->auto_tuning_mr1[1][0][2] = 1; + this->auto_tuning_mr2[1][0][2] = 1; + this->auto_tuning_mr3[1][0][2] = 1; + this->auto_tuning_mr1[1][0][3] = 3; + this->auto_tuning_mr2[1][0][3] = 3; + this->auto_tuning_mr3[1][0][3] = 3; + this->auto_tuning_mr1[1][0][4] = 4; + this->auto_tuning_mr2[1][0][4] = 1; + this->auto_tuning_mr3[1][0][4] = 3; + this->auto_tuning_mr1[1][0][5] = 5; + this->auto_tuning_mr2[1][0][5] = 3; + this->auto_tuning_mr3[1][0][5] = 3; + this->auto_tuning_mr1[1][0][6] = 5; + this->auto_tuning_mr2[1][0][6] = 4; + this->auto_tuning_mr3[1][0][6] = 4; + this->auto_tuning_mr1[1][0][7] = 3; + this->auto_tuning_mr2[1][0][7] = 4; + this->auto_tuning_mr3[1][0][7] = 4; + this->auto_tuning_mr1[1][0][8] = 3; + this->auto_tuning_mr2[1][0][8] = 4; + this->auto_tuning_mr3[1][0][8] = 4; + + this->auto_tuning_ts1[1][0][0] = 1; + this->auto_tuning_ts2[1][0][0] = 1; + this->auto_tuning_ts3[1][0][0] = 1; + this->auto_tuning_ts1[1][0][1] = 1; + this->auto_tuning_ts2[1][0][1] = 1; + this->auto_tuning_ts3[1][0][1] = 1; + this->auto_tuning_ts1[1][0][2] = 2; + this->auto_tuning_ts2[1][0][2] = 2; + this->auto_tuning_ts3[1][0][2] = 2; + this->auto_tuning_ts1[1][0][3] = 3; + this->auto_tuning_ts2[1][0][3] = 2; + this->auto_tuning_ts3[1][0][3] = 2; + this->auto_tuning_ts1[1][0][4] = 3; + this->auto_tuning_ts2[1][0][4] = 2; + this->auto_tuning_ts3[1][0][4] = 2; + this->auto_tuning_ts1[1][0][5] = 3; + this->auto_tuning_ts2[1][0][5] = 2; + this->auto_tuning_ts3[1][0][5] = 2; + this->auto_tuning_ts1[1][0][6] = 5; + this->auto_tuning_ts2[1][0][6] = 3; + this->auto_tuning_ts3[1][0][6] = 2; + this->auto_tuning_ts1[1][0][7] = 5; + this->auto_tuning_ts2[1][0][7] = 6; + this->auto_tuning_ts3[1][0][7] = 5; + this->auto_tuning_ts1[1][0][8] = 5; + this->auto_tuning_ts2[1][0][8] = 6; + this->auto_tuning_ts3[1][0][8] = 5; + // Volta-Double + + this->auto_tuning_cc[1][1][0] = 1; + this->auto_tuning_cc[1][1][1] = 1; + this->auto_tuning_cc[1][1][2] = 1; + this->auto_tuning_cc[1][1][3] = 1; + this->auto_tuning_cc[1][1][4] = 4; + this->auto_tuning_cc[1][1][5] = 5; + this->auto_tuning_cc[1][1][6] = 6; + this->auto_tuning_cc[1][1][7] = 6; + this->auto_tuning_cc[1][1][8] = 5; + + this->auto_tuning_mr1[1][1][0] = 1; + this->auto_tuning_mr2[1][1][0] = 1; + this->auto_tuning_mr3[1][1][0] = 1; + this->auto_tuning_mr1[1][1][1] = 1; + this->auto_tuning_mr2[1][1][1] = 1; + this->auto_tuning_mr3[1][1][1] = 1; + this->auto_tuning_mr1[1][1][2] = 1; + this->auto_tuning_mr2[1][1][2] = 1; + this->auto_tuning_mr3[1][1][2] = 1; + this->auto_tuning_mr1[1][1][3] = 1; + this->auto_tuning_mr2[1][1][3] = 3; + this->auto_tuning_mr3[1][1][3] = 1; + this->auto_tuning_mr1[1][1][4] = 4; + this->auto_tuning_mr2[1][1][4] = 3; + this->auto_tuning_mr3[1][1][4] = 3; + this->auto_tuning_mr1[1][1][5] = 5; + this->auto_tuning_mr2[1][1][5] = 5; + this->auto_tuning_mr3[1][1][5] = 5; + this->auto_tuning_mr1[1][1][6] = 4; + this->auto_tuning_mr2[1][1][6] = 6; + this->auto_tuning_mr3[1][1][6] = 6; + this->auto_tuning_mr1[1][1][7] = 6; + this->auto_tuning_mr2[1][1][7] = 6; + this->auto_tuning_mr3[1][1][7] = 5; + this->auto_tuning_mr1[1][1][8] = 6; + this->auto_tuning_mr2[1][1][8] = 6; + this->auto_tuning_mr3[1][1][8] = 5; + + this->auto_tuning_ts1[1][1][0] = 1; + this->auto_tuning_ts2[1][1][0] = 1; + this->auto_tuning_ts3[1][1][0] = 1; + this->auto_tuning_ts1[1][1][1] = 1; + this->auto_tuning_ts2[1][1][1] = 1; + this->auto_tuning_ts3[1][1][1] = 1; + this->auto_tuning_ts1[1][1][2] = 2; + this->auto_tuning_ts2[1][1][2] = 2; + this->auto_tuning_ts3[1][1][2] = 2; + this->auto_tuning_ts1[1][1][3] = 3; + this->auto_tuning_ts2[1][1][3] = 2; + this->auto_tuning_ts3[1][1][3] = 2; + this->auto_tuning_ts1[1][1][4] = 3; + this->auto_tuning_ts2[1][1][4] = 2; + this->auto_tuning_ts3[1][1][4] = 2; + this->auto_tuning_ts1[1][1][5] = 4; + this->auto_tuning_ts2[1][1][5] = 2; + this->auto_tuning_ts3[1][1][5] = 2; + this->auto_tuning_ts1[1][1][6] = 5; + this->auto_tuning_ts2[1][1][6] = 5; + this->auto_tuning_ts3[1][1][6] = 2; + this->auto_tuning_ts1[1][1][7] = 5; + this->auto_tuning_ts2[1][1][7] = 6; + this->auto_tuning_ts3[1][1][7] = 6; + this->auto_tuning_ts1[1][1][8] = 5; + this->auto_tuning_ts2[1][1][8] = 6; + this->auto_tuning_ts3[1][1][8] = 6; + + // Turing-Single + this->auto_tuning_cc[2][0][0] = 1; + this->auto_tuning_cc[2][0][1] = 1; + this->auto_tuning_cc[2][0][2] = 1; + this->auto_tuning_cc[2][0][3] = 1; + this->auto_tuning_cc[2][0][4] = 3; + this->auto_tuning_cc[2][0][5] = 5; + this->auto_tuning_cc[2][0][6] = 5; + this->auto_tuning_cc[2][0][7] = 5; + this->auto_tuning_cc[2][0][8] = 4; + + this->auto_tuning_mr1[2][0][0] = 1; + this->auto_tuning_mr2[2][0][0] = 1; + this->auto_tuning_mr3[2][0][0] = 1; + this->auto_tuning_mr1[2][0][1] = 1; + this->auto_tuning_mr2[2][0][1] = 1; + this->auto_tuning_mr3[2][0][1] = 1; + this->auto_tuning_mr1[2][0][2] = 1; + this->auto_tuning_mr2[2][0][2] = 1; + this->auto_tuning_mr3[2][0][2] = 1; + this->auto_tuning_mr1[2][0][3] = 1; + this->auto_tuning_mr2[2][0][3] = 1; + this->auto_tuning_mr3[2][0][3] = 3; + this->auto_tuning_mr1[2][0][4] = 4; + this->auto_tuning_mr2[2][0][4] = 3; + this->auto_tuning_mr3[2][0][4] = 4; + this->auto_tuning_mr1[2][0][5] = 4; + this->auto_tuning_mr2[2][0][5] = 3; + this->auto_tuning_mr3[2][0][5] = 3; + this->auto_tuning_mr1[2][0][6] = 6; + this->auto_tuning_mr2[2][0][6] = 3; + this->auto_tuning_mr3[2][0][6] = 3; + this->auto_tuning_mr1[2][0][7] = 5; + this->auto_tuning_mr2[2][0][7] = 4; + this->auto_tuning_mr3[2][0][7] = 4; + this->auto_tuning_mr1[2][0][8] = 5; + this->auto_tuning_mr2[2][0][8] = 4; + this->auto_tuning_mr3[2][0][8] = 4; + + this->auto_tuning_ts1[2][0][0] = 1; + this->auto_tuning_ts2[2][0][0] = 1; + this->auto_tuning_ts3[2][0][0] = 1; + this->auto_tuning_ts1[2][0][1] = 1; + this->auto_tuning_ts2[2][0][1] = 1; + this->auto_tuning_ts3[2][0][1] = 1; + this->auto_tuning_ts1[2][0][2] = 2; + this->auto_tuning_ts2[2][0][2] = 2; + this->auto_tuning_ts3[2][0][2] = 2; + this->auto_tuning_ts1[2][0][3] = 3; + this->auto_tuning_ts2[2][0][3] = 2; + this->auto_tuning_ts3[2][0][3] = 2; + this->auto_tuning_ts1[2][0][4] = 3; + this->auto_tuning_ts2[2][0][4] = 2; + this->auto_tuning_ts3[2][0][4] = 2; + this->auto_tuning_ts1[2][0][5] = 3; + this->auto_tuning_ts2[2][0][5] = 2; + this->auto_tuning_ts3[2][0][5] = 2; + this->auto_tuning_ts1[2][0][6] = 5; + this->auto_tuning_ts2[2][0][6] = 5; + this->auto_tuning_ts3[2][0][6] = 2; + this->auto_tuning_ts1[2][0][7] = 5; + this->auto_tuning_ts2[2][0][7] = 6; + this->auto_tuning_ts3[2][0][7] = 6; + this->auto_tuning_ts1[2][0][8] = 5; + this->auto_tuning_ts2[2][0][8] = 6; + this->auto_tuning_ts3[2][0][8] = 6; + // Turing-Double + + this->auto_tuning_cc[2][1][0] = 0; + this->auto_tuning_cc[2][1][1] = 0; + this->auto_tuning_cc[2][1][2] = 2; + this->auto_tuning_cc[2][1][3] = 2; + this->auto_tuning_cc[2][1][4] = 3; + this->auto_tuning_cc[2][1][5] = 4; + this->auto_tuning_cc[2][1][6] = 4; + this->auto_tuning_cc[2][1][7] = 6; + this->auto_tuning_cc[2][1][8] = 3; + + this->auto_tuning_mr1[2][1][0] = 1; + this->auto_tuning_mr2[2][1][0] = 1; + this->auto_tuning_mr3[2][1][0] = 1; + this->auto_tuning_mr1[2][1][1] = 1; + this->auto_tuning_mr2[2][1][1] = 1; + this->auto_tuning_mr3[2][1][1] = 1; + this->auto_tuning_mr1[2][1][2] = 1; + this->auto_tuning_mr2[2][1][2] = 1; + this->auto_tuning_mr3[2][1][2] = 1; + this->auto_tuning_mr1[2][1][3] = 1; + this->auto_tuning_mr2[2][1][3] = 1; + this->auto_tuning_mr3[2][1][3] = 1; + this->auto_tuning_mr1[2][1][4] = 4; + this->auto_tuning_mr2[2][1][4] = 4; + this->auto_tuning_mr3[2][1][4] = 1; + this->auto_tuning_mr1[2][1][5] = 1; + this->auto_tuning_mr2[2][1][5] = 1; + this->auto_tuning_mr3[2][1][5] = 1; + this->auto_tuning_mr1[2][1][6] = 1; + this->auto_tuning_mr2[2][1][6] = 1; + this->auto_tuning_mr3[2][1][6] = 1; + this->auto_tuning_mr1[2][1][7] = 1; + this->auto_tuning_mr2[2][1][7] = 1; + this->auto_tuning_mr3[2][1][7] = 1; + this->auto_tuning_mr1[2][1][8] = 1; + this->auto_tuning_mr2[2][1][8] = 1; + this->auto_tuning_mr3[2][1][8] = 1; + + this->auto_tuning_ts1[2][1][0] = 1; + this->auto_tuning_ts2[2][1][0] = 1; + this->auto_tuning_ts3[2][1][0] = 1; + this->auto_tuning_ts1[2][1][1] = 1; + this->auto_tuning_ts2[2][1][1] = 1; + this->auto_tuning_ts3[2][1][1] = 1; + this->auto_tuning_ts1[2][1][2] = 2; + this->auto_tuning_ts2[2][1][2] = 2; + this->auto_tuning_ts3[2][1][2] = 2; + this->auto_tuning_ts1[2][1][3] = 3; + this->auto_tuning_ts2[2][1][3] = 2; + this->auto_tuning_ts3[2][1][3] = 2; + this->auto_tuning_ts1[2][1][4] = 2; + this->auto_tuning_ts2[2][1][4] = 2; + this->auto_tuning_ts3[2][1][4] = 2; + this->auto_tuning_ts1[2][1][5] = 2; + this->auto_tuning_ts2[2][1][5] = 2; + this->auto_tuning_ts3[2][1][5] = 2; + this->auto_tuning_ts1[2][1][6] = 3; + this->auto_tuning_ts2[2][1][6] = 5; + this->auto_tuning_ts3[2][1][6] = 3; + this->auto_tuning_ts1[2][1][7] = 3; + this->auto_tuning_ts2[2][1][7] = 6; + this->auto_tuning_ts3[2][1][7] = 6; + this->auto_tuning_ts1[2][1][8] = 3; + this->auto_tuning_ts2[2][1][8] = 6; + this->auto_tuning_ts3[2][1][8] = 6; + } + + MGARDX_CONT + ~AutoTuningTable() { + for (int i = 0; i < num_arch; i++) { + for (int j = 0; j < num_precision; j++) { + delete[] this->auto_tuning_cc[i][j]; + delete[] this->auto_tuning_mr1[i][j]; + delete[] this->auto_tuning_mr2[i][j]; + delete[] this->auto_tuning_mr3[i][j]; + delete[] this->auto_tuning_ts1[i][j]; + delete[] this->auto_tuning_ts2[i][j]; + delete[] this->auto_tuning_ts3[i][j]; + } + delete[] this->auto_tuning_cc[i]; + delete[] this->auto_tuning_mr1[i]; + delete[] this->auto_tuning_mr2[i]; + delete[] this->auto_tuning_mr3[i]; + delete[] this->auto_tuning_ts1[i]; + delete[] this->auto_tuning_ts2[i]; + delete[] this->auto_tuning_ts3[i]; + } + delete[] this->auto_tuning_cc; + delete[] this->auto_tuning_mr1; + delete[] this->auto_tuning_mr2; + delete[] this->auto_tuning_mr3; + delete[] this->auto_tuning_ts1; + delete[] this->auto_tuning_ts2; + delete[] this->auto_tuning_ts3; + } + + int num_arch = 3; + int num_precision = 2; + int num_range = 9; + int ***auto_tuning_cc; + int ***auto_tuning_mr1, ***auto_tuning_ts1; + int ***auto_tuning_mr2, ***auto_tuning_ts2; + int ***auto_tuning_mr3, ***auto_tuning_ts3; + int arch, precision; +}; + +template <> class AutoTuner { +public: + MGARDX_CONT + AutoTuner(){}; + + static KernelConfigs kernelConfigs; + static AutoTuningTable autoTuningTable; + static bool ProfileKernels; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerOpenmp.h b/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerOpenmp.h new file mode 100644 index 000000000..915041a7b --- /dev/null +++ b/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerOpenmp.h @@ -0,0 +1,105 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_AUTOTUNER_OPENMP_H +#define MGARD_X_160_AUTOTUNER_OPENMP_H +// clang-format off +namespace mgard_x_160 { + +template <> class AutoTuningTable { +public: + static constexpr int num_types = 2; + static constexpr int num_dims = 9; + + static constexpr int gpk_reo_3d[num_types][num_dims] = {{5, 5, 0, 3, 3, 5, 5, 0, 0}, + {3, 6, 0, 3, 3, 3, 5, 0, 0}}; + + static constexpr int gpk_rev_3d[num_types][num_dims] = {{2, 4, 1, 5, 3, 5, 5, 0, 0}, + {3, 6, 1, 5, 3, 5, 6, 0, 0}}; + + static constexpr int gpk_reo_nd[num_types][num_dims] = {{5, 5, 5, 5, 5, 5, 5, 5, 5}, + {5, 5, 5, 5, 5, 5, 5, 5, 5}}; + + static constexpr int gpk_rev_nd[num_types][num_dims] = {{0, 0, 3, 4, 3, 0, 0, 0, 0}, + {0, 0, 3, 4, 5, 0, 0, 0, 0}}; + + static constexpr int lpk1_3d[num_types][num_dims] = {{4, 4, 0, 1, 1, 1, 1, 0, 0}, + {1, 1, 5, 1, 1, 1, 1, 0, 0}}; + + static constexpr int lpk2_3d[num_types][num_dims] = {{5, 4, 1, 4, 3, 3, 4, 0, 0}, + {4, 1, 2, 1, 1, 1, 3, 0, 0}}; + + static constexpr int lpk3_3d[num_types][num_dims] = {{4, 4, 2, 3, 2, 3, 4, 0, 0}, + {1, 1, 2, 1, 1, 1, 2, 0, 0}}; + + static constexpr int lpk1_nd[num_types][num_dims] = {{2, 0, 1, 1, 1, 0, 0, 0, 0}, + {0, 0, 1, 1, 1, 0, 0, 0, 0}}; + + static constexpr int lpk2_nd[num_types][num_dims] = {{2, 1, 3, 1, 0, 0, 0, 0, 0}, + {0, 2, 1, 1, 0, 0, 0, 0, 0}}; + + static constexpr int lpk3_nd[num_types][num_dims] = {{2, 3, 1, 1, 0, 0, 0, 0, 0}, + {0, 2, 1, 1, 0, 0, 0, 0, 0}}; + + static constexpr int ipk1_3d[num_types][num_dims] = {{3, 3, 1, 5, 5, 3, 4, 0, 0}, + {3, 6, 1, 4, 3, 3, 3, 0, 0}}; + + static constexpr int ipk2_3d[num_types][num_dims] = {{3, 3, 4, 2, 2, 2, 6, 0, 0}, + {2, 2, 4, 2, 2, 2, 5, 0, 0}}; + + static constexpr int ipk3_3d[num_types][num_dims] = {{3, 3, 4, 2, 2, 2, 1, 0, 0}, + {2, 2, 4, 2, 2, 2, 6, 0, 0}}; + + static constexpr int ipk1_nd[num_types][num_dims] = {{0, 2, 3, 3, 0, 0, 0, 0, 0}, + {0, 3, 3, 3, 0, 0, 0, 0, 0}}; + + static constexpr int ipk2_nd[num_types][num_dims] = {{0, 1, 2, 2, 0, 0, 0, 0, 0}, + {0, 2, 2, 2, 0, 0, 0, 0, 0}}; + + static constexpr int ipk3_nd[num_types][num_dims] = {{0, 2, 3, 2, 0, 0, 0, 0, 0}, + {0, 3, 4, 2, 0, 0, 0, 0, 0}}; + + static constexpr int lwpk[num_types][num_dims] = {{5, 2, 1, 1, 0, 2, 1, 0, 0}, + {2, 2, 1, 3, 0, 2, 5, 0, 0}}; + + static constexpr int lwqzk[num_types][num_dims] = {{0, 0, 2, 0, 0, 0, 0, 0, 0}, + {0, 0, 3, 0, 0, 0, 0, 0, 0}}; + + static constexpr int lwdqzk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int llk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int sdck[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int sdmtk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int encode[num_types][num_dims] = {{4, 4, 4, 4, 4, 4, 4, 4, 4}, + {4, 4, 4, 4, 4, 4, 4, 4, 4}}; + + static constexpr int deflate[num_types][num_dims] = {{3, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; + + static constexpr int decode[num_types][num_dims] = {{3, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; +}; + +template <> class AutoTuner { +public: + MGARDX_CONT + AutoTuner(){}; + static AutoTuningTable autoTuningTable; + static bool ProfileKernels; + static bool WriteToTable; +}; + +} // namespace mgard_x_160 +// clang-format on +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSerial.h b/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSerial.h new file mode 100644 index 000000000..d8f05ec8c --- /dev/null +++ b/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSerial.h @@ -0,0 +1,105 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_AUTOTUNER_SERIAL_H +#define MGARD_X_160_AUTOTUNER_SERIAL_H +// clang-format off +namespace mgard_x_160 { + +template <> class AutoTuningTable { +public: + static constexpr int num_types = 2; + static constexpr int num_dims = 9; + + static constexpr int gpk_reo_3d[num_types][num_dims] = {{5, 5, 5, 3, 3, 5, 5, 0, 0}, + {3, 6, 5, 3, 3, 3, 5, 0, 0}}; + + static constexpr int gpk_rev_3d[num_types][num_dims] = {{2, 4, 5, 5, 3, 5, 5, 0, 0}, + {3, 6, 6, 5, 3, 5, 6, 0, 0}}; + + static constexpr int gpk_reo_nd[num_types][num_dims] = {{5, 5, 5, 5, 5, 5, 5, 5, 5}, + {5, 5, 5, 5, 5, 5, 5, 5, 5}}; + + static constexpr int gpk_rev_nd[num_types][num_dims] = {{0, 0, 3, 4, 3, 0, 0, 0, 0}, + {0, 0, 3, 4, 5, 0, 0, 0, 0}}; + + static constexpr int lpk1_3d[num_types][num_dims] = {{4, 4, 1, 1, 1, 1, 1, 0, 0}, + {1, 1, 1, 1, 1, 1, 1, 0, 0}}; + + static constexpr int lpk2_3d[num_types][num_dims] = {{5, 4, 4, 4, 3, 3, 4, 0, 0}, + {4, 1, 1, 1, 1, 1, 3, 0, 0}}; + + static constexpr int lpk3_3d[num_types][num_dims] = {{4, 4, 3, 3, 2, 3, 4, 0, 0}, + {1, 1, 1, 1, 1, 1, 2, 0, 0}}; + + static constexpr int lpk1_nd[num_types][num_dims] = {{2, 0, 1, 1, 1, 0, 0, 0, 0}, + {0, 0, 1, 1, 1, 0, 0, 0, 0}}; + + static constexpr int lpk2_nd[num_types][num_dims] = {{2, 1, 3, 1, 0, 0, 0, 0, 0}, + {0, 2, 1, 1, 0, 0, 0, 0, 0}}; + + static constexpr int lpk3_nd[num_types][num_dims] = {{2, 3, 1, 1, 0, 0, 0, 0, 0}, + {0, 2, 1, 1, 0, 0, 0, 0, 0}}; + + static constexpr int ipk1_3d[num_types][num_dims] = {{3, 3, 4, 5, 5, 3, 4, 0, 0}, + {3, 6, 4, 4, 3, 3, 3, 0, 0}}; + + static constexpr int ipk2_3d[num_types][num_dims] = {{3, 3, 2, 2, 2, 2, 6, 0, 0}, + {2, 2, 2, 2, 2, 2, 5, 0, 0}}; + + static constexpr int ipk3_3d[num_types][num_dims] = {{3, 3, 2, 2, 2, 2, 1, 0, 0}, + {2, 2, 2, 2, 2, 2, 6, 0, 0}}; + + static constexpr int ipk1_nd[num_types][num_dims] = {{0, 2, 3, 3, 0, 0, 0, 0, 0}, + {0, 3, 3, 3, 0, 0, 0, 0, 0}}; + + static constexpr int ipk2_nd[num_types][num_dims] = {{0, 1, 2, 2, 0, 0, 0, 0, 0}, + {0, 2, 2, 2, 0, 0, 0, 0, 0}}; + + static constexpr int ipk3_nd[num_types][num_dims] = {{0, 2, 3, 2, 0, 0, 0, 0, 0}, + {0, 3, 4, 2, 0, 0, 0, 0, 0}}; + + static constexpr int lwpk[num_types][num_dims] = {{5, 2, 2, 1, 0, 2, 1, 0, 0}, + {2, 2, 3, 3, 0, 2, 5, 0, 0}}; + + static constexpr int lwqzk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int lwdqzk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int llk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int sdck[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int sdmtk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int encode[num_types][num_dims] = {{4, 4, 4, 4, 4, 4, 4, 4, 4}, + {4, 4, 4, 4, 4, 4, 4, 4, 4}}; + + static constexpr int deflate[num_types][num_dims] = {{3, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; + + static constexpr int decode[num_types][num_dims] = {{3, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; +}; + +template <> class AutoTuner { +public: + MGARDX_CONT + AutoTuner(){}; + static AutoTuningTable autoTuningTable; + static bool ProfileKernels; + static bool WriteToTable; +}; + +} // namespace mgard_x_160 +// clang-format on +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSycl.h b/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSycl.h new file mode 100644 index 000000000..8485e744f --- /dev/null +++ b/include/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSycl.h @@ -0,0 +1,106 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_AUTOTUNER_SYCL_H +#define MGARD_X_160_AUTOTUNER_SYCL_H +// clang-format off +namespace mgard_x_160 { + +template <> class AutoTuningTable { +public: + static constexpr int num_types = 2; + static constexpr int num_dims = 9; + + static constexpr int gpk_reo_3d[num_types][num_dims] = {{5, 5, 5, 3, 3, 5, 5, 0, 0}, + {3, 6, 5, 3, 3, 3, 5, 0, 0}}; + + static constexpr int gpk_rev_3d[num_types][num_dims] = {{2, 4, 5, 5, 3, 5, 5, 0, 0}, + {3, 6, 6, 5, 3, 5, 6, 0, 0}}; + + static constexpr int gpk_reo_nd[num_types][num_dims] = {{5, 5, 5, 5, 5, 5, 5, 5, 5}, + {5, 5, 5, 5, 5, 5, 5, 5, 5}}; + + static constexpr int gpk_rev_nd[num_types][num_dims] = {{0, 0, 3, 4, 3, 0, 0, 0, 0}, + {0, 0, 3, 4, 5, 0, 0, 0, 0}}; + + static constexpr int lpk1_3d[num_types][num_dims] = {{4, 4, 1, 1, 1, 1, 1, 0, 0}, + {1, 1, 1, 1, 1, 1, 1, 0, 0}}; + + static constexpr int lpk2_3d[num_types][num_dims] = {{5, 4, 4, 4, 3, 3, 4, 0, 0}, + {4, 1, 1, 1, 1, 1, 3, 0, 0}}; + + static constexpr int lpk3_3d[num_types][num_dims] = {{4, 4, 3, 3, 2, 3, 4, 0, 0}, + {1, 1, 1, 1, 1, 1, 2, 0, 0}}; + + static constexpr int lpk1_nd[num_types][num_dims] = {{2, 0, 1, 1, 1, 0, 0, 0, 0}, + {0, 0, 1, 1, 1, 0, 0, 0, 0}}; + + static constexpr int lpk2_nd[num_types][num_dims] = {{2, 1, 3, 1, 0, 0, 0, 0, 0}, + {0, 2, 1, 1, 0, 0, 0, 0, 0}}; + + static constexpr int lpk3_nd[num_types][num_dims] = {{2, 3, 1, 1, 0, 0, 0, 0, 0}, + {0, 2, 1, 1, 0, 0, 0, 0, 0}}; + + static constexpr int ipk1_3d[num_types][num_dims] = {{3, 3, 4, 5, 5, 3, 4, 0, 0}, + {3, 6, 4, 4, 3, 3, 3, 0, 0}}; + + static constexpr int ipk2_3d[num_types][num_dims] = {{3, 3, 2, 2, 2, 2, 6, 0, 0}, + {2, 2, 2, 2, 2, 2, 5, 0, 0}}; + + static constexpr int ipk3_3d[num_types][num_dims] = {{3, 3, 2, 2, 2, 2, 1, 0, 0}, + {2, 2, 2, 2, 2, 2, 6, 0, 0}}; + + static constexpr int ipk1_nd[num_types][num_dims] = {{0, 2, 3, 3, 0, 0, 0, 0, 0}, + {0, 3, 3, 3, 0, 0, 0, 0, 0}}; + + static constexpr int ipk2_nd[num_types][num_dims] = {{0, 1, 2, 2, 0, 0, 0, 0, 0}, + {0, 2, 2, 2, 0, 0, 0, 0, 0}}; + + static constexpr int ipk3_nd[num_types][num_dims] = {{0, 2, 3, 2, 0, 0, 0, 0, 0}, + {0, 3, 4, 2, 0, 0, 0, 0, 0}}; + + static constexpr int lwpk[num_types][num_dims] = {{5, 2, 2, 1, 0, 2, 1, 0, 0}, + {2, 2, 3, 3, 0, 2, 5, 0, 0}}; + + static constexpr int lwqzk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int lwdqzk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int llk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int sdck[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int sdmtk[num_types][num_dims] = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0, 0, 0, 0}}; + + static constexpr int encode[num_types][num_dims] = {{4, 4, 4, 4, 4, 4, 4, 4, 4}, + {4, 4, 4, 4, 4, 4, 4, 4, 4}}; + + static constexpr int deflate[num_types][num_dims] = {{3, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; + + static constexpr int decode[num_types][num_dims] = {{3, 3, 3, 3, 3, 3, 3, 3, 3}, + {3, 3, 3, 3, 3, 3, 3, 3, 3}}; + +}; + +template <> class AutoTuner { +public: + MGARDX_CONT + AutoTuner(){}; + static AutoTuningTable autoTuningTable; + static bool ProfileKernels; + static bool WriteToTable; +}; + +} // namespace mgard_x_160 +// clang-format on +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/DataStructures/Array.h b/include/mgard-x-160/RuntimeX/DataStructures/Array.h new file mode 100644 index 000000000..04bc86035 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/DataStructures/Array.h @@ -0,0 +1,69 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_ARRAY +#define MGARD_X_160_ARRAY + +#include + +namespace mgard_x_160 { + +template +class Array { +public: + Array(); + Array(std::vector shape, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + Array(std::vector shape, T *dv); + void initialize(std::vector shape); + void allocate(int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + void copy(const Array &array, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + void move(Array &&array); + void memset(int value, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + void free(int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + Array(const Array &array); + Array &operator=(const Array &array); + Array &operator=(Array &&array); + Array(Array &&array); + ~Array(); + void load(const T *data, SIZE ld = 0, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + void hostAllocate(bool keep = false, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + T *hostCopy(bool keep = false, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + T *data(SIZE &ld); + SIZE &shape(DIM d); + std::vector &shape(); + SIZE totalNumElems(); + T *data(); + T *dataHost(); + SIZE ld(DIM d); + bool isPitched(); + bool isManaged(); + bool hasDeviceAllocation(); + bool hasHostAllocation(); + int resideDevice(); + bool resize(std::vector shape, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + +private: + int dev_id; + bool keepHostCopy = false; + T *dv = nullptr; + T *hv = nullptr; + bool device_allocated = false; + bool host_allocated = false; + bool external_allocation = false; + std::vector __ldvs; + std::vector __shape; + std::vector __ldvs_allocation; + std::vector __shape_allocation; + SIZE linearized_width; +}; + +} // namespace mgard_x_160 +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/DataStructures/Array.hpp b/include/mgard-x-160/RuntimeX/DataStructures/Array.hpp new file mode 100644 index 000000000..868c3536c --- /dev/null +++ b/include/mgard-x-160/RuntimeX/DataStructures/Array.hpp @@ -0,0 +1,375 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_ARRAY_HPP +#define MGARD_X_160_ARRAY_HPP +// #include "Common.h" + +// #include "CommonInternal.h" +#include +#include + +#include "Array.h" + +// #include "../DeviceAdapters/DeviceAdapter.h" + +namespace mgard_x_160 { + +template +Array::Array() { + initialize(std::vector(D, 1)); +} + +template +Array::Array(std::vector shape, + int queue_idx) { + initialize(shape); + allocate(queue_idx); +} + +template +Array::Array(std::vector shape, + T *dv) { + initialize(shape); + __shape_allocation = shape; + __ldvs_allocation = shape; + device_allocated = true; + external_allocation = true; + this->dv = dv; +} + +template +void Array::initialize( + std::vector shape) { + if (shape.size() != D) { + throw std::runtime_error( + "Number of dimensions mismatch (" + std::to_string(shape.size()) + + "!=" + std::to_string(D) + "). mgard_x_160::Array not initialized!"); + } + dev_id = DeviceRuntime::GetDevice(); + __shape = shape; + free(); + __ldvs = __shape; + linearized_width = 1; + for (DIM d = 0; d < D - 1; d++) { + linearized_width *= __shape[d]; + } + host_allocated = false; + device_allocated = false; + external_allocation = false; +} + +template +void Array::allocate(int queue_idx) { + if (Pitched) { + if (!Managed) { + SIZE ld = 0; + MemoryManager::MallocND(dv, __shape[D - 1], linearized_width, + ld, queue_idx); + __ldvs[D - 1] = ld; + } else { + std::cerr << log::log_err + << "Does not support managed memory in pitched mode.\n"; + } + } else { + if (!Managed) { + MemoryManager::Malloc1D(dv, __shape[D - 1] * linearized_width, + queue_idx); + } else { + MemoryManager::MallocManaged1D( + dv, __shape[D - 1] * linearized_width, queue_idx); + } + } + __shape_allocation = __shape; + __ldvs_allocation = __ldvs; + device_allocated = true; + external_allocation = false; +} + +template +void Array::copy( + const Array &array, int queue_idx) { + initialize(array.__shape); + if (array.device_allocated) { + allocate(queue_idx); + MemoryManager::CopyND(dv, __ldvs[D - 1], array.dv, + array.__ldvs[D - 1], array.__shape[D - 1], + array.linearized_width, queue_idx); + } + if (array.host_allocated) { + hostCopy(array.keepHostCopy, queue_idx); + } +} + +template +void Array::move( + Array &&array) { + initialize(array.__shape); + this->dev_id = array.dev_id; + if (array.device_allocated) { + this->dv = array.dv; + this->__ldvs = array.__ldvs; + this->__shape_allocation = array.__shape_allocation; + this->__ldvs_allocation = array.__ldvs_allocation; + this->device_allocated = true; + array.device_allocated = false; + this->external_allocation = array.external_allocation; + array.dv = nullptr; + } + if (array.host_allocated) { + this->hv = array.hv; + this->host_allocated = true; + array.host_allocated = false; + } +} + +template +void Array::memset(int value, + int queue_idx) { + if (Pitched) { + MemoryManager::MemsetND(dv, __ldvs[D - 1], __shape[D - 1], + linearized_width, value, queue_idx); + } else { + MemoryManager::Memset1D(dv, __ldvs[D - 1] * linearized_width, + value, queue_idx); + } +} + +template +void Array::free(int queue_idx) { + if (device_allocated && !external_allocation) { + MemoryManager::Free(dv, queue_idx); + device_allocated = false; + dv = nullptr; + } + if (host_allocated && !keepHostCopy) { + MemoryManager::FreeHost(hv, queue_idx); + host_allocated = false; + hv = nullptr; + } +} + +template +Array::Array( + const Array &array) { + this->copy(array); +} + +template +Array & +Array::operator=( + const Array &array) { + // printf("Array operator =\n"); + this->copy(array); + return *this; +} + +template +Array & +Array::operator=( + Array &&array) { + // printf("Array move = \n"); + this->move(std::move(array)); + return *this; +} + +template +Array::Array( + Array &&array) { + // printf("Array move\n"); + this->move(std::move(array)); +} + +template +Array::~Array() { + this->free(); +} + +template +void Array::load(const T *data, SIZE ld, + int queue_idx) { + log::dbg("Calling Array::load"); + if (ld == 0) { + ld = __shape[D - 1]; + } + MemoryManager::CopyND(dv, __ldvs[D - 1], data, ld, __shape[D - 1], + linearized_width, queue_idx); +} +template +void Array::hostAllocate(bool keep, + int queue_idx) { + log::dbg("Calling Array::hostAllocate"); + if (!device_allocated) { + throw std::runtime_error("device buffer not initialized."); + } + if (!host_allocated) { + MemoryManager::MallocHost(hv, __shape[D - 1] * linearized_width, + queue_idx); + host_allocated = true; + } + keepHostCopy = keep; +} + +template +T *Array::hostCopy(bool keep, + int queue_idx) { + log::dbg("Calling Array::hostCopy"); + if (!device_allocated) { + throw std::runtime_error("device buffer not initialized."); + } + if (!host_allocated) { + MemoryManager::MallocHost(hv, __shape[D - 1] * linearized_width, + queue_idx); + host_allocated = true; + } + MemoryManager::CopyND(hv, __shape[D - 1], dv, __ldvs[D - 1], + __shape[D - 1], linearized_width, + queue_idx); + keepHostCopy = keep; + return hv; +} + +template +T *Array::data(SIZE &ld) { + if (!device_allocated) { + throw std::runtime_error("device buffer not initialized."); + } + ld = __ldvs[D - 1]; + return dv; +} + +template +SIZE &Array::shape(DIM d) { + return __shape[d]; +} + +template +std::vector &Array::shape() { + return __shape; +} + +template +SIZE Array::totalNumElems() { + SIZE total_num_elems = 1; + for (DIM d = 0; d < D; d++) { + total_num_elems *= __shape[d]; + } + return total_num_elems; +} + +template +T *Array::data() { + if (!device_allocated) { + throw std::runtime_error("device buffer not initialized."); + } + return dv; +} + +template +T *Array::dataHost() { + if (!host_allocated) { + throw std::runtime_error("host buffer not initialized."); + } + return hv; +} + +template +SIZE Array::ld(DIM d) { + return __ldvs[d]; +} + +template +bool Array::isPitched() { + return Pitched; +} + +template +bool Array::isManaged() { + return Managed; +} + +template +int Array::resideDevice() { + return dev_id; +} + +template +bool Array::hasDeviceAllocation() { + return device_allocated; +} + +template +bool Array::hasHostAllocation() { + return host_allocated; +} + +template +bool Array::resize(std::vector shape, + int queue_idx) { + if (!device_allocated) { + initialize(shape); + allocate(queue_idx); + return false; + } else { + bool inplace_resizable = false; + if (Pitched) { + // check total number of elements + SIZE original_num_elems = 1; + SIZE new_num_elems = 1; + for (DIM d = 0; d < D; d++) { + original_num_elems *= __shape_allocation[d]; + new_num_elems *= shape[d]; + } + if (original_num_elems >= new_num_elems) { + // We can reuse existing allocation + inplace_resizable = true; + __shape = shape; + __ldvs = __shape; + linearized_width = 1; + for (DIM d = 0; d < D - 1; d++) { + linearized_width *= __shape[d]; + } + } + } else { + bool shape_compatiable = true; + for (DIM d = 0; d < D; d++) { + if (__shape_allocation[d] < shape[d]) { + shape_compatiable = false; + break; + } + } + if (shape_compatiable) { + // We can reuse existing allocation + inplace_resizable = true; + __shape = shape; + // Keep the leading dimensions in step with the shape, as initialize() + // and the pitched branch above both do. A non-pitched array is dense, + // and callers rely on that: several of them derive a stride from the + // shape rather than asking the array for its ld (see the + // setLd(fine_shapes[0][d]) sites in + // BlockLocalHierarchyDataRefactor). Leaving a shrunk array with the + // wider ld of its previous shape leaves those two strides disagreeing + // over the same buffer, which silently corrupts the data. + __ldvs = __shape; + linearized_width = 1; + for (DIM d = 0; d < D - 1; d++) { + linearized_width *= __shape[d]; + } + } + } + // If cannot reuse existing allocation or there is no existing allocation + if (!inplace_resizable) { + initialize(shape); + allocate(queue_idx); + } + return inplace_resizable; + } +} + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/DataStructures/MDRData.hpp b/include/mgard-x-160/RuntimeX/DataStructures/MDRData.hpp new file mode 100644 index 000000000..2a73e0174 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/DataStructures/MDRData.hpp @@ -0,0 +1,199 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MDR_X_MDR_DATA_HPP +#define MDR_X_MDR_DATA_HPP + +#include "MDRMetadata.hpp" + +namespace mgard_x_160 { +namespace MDR { + +template class MDRData { +public: + MDRData() {} + // Data + std::vector>> compressed_bitplanes; + std::vector> level_signs; + + MDRData(SIZE num_levels, SIZE num_bitplanes) { + compressed_bitplanes.resize(num_levels); + level_signs.resize(num_levels); + for (int level_idx = 0; level_idx < num_levels; level_idx++) { + compressed_bitplanes[level_idx].resize(num_bitplanes); + } + } + + // Resize before refactoring + template + void Resize(RefactorType &refactor, HierarchyType &hierarchy, int queue_idx) { + std::vector> estimation = + RefactorType::EstimateMaxBitplaneSizes(hierarchy); + SIZE num_levels = estimation.size(); + SIZE num_bitplanes = estimation[0].size(); + std::vector level_num_elems = hierarchy.level_num_elems(); + compressed_bitplanes.resize(num_levels); + level_signs.resize(num_levels); + for (int level_idx = 0; level_idx < num_levels; level_idx++) { + compressed_bitplanes[level_idx].resize(num_bitplanes); + // The decoder's batched bitplane kernels (e.g. + // BPDecoderRegisterBlockFunctor::DecodeBinary) write one sign per slot + // across the full BATCH_SIZE-aligned range, exactly like + // level_data_array in ComposedRefactor/ComposedReconstructor -- so this + // needs the same round_up, not the raw element count, or the decoder + // overruns it whenever a level's element count isn't already a + // multiple of the batch size. + level_signs[level_idx].resize( + {round_up(hierarchy.level_num_elems(level_idx), + RefactorType::BATCH_SIZE)}, + queue_idx); + for (int bitplane_idx = 0; bitplane_idx < num_bitplanes; bitplane_idx++) { + compressed_bitplanes[level_idx][bitplane_idx].resize( + {estimation[level_idx][bitplane_idx]}, queue_idx); + } + } + } + + // Resize before reconstruction + void Resize(MDRMetadata &mdr_metadata, int queue_idx) { + compressed_bitplanes.resize(mdr_metadata.num_levels); + level_signs.resize(mdr_metadata.num_levels); + for (int level_idx = 0; level_idx < mdr_metadata.num_levels; level_idx++) { + compressed_bitplanes[level_idx].resize(mdr_metadata.num_bitplanes); + level_signs[level_idx].resize({mdr_metadata.level_num_elems[level_idx]}, + queue_idx); + for (int bitplane_idx = 0; bitplane_idx < mdr_metadata.num_bitplanes; + bitplane_idx++) { + compressed_bitplanes[level_idx][bitplane_idx].resize( + {mdr_metadata.level_sizes[level_idx][bitplane_idx]}, queue_idx); + } + } + } + + void ResetSigns(int queue_idx) { + for (int level_idx = 0; level_idx < level_signs.size(); level_idx++) { + level_signs[level_idx].memset(0, queue_idx); + } + } + + void CopyFromRefactoredData(MDRMetadata &mdr_metadata, + std::vector> &refactored_data, + int queue_idx) { + for (int level_idx = 0; level_idx < mdr_metadata.num_levels; level_idx++) { + for (int bitplane_idx = + mdr_metadata.loaded_level_num_bitplanes[level_idx]; + bitplane_idx < mdr_metadata.requested_level_num_bitplanes[level_idx]; + bitplane_idx++) { + MemoryManager::Copy1D( + compressed_bitplanes[level_idx][bitplane_idx].data(), + refactored_data[level_idx][bitplane_idx], + mdr_metadata.level_sizes[level_idx][bitplane_idx], queue_idx); + } + } + mdr_metadata.DoneLoadingBitplans(); + } + + void CopyFromRefactoredSigns(MDRMetadata &mdr_metadata, + std::vector &refactored_level_signs, + int queue_idx) { + for (int level_idx = 0; level_idx < mdr_metadata.num_levels; level_idx++) { + MemoryManager::Copy1D( + level_signs[level_idx].data(), refactored_level_signs[level_idx], + mdr_metadata.level_num_elems[level_idx], queue_idx); + } + } + + void CopyToRefactoredData(MDRMetadata &mdr_metadata, + std::vector> &refactored_data, + std::vector> &allocation_size, + int queue_idx) { + refactored_data.resize(mdr_metadata.num_levels); + for (int level_idx = 0; level_idx < mdr_metadata.num_levels; level_idx++) { + refactored_data[level_idx].resize(mdr_metadata.num_bitplanes); + for (int bitplane_idx = 0; bitplane_idx < mdr_metadata.num_bitplanes; + bitplane_idx++) { + if (allocation_size[level_idx][bitplane_idx] >= + mdr_metadata.level_sizes[level_idx][bitplane_idx]) { + MemoryManager::Copy1D( + refactored_data[level_idx][bitplane_idx], + compressed_bitplanes[level_idx][bitplane_idx].data(), + mdr_metadata.level_sizes[level_idx][bitplane_idx], queue_idx); + } else { + throw std::runtime_error( + "Bitplane copy failed. level_idx(" + std::to_string(level_idx) + + ") bitplane_idx(" + std::to_string(bitplane_idx) + + ") Insufficient buffer space " + + std::to_string(allocation_size[level_idx][bitplane_idx]) + + " vs. " + + std::to_string( + mdr_metadata.level_sizes[level_idx][bitplane_idx])); + } + } + } + } + + void CopyToRefactoredSigns(MDRMetadata &mdr_metadata, + std::vector &refactored_level_signs, + int queue_idx) { + for (int level_idx = 0; level_idx < mdr_metadata.num_levels; level_idx++) { + MemoryManager::Copy1D( + refactored_level_signs[level_idx], level_signs[level_idx].data(), + mdr_metadata.level_num_elems[level_idx], queue_idx); + } + } + + void VerifyLoadedBitplans(MDRMetadata &mdr_metadata) { + // TODO: load + for (int level_idx = 0; level_idx < mdr_metadata.num_levels; level_idx++) { + if (!level_signs[level_idx].hasDeviceAllocation()) { + throw std::runtime_error("Bitplane verification failed. level_signs(" + + std::to_string(level_idx) + + ") No allocation found.\n"); + } + // level_signs is allocated rounded up to the decoder's batch size (see + // Resize above), so it is expected to be >= the raw element count, not + // exactly equal to it. + if (level_signs[level_idx].shape(0) < + mdr_metadata.level_num_elems[level_idx]) { + throw std::runtime_error( + "Bitplane verification failed. level_signs(" + + std::to_string(level_idx) + ") Insufficient buffer space " + + std::to_string(level_signs[level_idx].shape(0)) + " vs. " + + std::to_string(mdr_metadata.level_num_elems[level_idx])); + } + for (int bitplane_idx = + mdr_metadata.prev_used_level_num_bitplanes[level_idx]; + bitplane_idx < mdr_metadata.loaded_level_num_bitplanes[level_idx]; + bitplane_idx++) { + if (!compressed_bitplanes[level_idx][bitplane_idx] + .hasDeviceAllocation()) { + throw std::runtime_error( + "Bitplane verification failed. level_idx(" + + std::to_string(level_idx) + ") bitplane_idx(" + + std::to_string(bitplane_idx) + ") No allocation found.\n"); + } + if (compressed_bitplanes[level_idx][bitplane_idx].shape(0) != + mdr_metadata.level_sizes[level_idx][bitplane_idx]) { + throw std::runtime_error( + "Bitplane verification failed. level_idx(" + + std::to_string(level_idx) + ") bitplane_idx(" + + std::to_string(bitplane_idx) + ") Size mismatch " + + std::to_string( + compressed_bitplanes[level_idx][bitplane_idx].shape(0)) + + " vs. " + + std::to_string( + mdr_metadata.level_sizes[level_idx][bitplane_idx])); + } + } + } + } +}; + +} // namespace MDR +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/DataStructures/MDRMetadata.hpp b/include/mgard-x-160/RuntimeX/DataStructures/MDRMetadata.hpp new file mode 100644 index 000000000..d8f3d2d00 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/DataStructures/MDRMetadata.hpp @@ -0,0 +1,210 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MDR_X_MDR_METADATA_HPP +#define MDR_X_MDR_METADATA_HPP + +#include + +namespace mgard_x_160 { +namespace MDR { + +class MDRMetadata { +public: + MDRMetadata() : num_levels(0), num_bitplanes(0) {} + + void Initialize(SIZE num_levels, SIZE num_bitplanes) { + this->num_levels = num_levels; + this->num_bitplanes = num_bitplanes; + level_error_bounds.resize(num_levels); + level_squared_errors.resize(num_levels); + level_sizes.resize(num_levels); + level_num_elems.resize(num_levels); + for (int i = 0; i < num_levels; i++) { + level_squared_errors[i].resize(num_bitplanes + 1); + level_sizes[i].resize(num_bitplanes); + } + } + MDRMetadata(SIZE num_levels, SIZE num_bitplanes) + : num_levels(num_levels), num_bitplanes(num_bitplanes) { + Initialize(num_levels, num_bitplanes); + } + + using T_error = double; + // Metadata + SIZE num_levels; + SIZE num_bitplanes; + std::vector level_error_bounds; + std::vector> level_squared_errors; + std::vector> level_sizes; + std::vector level_num_elems; + bool segmented = false; + bool corresponding_error_return = false; + size_t retrieved_size = 0; + + // For progressive reconstruction + T_error loaded_tol, loaded_s; + T_error requested_tol, requested_s; + T_error prev_tol, prev_s; + T_error tau; + uint32_t requested_size; + size_t num_elements; + double corresponding_error; + std::vector loaded_level_num_bitplanes; + std::vector requested_level_num_bitplanes; + std::vector prev_used_level_num_bitplanes; + + void InitializeForReconstruction() { + loaded_tol = 0, loaded_s = 0; + requested_tol = 0, requested_s = 0; + prev_tol = 0, prev_s = 0; + loaded_level_num_bitplanes = std::vector(num_levels, 0); + requested_level_num_bitplanes = std::vector(num_levels, 0); + prev_used_level_num_bitplanes = std::vector(num_levels, 0); + } + + void PrintLevelSizes() { + for (int level_idx = 0; level_idx < num_levels; level_idx++) { + for (int bitplane_idx = 0; bitplane_idx < num_bitplanes; bitplane_idx++) { + std::cout << level_sizes[level_idx][bitplane_idx] << " "; + } + std::cout << "\n"; + } + } + + uint32_t GetLoadedBitPlaneSizes() { + uint32_t bitplanes_size = 0; + for (int level_idx = 0; level_idx < num_levels; level_idx++) { + // std::cout << "level[" << level_idx << "]" << ", loaded bitplanes: " << + // (int)loaded_level_num_bitplanes[level_idx] << ":" << std::endl; + for (int bitplane_idx = 0; + bitplane_idx < loaded_level_num_bitplanes[level_idx]; + bitplane_idx++) { + // std::cout << (int)level_sizes[level_idx][bitplane_idx] << " "; + bitplanes_size += level_sizes[level_idx][bitplane_idx]; + } + // std::cout << "\n"; + } + return bitplanes_size; + } + + void PrintStatus() { + printf("Request size: %u, s: %f\n", requested_size, requested_s); + for (int level_idx = 0; level_idx < num_levels; level_idx++) { + printf("Level %d bitplanes: used [%2d] loaded [%2d] requested [%2d]\n", + level_idx, prev_used_level_num_bitplanes[level_idx], + loaded_level_num_bitplanes[level_idx], + requested_level_num_bitplanes[level_idx]); + } + // printf("level_num_elems: "); + // for (int level_idx = 0; level_idx < num_levels; level_idx++) { + // printf("%llu ", level_num_elems[level_idx]); + // } + // printf("\n"); + } + + void DoneLoadingBitplans() { + // TODO: load + loaded_tol = requested_tol; + loaded_s = requested_s; + for (int level_idx = 0; level_idx < num_levels; level_idx++) { + loaded_level_num_bitplanes[level_idx] = + requested_level_num_bitplanes[level_idx]; + } + } + + void DoneReconstruct() { + prev_tol = loaded_tol; + prev_s = loaded_s; + for (int level_idx = 0; level_idx < num_levels; level_idx++) { + prev_used_level_num_bitplanes[level_idx] = + loaded_level_num_bitplanes[level_idx]; + } + } + + int PrevFinalLevel() { + int final_level = 0; + for (int level_idx = num_levels - 1; level_idx >= 0; level_idx--) { + SIZE num_bitplanes = prev_used_level_num_bitplanes[level_idx]; + if (num_bitplanes != 0) { + final_level = level_idx; + break; + } + } + return final_level; + } + + int CurrFinalLevel() { + int final_level = 0; + for (int level_idx = num_levels - 1; level_idx >= 0; level_idx--) { + SIZE num_bitplanes = loaded_level_num_bitplanes[level_idx]; + if (num_bitplanes != 0) { + final_level = level_idx; + break; + } + } + return final_level; + } + + SIZE MetadataSize() { + SIZE metadata_size = 0; + metadata_size += sizeof(SIZE) * 2; + metadata_size += sizeof(T_error) * num_levels; + metadata_size += sizeof(T_error) * num_levels * (num_bitplanes + 1); + metadata_size += sizeof(SIZE) * num_levels * num_bitplanes; + metadata_size += sizeof(SIZE) * num_levels; + return metadata_size; + } + + template void Serialize(Byte *&ptr, T *data, SIZE bytes) { + std::memcpy(ptr, (Byte *)data, bytes); + ptr += bytes; + } + + template void Deserialize(Byte *&ptr, T *data, SIZE bytes) { + std::memcpy((Byte *)data, ptr, bytes); + ptr += bytes; + } + + std::vector Serialize() { + std::vector serialize_metadata(MetadataSize()); + Byte *ptr = serialize_metadata.data(); + Serialize(ptr, &num_levels, sizeof(SIZE)); + Serialize(ptr, &num_bitplanes, sizeof(SIZE)); + Serialize(ptr, level_error_bounds.data(), sizeof(T_error) * num_levels); + for (int i = 0; i < num_levels; i++) { + Serialize(ptr, level_squared_errors[i].data(), + sizeof(T_error) * (num_bitplanes + 1)); + } + for (int i = 0; i < num_levels; i++) { + Serialize(ptr, level_sizes[i].data(), sizeof(SIZE) * (num_bitplanes)); + } + Serialize(ptr, level_num_elems.data(), sizeof(SIZE) * num_levels); + return serialize_metadata; + } + + void Deserialize(std::vector serialize_metadata) { + Byte *ptr = serialize_metadata.data(); + Deserialize(ptr, &num_levels, sizeof(SIZE)); + Deserialize(ptr, &num_bitplanes, sizeof(SIZE)); + Initialize(num_levels, num_bitplanes); + Deserialize(ptr, level_error_bounds.data(), sizeof(T_error) * num_levels); + for (int i = 0; i < num_levels; i++) { + Deserialize(ptr, level_squared_errors[i].data(), + sizeof(T_error) * (num_bitplanes + 1)); + } + for (int i = 0; i < num_levels; i++) { + Deserialize(ptr, level_sizes[i].data(), sizeof(SIZE) * (num_bitplanes)); + } + Deserialize(ptr, level_num_elems.data(), sizeof(SIZE) * num_levels); + } +}; + +} // namespace MDR +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/DataStructures/SubArray.hpp b/include/mgard-x-160/RuntimeX/DataStructures/SubArray.hpp new file mode 100644 index 000000000..bfd9a82dc --- /dev/null +++ b/include/mgard-x-160/RuntimeX/DataStructures/SubArray.hpp @@ -0,0 +1,377 @@ +/* + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_SUBARRAY_HPP +#define MGARD_X_160_SUBARRAY_HPP + +#include +#include +#include + +namespace mgard_x_160 { + +template +class SubArray { +public: + MGARDX_CONT_EXEC + SubArray(); + + MGARDX_CONT + SubArray(Array &array); + + MGARDX_CONT + SubArray(std::vector shape, T *dv); + + MGARDX_CONT_EXEC + void initialize(); + + MGARDX_CONT_EXEC + T *data() { return this->dv; } + + MGARDX_CONT_EXEC + void setData(T *dv) { this->dv = dv; } + + MGARDX_CONT_EXEC + bool hasHostData() { return has_host_pointer; }; + + MGARDX_CONT + T *dataHost() { + if (!has_host_pointer) { + throw std::runtime_error("Host pointer not initialized!"); + } + return v; + } + + MGARDX_CONT + void setDataHost(T *v) { + this->has_host_pointer = true; + this->v = v; + } + + MGARDX_CONT_EXEC + void setShape(DIM d, SIZE n) { + if (d >= D) { + return; + } + __shape[d] = n; + } + + MGARDX_CONT_EXEC + SIZE shape(DIM d) const { + if (d >= D) + return 1; + return __shape[d]; + } + + MGARDX_CONT_EXEC + SIZE ld(DIM d) const { + if (d >= D) + return 1; + return __ldvs[d]; + } + + MGARDX_CONT_EXEC + void setLd(DIM d, SIZE ld) { __ldvs[d] = ld; } + + MGARDX_CONT_EXEC + bool isPitched() { return Pitched; } + + // MGARDX_CONT_EXEC + // void setPitched(bool pitched) { this->pitched = pitched; } + + MGARDX_CONT_EXEC + SIZE lddv1() const { return __lddv1; } + + MGARDX_CONT_EXEC + SIZE lddv2() const { return __lddv2; } + + void offset(std::vector idx); + + MGARDX_CONT + void resize(std::vector shape); + + MGARDX_CONT + void offset_dim(DIM dim, SIZE offset_value); + + MGARDX_CONT + void resize(DIM dim, SIZE new_size); + + MGARDX_CONT + void project(DIM dim_slowest, DIM dim_medium, DIM dim_fastest); + + MGARDX_CONT + SubArray<1, T, DeviceType> Linearize(); + + MGARDX_CONT + SubArray<3, T, DeviceType> Slice3D(DIM d1, DIM d2, DIM d3); + + MGARDX_CONT_EXEC + T &operator[](SIZE idx[D]) { + SIZE curr_stride = 1; + SIZE offset = 0; + for (int d = D - 1; d >= 0; d--) { + offset += idx[d] * curr_stride; + curr_stride *= __ldvs[d]; + } + return dv[offset]; + } + + MGARDX_CONT_EXEC + T *operator()(IDX l, IDX z, IDX y, IDX x) { + return dv + __ldvs[1] * __ldvs[2] * __ldvs[3] * l + + __ldvs[2] * __ldvs[3] * z + __ldvs[3] * y + x; + } + MGARDX_CONT_EXEC + T *operator()(IDX z, IDX y, IDX x) { + return dv + __lddv2 * __lddv1 * z + __lddv1 * y + x; + } + MGARDX_CONT_EXEC + T *operator()(IDX y, IDX x) { return dv + __lddv1 * y + x; } + MGARDX_CONT_EXEC + T *operator()(IDX x) { return dv + x; } + + MGARDX_EXEC void offset(SIZE idx[D]) { + ptr_offset += calc_offset(idx); + dv += calc_offset(idx); + } + + MGARDX_EXEC void offset_3d(IDX z, IDX y, IDX x) { + ptr_offset += __lddv2 * __lddv1 * z + __lddv1 * y + x; + dv += __lddv2 * __lddv1 * z + __lddv1 * y + x; + } + MGARDX_EXEC void offset_2d(IDX y, IDX x) { + ptr_offset += __lddv1 * y + x; + dv += __lddv1 * y + x; + } + MGARDX_EXEC void offset_1d(IDX x) { + ptr_offset += x; + dv += x; + } + + MGARDX_EXEC void reset_offset() { + dv -= ptr_offset; + ptr_offset = 0; + } + + MGARDX_CONT_EXEC + bool isNull() { return dv == nullptr; } + + using DataType = T; + using DevType = DeviceType; + static const DIM NumDims = D; + +private: + T *dv; // device pointer + T *v; // host pointer + bool has_host_pointer; + + SIZE __ldvs[D]; + SIZE __shape[D]; + + DIM projected_dim_fastest; + DIM projected_dim_medium; + DIM projected_dim_slowest; + + SIZE __lddv1; + SIZE __lddv2; + + SIZE ptr_offset; + + MGARDX_CONT_EXEC + SIZE calc_offset(SIZE idx[D]) { + SIZE curr_stride = 1; + SIZE offset = 0; + for (int d = D - 1; d >= 0; d--) { + offset += idx[d] * curr_stride; + curr_stride *= __ldvs[d]; + } + return offset; + } +}; + +template +MGARDX_CONT_EXEC void +SubArray::initialize() { + dv = nullptr; + v = nullptr; + has_host_pointer = false; + for (DIM d = 0; d < D; d++) { + __ldvs[d] = 1; + __shape[d] = 1; + } + + projected_dim_fastest = D - 1; + projected_dim_medium = D - 2; + projected_dim_slowest = D - 3; + __lddv1 = 1; + __lddv2 = 1; + + ptr_offset = 0; +} + +template +MGARDX_CONT_EXEC SubArray::SubArray() { + initialize(); +} + +template +MGARDX_CONT SubArray::SubArray( + Array &array) { + initialize(); + dv = array.data(); + for (DIM d = 0; d < D; d++) { + __shape[d] = array.shape(d); + __ldvs[d] = array.ld(d); + } + __lddv1 = __ldvs[D - 1]; + if (D > 1) + __lddv2 = __ldvs[D - 2]; + if (array.hasHostAllocation()) { + v = array.dataHost(); + has_host_pointer = true; + } +} + +// TODO: update shape +template +MGARDX_CONT +SubArray::SubArray(std::vector shape, + T *dv) { + initialize(); + this->dv = dv; + for (DIM d = 0; d < D; d++) { + this->__shape[d] = shape[d]; + this->__ldvs[d] = shape[d]; + } + __lddv1 = __ldvs[D - 1]; + if (D > 1) + __lddv2 = __ldvs[D - 2]; +} + +template +MGARDX_CONT SubArray<1, T, DeviceType> +SubArray::Linearize() { + SubArray<1, T, DeviceType> subArray; + if (!Pitched) { + SIZE linearized_shape = 1; + for (DIM d = 0; d < D; d++) + linearized_shape *= this->__shape[d]; + subArray.setData(data()); + subArray.setShape(0, linearized_shape); + subArray.setLd(0, linearized_shape); + subArray.project(0, 1, 2); + + if (has_host_pointer) { + subArray.setDataHost(dataHost()); + } + } else { + throw std::runtime_error("Linearized pitched SubArray not implemented!"); + } + return subArray; +} + +template +MGARDX_CONT SubArray<3, T, DeviceType> +SubArray::Slice3D(DIM d2, DIM d1, DIM d0) { + // d2 is slowest dim. + // d0 is fastest dim. + if (D < 3) { + throw std::runtime_error("calling Slice3D on SubArray with " + + std::to_string(D) + "D data."); + } + SubArray<3, T, DeviceType> subArray; + subArray.setShape(2, __shape[d0]); + subArray.setShape(1, __shape[d1]); + subArray.setShape(0, __shape[d2]); + subArray.setData(dv); + subArray.setLd(2, __ldvs[d0]); + subArray.setLd(1, __ldvs[d1]); + subArray.setLd(0, __ldvs[d2]); + subArray.project(d2, d1, d0); + + if (has_host_pointer) { + subArray.setDataHost(v); + } + return subArray; +} + +template +MGARDX_CONT void +SubArray::offset(std::vector idx) { + if (idx.size() < D) { + std::cerr << log::log_err << "SubArray::resize insufficient idx length.\n"; + } + // In case shape.size > D; + DIM skip_dim = idx.size() - D; + SIZE _idx[D]; + for (DIM d = 0; d < D; d++) + _idx[d] = idx[skip_dim + d]; + dv += calc_offset(_idx); +} + +template +MGARDX_CONT void +SubArray::resize(std::vector shape) { + if (shape.size() < D) { + std::cerr << log::log_err + << "SubArray::resize insufficient shape length.\n"; + } + // In case shape.size > D; + DIM skip_dim = shape.size() - D; + for (DIM d = 0; d < D; d++) { + __shape[d] = shape[skip_dim + d]; + } +} + +template +MGARDX_CONT void +SubArray::offset_dim(DIM dim, + SIZE offset_value) { + if (dim >= D) + return; + SIZE idx[D]; + for (DIM d = 0; d < D; d++) { + idx[d] = 0; + } + idx[dim] = offset_value; + dv += calc_offset(idx); +} + +template +MGARDX_CONT void +SubArray::resize(DIM dim, SIZE new_size) { + if (dim >= D) + return; + __shape[dim] = new_size; +} + +template +MGARDX_CONT void SubArray::project( + DIM dim_slowest, DIM dim_medium, DIM dim_fastest) { + projected_dim_slowest = dim_slowest; + projected_dim_medium = dim_medium; + projected_dim_fastest = dim_fastest; + // Callers routinely pass the 3D triple (0, 1, 2) or (D-3, D-2, D-1); on a + // 1D or 2D SubArray the leading dimensions wrap or fall off the end, so + // clamp all three into range before indexing __ldvs. + if (projected_dim_fastest >= D) + projected_dim_fastest = D - 1; + if (projected_dim_slowest >= D) + projected_dim_slowest = 0; + if (projected_dim_medium >= D) + projected_dim_medium = 0; + __lddv1 = 1, __lddv2 = 1; + for (int d = projected_dim_fastest; d > projected_dim_medium; d--) { + __lddv1 *= __ldvs[d]; + } + for (int d = projected_dim_medium; d > projected_dim_slowest; d--) { + __lddv2 *= __ldvs[d]; + } +} + +} // namespace mgard_x_160 +#endif diff --git a/include/mgard-x-160/RuntimeX/DataStructures/SubArrayCopy.hpp b/include/mgard-x-160/RuntimeX/DataStructures/SubArrayCopy.hpp new file mode 100644 index 000000000..d8a7702ff --- /dev/null +++ b/include/mgard-x-160/RuntimeX/DataStructures/SubArrayCopy.hpp @@ -0,0 +1,54 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_SUBARRAY_COPY_HPP +#define MGARD_X_160_SUBARRAY_COPY_HPP + +namespace mgard_x_160 { + +template +class CompatibleDeviceType { + using DeviceType = std::conditional< + std::is_same::value && + std::is_same::value, + SERIAL, + std::conditional< + std::is_same::value || + std::is_same::value, + CUDA, + std::conditional< + std::is_same::value || + std::is_same::value, + HIP, + std::conditional::value || + std::is_same::value, + KOKKOS, NONE>>>>; +}; + +template +void SubArrayCopy1D(SubArrayType1 subArray1, SubArrayType2 subArray2, + SIZE count, int queue_idx) { + using DeviceType = typename CompatibleDeviceType< + typename SubArrayType1::DevType, + typename SubArrayType2::DevType>::DeviceType; + MemoryManager::Copy1D(subArray1.data(), subArray2.data(), count, + queue_idx); +} + +template +void SubArrayCopyND(SubArrayType1 subArray1, SubArrayType2 subArray2, + SIZE count1, SIZE count2, int queue_idx) { + using DeviceType = typename CompatibleDeviceType< + typename SubArrayType1::DevType, + typename SubArrayType2::DevType>::DeviceType; + MemoryManager::CopyND(subArray1.data(), subArray1.getLd(0), + subArray2.data(), subArray2.getLd(0), + count1, count2, queue_idx); +} + +} // namespace mgard_x_160 +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/DataTypes.h b/include/mgard-x-160/RuntimeX/DataTypes.h new file mode 100644 index 000000000..82eb1cf14 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/DataTypes.h @@ -0,0 +1,137 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_RUNTIME_X_DATA_TYPES_H +#define MGARD_X_160_RUNTIME_X_DATA_TYPES_H + +// Types defined here are for RuntimeX + +#if defined __CUDACC__ +#define MGARDX_COMPILE_CUDA +#elif defined __HIPCC__ +#define MGARDX_COMPILE_HIP +#elif defined SYCL_LANGUAGE_VERSION +#define MGARDX_COMPILE_SYCL +#else +#define MGARDX_COMPILE_SERIAL +#if MGARD_ENABLE_OPENMP +#define MGARDX_COMPILE_OPENMP +#endif +#endif + +#include + +#if defined MGARDX_COMPILE_KOKKOS +#include "Kokkos_Core.hpp" +#endif + +namespace mgard_x_160 { + +#if defined(__CUDACC__) // NVCC +#define MGARDX_ALIGN(n) __align__(n) +#elif defined(__GNUC__) // GCC +#define MGARDX_ALIGN(n) __attribute__((aligned(n))) +#elif defined(_MSC_VER) // MSVC +#define MGARDX_ALIGN(n) __declspec(align(n)) +#else +#error "Please provide a definition for MY_ALIGN macro for your host compiler!" +#endif + +#if defined(MGARDX_COMPILE_SERIAL) || defined(MGARDX_COMPILE_OPENMP) +#define MGARDX_CONT __inline__ +#define MGARDX_KERL +#define MGARDX_EXEC __inline__ +#define MGARDX_CONT_EXEC __inline__ +#define MGARDX_MANAGED +#endif + +#ifdef MGARDX_COMPILE_CUDA +#define MGARDX_CONT __host__ __inline__ +#define MGARDX_KERL __global__ +#define MGARDX_EXEC __device__ __forceinline__ +#define MGARDX_CONT_EXEC __host__ __device__ __forceinline__ +#define MGARDX_MANAGED __managed__ +#endif + +#ifdef MGARDX_COMPILE_HIP +#define MGARDX_CONT __host__ __inline__ +#define MGARDX_KERL __global__ +#define MGARDX_EXEC __device__ __inline__ +#define MGARDX_CONT_EXEC __host__ __device__ __inline__ +#define MGARDX_MANAGED __managed__ +#endif + +#ifdef MGARDX_COMPILE_SYCL +#define MGARDX_CONT __inline__ +#define MGARDX_KERL +#define MGARDX_EXEC __inline__ +#define MGARDX_CONT_EXEC __inline__ +#define MGARDX_MANAGED +#endif + +#if defined MGARDX_COMPILE_KOKKOS +#define MGARDX_CONT __inline__ +#define MGARDX_KERL +#define MGARDX_EXEC __inline__ +#define MGARDX_CONT_EXEC KOKKOS_INLINE_FUNCTION +#define MGARDX_MANAGED +#endif + +#define MGARDX_WARP_SIZE 32 +#define MGARDX_MAX_NUM_WARPS_PER_TB 32 +#define MGARDX_NUM_SMs 68 + +// 16 Async queues + 1 Sync queue +#define MGARDX_NUM_QUEUES 17 +#define MGARDX_NUM_ASYNC_QUEUES 16 +#define MGARDX_SYNCHRONIZED_QUEUE 16 +#define SIZE_MAX_VALUE 4294967295 + +#define COPY 0 +#define ADD 1 +#define SUBTRACT 2 + +class Device {}; +class SERIAL : public Device {}; +class OPENMP : public Device {}; +class CUDA : public Device {}; +class HIP : public Device {}; +class SYCL : public Device {}; +class NONE : public Device {}; + +enum class device_type : uint8_t { + AUTO, + SERIAL, + OPENMP, + CUDA, + HIP, + SYCL, + NONE +}; + +enum class cpu_parallelization_mode : uint8_t { INTRA_BLOCK, INTER_BLOCK }; + +#if defined MGARDX_COMPILE_KOKKOS +using KOKKOS = Kokkos::DefaultExecutionSpace; +#else +using KOKKOS = NONE; +#endif + +using IDX = uint64_t; +using ATOMIC_IDX = unsigned long long int; +using SIZE = uint64_t; +using DIM = uint8_t; +using QUANTIZED_INT = int64_t; +using QUANTIZED_UNSIGNED_INT = uint64_t; +using HUFFMAN_CODE = uint64_t; +using SERIALIZED_TYPE = unsigned char; +using Byte = unsigned char; +using OPTION = int8_t; +using THREAD_IDX = unsigned int; +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapter.h b/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapter.h new file mode 100644 index 000000000..3378a6869 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapter.h @@ -0,0 +1,354 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ +#ifndef MGARD_X_160_DEVICE_ADAPTER_H +#define MGARD_X_160_DEVICE_ADAPTER_H + +#include "../DataStructures/SubArray.hpp" +#include "../DataTypes.h" +#include "../Functors/Functor.h" +#include "../Tasks/Task.h" +#include "../Utilities/Log.h" + +#include + +namespace mgard_x_160 { + +struct ExecutionReturn { + bool success = true; + double execution_time = std::numeric_limits::max(); +}; + +template struct SyncBlock { + MGARDX_EXEC static void Sync(); +}; + +template struct SyncGrid { + MGARDX_EXEC static void Sync(); +}; + +#define AtomicSystemScope 0 +#define AtomicDeviceScope 1 +#define AtomicBlockScope 2 + +#define AtomicGlobalMemory 0 +#define AtomicSharedMemory 1 + +#define RESOURCE_ENOUGH 0 +#define THREADBLOCK_TOO_LARGE 1 +#define SHARED_MEMORY_TOO_LARGE 2 + +template +struct Atomic { + MGARDX_EXEC static T Min(T *result, T value); + MGARDX_EXEC static T Max(T *result, T value); + MGARDX_EXEC static T Add(T *result, T value); + MGARDX_EXEC static T Or(T *result, T value); +}; + +// based on de Bruijn sequence: +// 0000001000011000101000111001001011001101001111010101110110111111 +// = 151050438420815295 +static const int DeBruijnSubString[64] = { + 0, 1, 2, 4, 8, 16, 33, 3, 6, 12, 24, 49, 34, 5, 10, 20, + 40, 17, 35, 7, 14, 28, 57, 50, 36, 9, 18, 37, 11, 22, 44, 25, + + 51, 38, 13, 26, 52, 41, 19, 39, 15, 30, 61, 58, 53, 42, 21, 43, + + 23, 46, 29, 59, 54, 45, 27, 55, 47, 31, 63, 62, 60, 56, 48, 32}; + +static const int MultiplyDeBruijnBitPosition[64] = { + 0, 1, 2, 7, 3, 13, 8, 19, 4, 25, 14, 28, 9, 34, 20, 40, + + 5, 17, 26, 38, 15, 46, 29, 48, 10, 31, 35, 54, 21, 50, 41, 57, + + 63, 6, 12, 18, 24, 27, 33, 39, 16, 37, 45, 47, 30, 53, 49, 56, + + 62, 11, 23, 32, 36, 44, 52, 55, 61, 22, 43, 51, 60, 42, 59, 58}; + +template struct Math { + template MGARDX_EXEC static T Min(T a, T b); + template MGARDX_EXEC static T Max(T a, T b); + + MGARDX_EXEC static int ffs(unsigned int a); + MGARDX_EXEC static int ffsll(long long unsigned int a); + MGARDX_EXEC static uint64_t binary2negabinary(const int64_t x); + MGARDX_EXEC static uint32_t binary2negabinary(const int32_t x); + MGARDX_EXEC static int64_t negabinary2binary(const uint64_t x); + MGARDX_EXEC static int32_t negabinary2binary(const uint32_t x); +}; + +template +struct BlockReduce { + MGARDX_EXEC + static void Sum(T intput, T &output); + MGARDX_EXEC + static void Max(T intput, T &output); +}; + +template +struct BlockBitTranspose { + MGARDX_EXEC + static void Transpose(T_org *v, T_trans *tv, SIZE b, SIZE B, SIZE IdX, + SIZE IdY); +}; + +template +struct WarpBitTranspose { + MGARDX_EXEC + static void Transpose(T_org *v, SIZE inc_v, T_trans *tv, SIZE inc_tv, + SIZE LaneId); +}; + +template +struct BlockErrorCollect { + MGARDX_EXEC + static void Collect(T *v, T_error *temp, T_error *errors, SIZE num_elems, + SIZE num_bitplanes, SIZE IdX, SIZE IdY); +}; + +template +struct WarpErrorCollect { + MGARDX_EXEC + static void Collect(T *v, T_error *errors, SIZE LaneId); +}; + +template class DeviceSpecification { +public: + MGARDX_CONT + DeviceSpecification() {} + int NumDevices; + int *MaxSharedMemorySize; + int *WarpSize; + int *NumSMs; + int *ArchitectureGeneration; + int *MaxNumThreadsPerSM; + int *MaxNumThreadsPerTB; + size_t *AvailableMemory; + std::string *DeviceNames; +}; + +template class DeviceQueues { +public: + MGARDX_CONT + DeviceQueues() {} + + template + MGARDX_CONT QueueType GetQueue(int dev_id, SIZE queue_id) {} + + MGARDX_CONT + void SyncQueue(int dev_id, SIZE queue_id) {} + + MGARDX_CONT + void SyncAllQueues(int dev_id) {} + + MGARDX_CONT + ~DeviceQueues() {} +}; + +template class DeviceAdapter { +public: + MGARDX_CONT + DeviceAdapter(){}; + MGARDX_CONT + int IsResourceEnough() { return false; } + MGARDX_CONT + ExecutionReturn Execute(){}; +}; + +template class DeviceLauncher { +public: + template + MGARDX_CONT static void Execute(KernelType kernel); +}; + +// --------------------------------------------------------------------------- +// Portable sub-group abstraction. A "sub-group" is a set of lanes that run in +// lockstep and can exchange values (shfl), vote (ballot -> mask), find the +// first set lane (ffs, 1-indexed), and barrier (sync). It lets a cooperative +// kernel be written once and lowered to each backend's native primitives: +// SubGroup -> 32-lane warp SubGroup -> wavefront +// SubGroup -> sycl::sub_group SubGroup -> size 1 +// Each backend specializes SubGroup in its DeviceAdapter file. +// +// SubGroupScalar is the size-1 group: every collective is an identity, so a +// cooperative kernel run through it degenerates to the plain serial algorithm. +// It is what the SERIAL/OPENMP specializations are, and what any +// 1-thread-per-item (non-cooperative) kernel should use on ANY backend. +struct SubGroupScalar { + using mask_t = uint32_t; + static constexpr int size() { return 1; } + MGARDX_EXEC int lane() const { return 0; } + MGARDX_EXEC mask_t full_mask() const { return 1u; } + template MGARDX_EXEC T shfl(T v, int) const { return v; } + MGARDX_EXEC mask_t ballot(int pred) const { return pred ? 1u : 0u; } + MGARDX_EXEC int ffs(mask_t m) const { return m ? 1 : 0; } + MGARDX_EXEC void sync() const {} +}; + +// Primary template -- each backend provides an explicit specialization in its +// DeviceAdapter file. Left undefined so an unsupported backend fails loudly. +template struct SubGroup; + +template struct KeyValueComparator { + bool operator()(std::pair a, std::pair b) const { + return a.first < b.first; + } +}; + +template class DeviceCollective { +public: + template MGARDX_CONT DeviceCollective(){}; + template + MGARDX_CONT static void Sum(SIZE n, SubArray<1, T, DeviceType> v, + SubArray<1, T, DeviceType> result, + Array<1, Byte, DeviceType> &workspace, + bool workspace_allocated, int queue_idx); + + template + MGARDX_CONT static void AbsMax(SIZE n, SubArray<1, T, DeviceType> v, + SubArray<1, T, DeviceType> result, + Array<1, Byte, DeviceType> &workspace, + bool workspace_allocated, int queue_idx); + + template + MGARDX_CONT static void AbsMin(SIZE n, SubArray<1, T, DeviceType> v, + SubArray<1, T, DeviceType> result, + Array<1, Byte, DeviceType> &workspace, + bool workspace_allocated, int queue_idx); + + template + MGARDX_CONT static void SquareSum(SIZE n, SubArray<1, T, DeviceType> v, + SubArray<1, T, DeviceType> result, + Array<1, Byte, DeviceType> &workspace, + bool workspace_allocated, int queue_idx); + + template + MGARDX_CONT static void + ScanSumInclusive(SIZE n, SubArray<1, T, DeviceType> v, + SubArray<1, T, DeviceType> result, + Array<1, Byte, DeviceType> &workspace, + bool workspace_allocated, int queue_idx); + + template + MGARDX_CONT static void + ScanSumExclusive(SIZE n, SubArray<1, T, DeviceType> v, + SubArray<1, T, DeviceType> result, + Array<1, Byte, DeviceType> &workspace, + bool workspace_allocated, int queue_idx); + + template + MGARDX_CONT static void ScanSumExtended(SIZE n, SubArray<1, T, DeviceType> v, + SubArray<1, T, DeviceType> result, + Array<1, Byte, DeviceType> &workspace, + bool workspace_allocated, + int queue_idx); + + template + MGARDX_CONT static void SortByKey(SIZE n, + SubArray<1, KeyT, DeviceType> in_keys, + SubArray<1, ValueT, DeviceType> in_values, + SubArray<1, KeyT, DeviceType> out_keys, + SubArray<1, ValueT, DeviceType> out_values, + Array<1, Byte, DeviceType> &workspace, + bool workspace_allocated, int queue_idx); +}; + +template class MemoryManager { +public: + MGARDX_CONT + MemoryManager(){}; + + template + MGARDX_CONT static void Malloc1D(T *&ptr, SIZE n, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + + template + MGARDX_CONT static void MallocND(T *&ptr, SIZE n1, SIZE n2, SIZE &ld, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + + template + MGARDX_CONT static void Free(T *ptr, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + + template + MGARDX_CONT static void Copy1D(T *dst_ptr, const T *src_ptr, SIZE n, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + + template + MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, + SIZE src_ld, SIZE n1, SIZE n2, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + + template + MGARDX_CONT static void MallocHost(T *&ptr, SIZE n, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + + template + MGARDX_CONT static void FreeHost(T *ptr, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + + template + MGARDX_CONT static void Memset1D(T *ptr, SIZE n, int value, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + + template + MGARDX_CONT static void MemsetND(T *ptr, SIZE ld, SIZE n1, SIZE n2, int value, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + + template MGARDX_CONT static bool IsDevicePointer(T *ptr); + + template MGARDX_CONT static int GetPointerDevice(T *ptr); + + template MGARDX_CONT static bool CheckHostRegister(T *ptr); + + template MGARDX_CONT static void HostRegister(T *ptr, SIZE n); + + template MGARDX_CONT static void HostUnregister(T *ptr); +}; + +template class DeviceRuntime { +public: + MGARDX_CONT + DeviceRuntime() {} + + MGARDX_CONT static void Initialize() {} + + MGARDX_CONT static void Destroy() {} + + MGARDX_CONT static void SelectDevice(SIZE dev_id) {} + + MGARDX_CONT static int GetDevice() {} + + template + MGARDX_CONT static QueueType GetQueue(SIZE queue_id) {} + + MGARDX_CONT static void SyncQueue(SIZE queue_id) {} + + MGARDX_CONT static void SyncAllQueues() {} + + MGARDX_CONT static void SyncDevice() {} + + MGARDX_CONT static std::string GetDeviceName() { return ""; } + + MGARDX_CONT + ~DeviceRuntime() {} + + static DeviceQueues queues; + static bool SyncAllKernelsAndCheckErrors; + static DeviceSpecification DeviceSpecs; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h b/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h new file mode 100644 index 000000000..2e0990af6 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h @@ -0,0 +1,2792 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../Utilities/Exceptions.h" +#include "DeviceAdapter.h" +#include +#include +#include +// #include + +#if defined(CUB_VERSION) && CUB_VERSION >= 300000 +#include +#endif +#include +#include +#include +#include +#include +#include + +// using namespace nvcuda; +namespace cg = cooperative_groups; + +#ifndef MGARD_X_160_DEVICE_ADAPTER_CUDA_H +#define MGARD_X_160_DEVICE_ADAPTER_CUDA_H + +template struct SharedMemory { + MGARDX_EXEC operator T *() { + extern __shared__ int __smem[]; + return (T *)__smem; + } + + MGARDX_EXEC operator const T *() const { + extern __shared__ int __smem[]; + return (T *)__smem; + } +}; + +static __device__ __inline__ uint32_t __mywarpid() { + uint32_t warpid = threadIdx.y; + // asm volatile("mov.u32 %0, %%warpid;" : "=r"(warpid)); + return warpid; +} + +static __device__ __inline__ uint32_t __mylaneid() { + uint32_t laneid = threadIdx.x; + // asm volatile("mov.u32 %0, %%laneid;" : "=r"(laneid)); + return laneid; +} + +MGARDX_EXEC static float atomicMax(float *address, float val) { + int *address_as_i = (int *)address; + int old = *address_as_i, assumed; + do { + assumed = old; + old = ::atomicCAS(address_as_i, assumed, + __float_as_int(::fmaxf(val, __int_as_float(assumed)))); + } while (assumed != old); + return __int_as_float(old); +} + +MGARDX_EXEC static double atomicMax(double *address, double val) { + unsigned long long int *address_as_i = (unsigned long long int *)address; + unsigned long long int old = *address_as_i, assumed; + do { + assumed = old; + old = ::atomicCAS(address_as_i, assumed, + (unsigned long long int)::fmax(val, (double)assumed)); + } while (assumed != old); + return (double)old; +} + +MGARDX_EXEC static uint64_t atomicAdd(uint64_t *address, uint64_t val) { + unsigned long long int *address_as_i = (unsigned long long int *)address; + unsigned long long int old = *address_as_i, assumed; + do { + assumed = old; + old = ::atomicCAS(address_as_i, assumed, + (unsigned long long int)(val + (uint64_t)assumed)); + } while (assumed != old); + return (uint64_t)old; +} + +MGARDX_EXEC static uint64_t atomicAdd_block(uint64_t *address, uint64_t val) { + unsigned long long int *address_as_i = (unsigned long long int *)address; + unsigned long long int old = *address_as_i, assumed; + do { + assumed = old; + old = ::atomicCAS(address_as_i, assumed, + (unsigned long long int)(val + (uint64_t)assumed)); + } while (assumed != old); + return (uint64_t)old; +} + +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ < 600 +MGARDX_EXEC static double atomicAdd(double *address, double val) { + unsigned long long int *address_as_ull = (unsigned long long int *)address; + unsigned long long int old = *address_as_ull, assumed; + + do { + assumed = old; + old = atomicCAS(address_as_ull, assumed, + __double_as_longlong(val + __longlong_as_double(assumed))); + + // Note: uses integer comparison to avoid hang in case of NaN (since NaN != + // NaN) + } while (assumed != old); + + return __longlong_as_double(old); +} +#endif + +namespace mgard_x_160 { + +template +inline void ErrorAsyncCheckTask(cudaError_t code, TaskType &task, + bool abort = true) { + if (code != cudaSuccess) { + std::string msg = std::string(cudaGetErrorString(code)) + + " while executing " + task.GetFunctorName().c_str() + + " with CUDA (Async-check)"; + if (abort) + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; + } +} + +template +inline void ErrorSyncCheckTask(cudaError_t code, TaskType &task, + bool abort = true) { + if (code != cudaSuccess) { + std::string msg = std::string(cudaGetErrorString(code)) + + " while executing " + task.GetFunctorName().c_str() + + " with CUDA (Sync-check)"; + if (abort) + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; + } +} + +inline void ErrorAsyncCheck(cudaError_t code, std::string task, + bool abort = true) { + if (code != cudaSuccess) { + std::string msg = std::string(cudaGetErrorString(code)) + + " while executing " + task.c_str() + + " with CUDA (Async-check)"; + if (abort) + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; + } +} + +inline void ErrorSyncCheck(cudaError_t code, std::string task, + bool abort = true) { + if (code != cudaSuccess) { + std::string msg = std::string(cudaGetErrorString(code)) + + " while executing " + task.c_str() + + " with CUDA (Sync-check)"; + if (abort) + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; + } +} + +#define gpuErrchk(ans) \ + { gpuAssert((ans), __FILE__, __LINE__); } + +inline void gpuAssert(cudaError_t code, const char *file, int line, + bool abort = true) { + if (code != cudaSuccess) { + if (abort) + throw ProcessingException(std::string("GPUassert: ") + + cudaGetErrorString(code) + " " + file + " " + + std::to_string(line)); + else + fprintf(stderr, "GPUassert: %s %s %d\n", cudaGetErrorString(code), file, + line); + } +} + +template <> struct SyncBlock { + MGARDX_EXEC static void Sync() { __syncthreads(); } +}; + +template <> struct SyncGrid { + MGARDX_EXEC static void Sync() { cg::this_grid().sync(); } +}; + +template +struct Atomic { + MGARDX_EXEC static T Min(T *result, T value) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 600 + if constexpr (Scope == AtomicSystemScope) { + return atomicMin_system(result, value); + } else if constexpr (Scope == AtomicDeviceScope) { + return atomicMin(result, value); + } else { + return atomicMin_block(result, value); + } +#else + return atomicMin(result, value); +#endif + } + MGARDX_EXEC static T Max(T *result, T value) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 600 + if constexpr (Scope == AtomicSystemScope) { + return atomicMax_system(result, value); + } else if constexpr (Scope == AtomicDeviceScope) { + return atomicMax(result, value); + } else { + return atomicMax_block(result, value); + } +#else + return atomicMax(result, value); +#endif + } + MGARDX_EXEC static T Add(T *result, T value) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 600 + if constexpr (Scope == AtomicSystemScope) { + return atomicAdd_system(result, value); + } else if constexpr (Scope == AtomicDeviceScope) { + return atomicAdd(result, value); + } else { + return atomicAdd_block(result, value); + } +#else + return atomicAdd(result, value); +#endif + } + MGARDX_EXEC static T Or(T *result, T value) { + // atomicOr is only overloaded for 32-/64-bit unsigned integers, so cast to + // the matching-width unsigned type to stay portable across H = uint32/64. + if constexpr (sizeof(T) == 8) { + using U = unsigned long long int; + U *r = reinterpret_cast(result); + U v = static_cast(value); +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 600 + if constexpr (Scope == AtomicSystemScope) { + return static_cast(atomicOr_system(r, v)); + } else if constexpr (Scope == AtomicDeviceScope) { + return static_cast(atomicOr(r, v)); + } else { + return static_cast(atomicOr_block(r, v)); + } +#else + return static_cast(atomicOr(r, v)); +#endif + } else { + using U = unsigned int; + U *r = reinterpret_cast(result); + U v = static_cast(value); +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 600 + if constexpr (Scope == AtomicSystemScope) { + return static_cast(atomicOr_system(r, v)); + } else if constexpr (Scope == AtomicDeviceScope) { + return static_cast(atomicOr(r, v)); + } else { + return static_cast(atomicOr_block(r, v)); + } +#else + return static_cast(atomicOr(r, v)); +#endif + } + } +}; + +template <> struct Math { + template MGARDX_EXEC static T Min(T a, T b) { return min(a, b); } + template MGARDX_EXEC static T Max(T a, T b) { return max(a, b); } + MGARDX_EXEC static int ffs(unsigned int a) { return __ffs(a); } + MGARDX_EXEC static int ffsll(long long unsigned int a) { return __ffsll(a); } + MGARDX_EXEC + static uint64_t binary2negabinary(const int64_t x) { + return (x + (uint64_t)0xaaaaaaaaaaaaaaaaull) ^ + (uint64_t)0xaaaaaaaaaaaaaaaaull; + } + + MGARDX_EXEC + static uint32_t binary2negabinary(const int32_t x) { + return (x + (uint32_t)0xaaaaaaaau) ^ (uint32_t)0xaaaaaaaau; + } + + MGARDX_EXEC + static int64_t negabinary2binary(const uint64_t x) { + return (x ^ 0xaaaaaaaaaaaaaaaaull) - 0xaaaaaaaaaaaaaaaaull; + } + + MGARDX_EXEC + static int32_t negabinary2binary(const uint32_t x) { + return (x ^ 0xaaaaaaaau) - 0xaaaaaaaau; + } +}; + +template +MGARDX_KERL void CudaKernel(Task task, THREAD_IDX blockz_offset, + THREAD_IDX blocky_offset, + THREAD_IDX blockx_offset) { + Byte *shared_memory = SharedMemory(); + task.GetFunctor().Init(task.GetGridDimZ(), task.GetGridDimY(), + task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX(), + blockz_offset + blockIdx.z, blocky_offset + blockIdx.y, + blockx_offset + blockIdx.x, threadIdx.z, threadIdx.y, + threadIdx.x, shared_memory); + + task.GetFunctor().Operation1(); + SyncBlock::Sync(); + task.GetFunctor().Operation2(); + SyncBlock::Sync(); + task.GetFunctor().Operation3(); + SyncBlock::Sync(); + task.GetFunctor().Operation4(); + SyncBlock::Sync(); + task.GetFunctor().Operation5(); + SyncBlock::Sync(); + task.GetFunctor().Operation6(); + SyncBlock::Sync(); + task.GetFunctor().Operation7(); + SyncBlock::Sync(); + task.GetFunctor().Operation8(); + SyncBlock::Sync(); + task.GetFunctor().Operation9(); + SyncBlock::Sync(); + task.GetFunctor().Operation10(); + // SyncBlock::Sync(); + // task.GetFunctor().Operation11(); + // SyncBlock::Sync(); + // task.GetFunctor().Operation12(); + // SyncBlock::Sync(); + // task.GetFunctor().Operation13(); + // SyncBlock::Sync(); + // task.GetFunctor().Operation14(); + // SyncBlock::Sync(); + // task.GetFunctor().Operation15(); + // SyncBlock::Sync(); + // task.GetFunctor().Operation16(); + // SyncBlock::Sync(); + // task.GetFunctor().Operation17(); + // SyncBlock::Sync(); + // task.GetFunctor().Operation18(); + // SyncBlock::Sync(); + // task.GetFunctor().Operation19(); + // SyncBlock::Sync(); + // task.GetFunctor().Operation20(); +} + +template +MGARDX_KERL void CudaIterKernel(Task task, THREAD_IDX blockz_offset, + THREAD_IDX blocky_offset, + THREAD_IDX blockx_offset) { + Byte *shared_memory = SharedMemory(); + + task.GetFunctor().Init(task.GetGridDimZ(), task.GetGridDimY(), + task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX(), + blockz_offset + blockIdx.z, blocky_offset + blockIdx.y, + blockx_offset + blockIdx.x, threadIdx.z, threadIdx.y, + threadIdx.x, shared_memory); + + task.GetFunctor().Operation1(); + SyncBlock::Sync(); + + task.GetFunctor().Operation2(); + SyncBlock::Sync(); + + while (task.GetFunctor().LoopCondition1()) { + task.GetFunctor().Operation3(); + SyncBlock::Sync(); + task.GetFunctor().Operation4(); + SyncBlock::Sync(); + task.GetFunctor().Operation5(); + SyncBlock::Sync(); + task.GetFunctor().Operation6(); + SyncBlock::Sync(); + } + + task.GetFunctor().Operation7(); + SyncBlock::Sync(); + task.GetFunctor().Operation8(); + SyncBlock::Sync(); + task.GetFunctor().Operation9(); + SyncBlock::Sync(); + task.GetFunctor().Operation10(); + SyncBlock::Sync(); + + while (task.GetFunctor().LoopCondition2()) { + task.GetFunctor().Operation11(); + SyncBlock::Sync(); + task.GetFunctor().Operation12(); + SyncBlock::Sync(); + task.GetFunctor().Operation13(); + SyncBlock::Sync(); + task.GetFunctor().Operation14(); + SyncBlock::Sync(); + } + + task.GetFunctor().Operation15(); + SyncBlock::Sync(); + task.GetFunctor().Operation16(); + SyncBlock::Sync(); + task.GetFunctor().Operation17(); + SyncBlock::Sync(); +} + +template +MGARDX_KERL void CudaHuffmanCLCustomizedKernel(Task task) { + Byte *shared_memory = SharedMemory(); + + task.GetFunctor().Init(gridDim.z, gridDim.y, gridDim.x, blockDim.z, + blockDim.y, blockDim.x, blockIdx.z, blockIdx.y, + blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, + shared_memory); + + task.GetFunctor().Operation1(); + SyncGrid::Sync(); + while (task.GetFunctor().LoopCondition1()) { + task.GetFunctor().Operation2(); + SyncGrid::Sync(); + task.GetFunctor().Operation3(); + SyncGrid::Sync(); + task.GetFunctor().Operation4(); + SyncGrid::Sync(); + task.GetFunctor().Operation5(); + SyncGrid::Sync(); + if (task.GetFunctor().BranchCondition1()) { + task.GetFunctor().Operation6(); + SyncBlock::Sync(); + while (task.GetFunctor().LoopCondition2()) { + task.GetFunctor().Operation7(); + SyncBlock::Sync(); + task.GetFunctor().Operation8(); + SyncBlock::Sync(); + task.GetFunctor().Operation9(); + SyncBlock::Sync(); + } + task.GetFunctor().Operation10(); + SyncGrid::Sync(); + task.GetFunctor().Operation11(); + SyncGrid::Sync(); + } + task.GetFunctor().Operation12(); + SyncGrid::Sync(); + task.GetFunctor().Operation13(); + SyncGrid::Sync(); + task.GetFunctor().Operation14(); + SyncGrid::Sync(); + task.GetFunctor().Operation15(); + SyncGrid::Sync(); + } +} + +#define SINGLE_KERNEL(OPERATION) \ + template \ + MGARDX_KERL void Single_##OPERATION##_Kernel(Task task) { \ + Byte *shared_memory = SharedMemory(); \ + task.GetFunctor().Init(gridDim.z, gridDim.y, gridDim.x, blockDim.z, \ + blockDim.y, blockDim.x, blockIdx.z, blockIdx.y, \ + blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, \ + shared_memory); \ + task.GetFunctor().OPERATION(); \ + } + +SINGLE_KERNEL(Operation1); +SINGLE_KERNEL(Operation2); +SINGLE_KERNEL(Operation3); +SINGLE_KERNEL(Operation4); +SINGLE_KERNEL(Operation5); +SINGLE_KERNEL(Operation6); +SINGLE_KERNEL(Operation7); +SINGLE_KERNEL(Operation8); +SINGLE_KERNEL(Operation9); +SINGLE_KERNEL(Operation10); +SINGLE_KERNEL(Operation11); +SINGLE_KERNEL(Operation12); +SINGLE_KERNEL(Operation13); +SINGLE_KERNEL(Operation14); +SINGLE_KERNEL(Operation15); + +#undef SINGLE_KERNEL + +template MGARDX_KERL void CudaParallelMergeKernel(Task task) { + Byte *shared_memory = SharedMemory(); + + task.GetFunctor().Init(gridDim.z, gridDim.y, gridDim.x, blockDim.z, + blockDim.y, blockDim.x, blockIdx.z, blockIdx.y, + blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, + shared_memory); + + task.GetFunctor().Operation6(); + SyncBlock::Sync(); + while (task.GetFunctor().LoopCondition2()) { + task.GetFunctor().Operation7(); + SyncBlock::Sync(); + task.GetFunctor().Operation8(); + SyncBlock::Sync(); + task.GetFunctor().Operation9(); + SyncBlock::Sync(); + } + task.GetFunctor().Operation10(); +} + +template +MGARDX_KERL void CudaHuffmanCWCustomizedKernel(Task task) { + Byte *shared_memory = SharedMemory(); + + task.GetFunctor().Init(gridDim.z, gridDim.y, gridDim.x, blockDim.z, + blockDim.y, blockDim.x, blockIdx.z, blockIdx.y, + blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, + shared_memory); + + task.GetFunctor().Operation1(); + SyncGrid::Sync(); + task.GetFunctor().Operation2(); + SyncGrid::Sync(); + task.GetFunctor().Operation3(); + SyncGrid::Sync(); + + while (task.GetFunctor().LoopCondition1()) { + task.GetFunctor().Operation4(); + SyncGrid::Sync(); + task.GetFunctor().Operation5(); + SyncGrid::Sync(); + task.GetFunctor().Operation6(); + SyncGrid::Sync(); + task.GetFunctor().Operation7(); + SyncGrid::Sync(); + task.GetFunctor().Operation8(); + SyncGrid::Sync(); + } + task.GetFunctor().Operation9(); + SyncGrid::Sync(); + task.GetFunctor().Operation10(); + SyncGrid::Sync(); +} + +template <> class DeviceSpecification { +public: + MGARDX_CONT + DeviceSpecification() { + int original_device = 0; + cudaGetDevice(&original_device); + cudaGetDeviceCount(&NumDevices); + MaxSharedMemorySize = new int[NumDevices]; + WarpSize = new int[NumDevices]; + NumSMs = new int[NumDevices]; + ArchitectureGeneration = new int[NumDevices]; + MaxNumThreadsPerSM = new int[NumDevices]; + MaxNumThreadsPerTB = new int[NumDevices]; + AvailableMemory = new size_t[NumDevices]; + SupportCooperativeGroups = new bool[NumDevices]; + DeviceNames = new std::string[NumDevices]; + + for (int d = 0; d < NumDevices; d++) { + gpuErrchk(cudaSetDevice(d)); + int maxbytes; + int maxbytesOptIn; + cudaDeviceGetAttribute(&maxbytes, cudaDevAttrMaxSharedMemoryPerBlock, d); + cudaDeviceGetAttribute(&maxbytesOptIn, + cudaDevAttrMaxSharedMemoryPerBlockOptin, d); + MaxSharedMemorySize[d] = std::max(maxbytes, maxbytesOptIn); + cudaDeviceGetAttribute(&WarpSize[d], cudaDevAttrWarpSize, d); + cudaDeviceGetAttribute(&NumSMs[d], cudaDevAttrMultiProcessorCount, d); + cudaDeviceGetAttribute(&MaxNumThreadsPerSM[d], + cudaDevAttrMaxThreadsPerMultiProcessor, d); + cudaDeviceGetAttribute(&MaxNumThreadsPerTB[d], + cudaDevAttrMaxThreadsPerBlock, d); + SupportCooperativeGroups[d] = true; + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, d); + DeviceNames[d] = std::string(prop.name); + } + // This loop's per-device cudaSetDevice(d) calls otherwise leave the + // active CUDA device at NumDevices-1 for the rest of the process, + // silently out of sync with DeviceRuntime::cuda_dev_id (which + // stays at its default of 0 unless explicitly changed). Streams and + // allocations made before the mismatch is next resolved end up tied to + // the wrong device's context, which can surface much later as an + // unrelated illegal memory access. Restore the device that was active + // before this query loop ran. + gpuErrchk(cudaSetDevice(original_device)); + } + + MGARDX_CONT int GetNumDevices() { return NumDevices; } + + MGARDX_CONT int GetMaxSharedMemorySize(int dev_id) { + return MaxSharedMemorySize[dev_id]; + } + + MGARDX_CONT int GetWarpSize(int dev_id) { return WarpSize[dev_id]; } + + MGARDX_CONT int GetNumSMs(int dev_id) { return NumSMs[dev_id]; } + + MGARDX_CONT int GetArchitectureGeneration(int dev_id) { + return ArchitectureGeneration[dev_id]; + } + + MGARDX_CONT int GetMaxNumThreadsPerSM(int dev_id) { + return MaxNumThreadsPerSM[dev_id]; + } + + MGARDX_CONT int GetMaxNumThreadsPerTB(int dev_id) { + return MaxNumThreadsPerTB[dev_id]; + } + + MGARDX_CONT size_t GetAvailableMemory(int dev_id) { + gpuErrchk(cudaSetDevice(dev_id)); + size_t free, total; + cudaMemGetInfo(&free, &total); + AvailableMemory[dev_id] = total; + + return AvailableMemory[dev_id]; + } + + MGARDX_CONT bool SupportCG(int dev_id) { + return SupportCooperativeGroups[dev_id]; + } + + MGARDX_CONT std::string GetDeviceName(int dev_id) { + return DeviceNames[dev_id]; + } + + MGARDX_CONT + ~DeviceSpecification() { + delete[] MaxSharedMemorySize; + delete[] WarpSize; + delete[] NumSMs; + delete[] ArchitectureGeneration; + delete[] MaxNumThreadsPerSM; + delete[] MaxNumThreadsPerTB; + delete[] AvailableMemory; + delete[] SupportCooperativeGroups; + delete[] DeviceNames; + } + + int NumDevices; + int *MaxSharedMemorySize; + int *WarpSize; + int *NumSMs; + int *ArchitectureGeneration; + int *MaxNumThreadsPerSM; + int *MaxNumThreadsPerTB; + size_t *AvailableMemory; + bool *SupportCooperativeGroups; + std::string *DeviceNames; +}; + +template <> class DeviceQueues { +public: + MGARDX_CONT + void Initialize() { + if (!initialized) { + log::dbg("Calling DeviceQueues::Initialize"); + int original_device = 0; + cudaGetDevice(&original_device); + cudaGetDeviceCount(&NumDevices); + streams = new cudaStream_t *[NumDevices]; + for (int d = 0; d < NumDevices; d++) { + gpuErrchk(cudaSetDevice(d)); + streams[d] = new cudaStream_t[MGARDX_NUM_QUEUES]; + for (SIZE i = 0; i < MGARDX_NUM_QUEUES; i++) { + gpuErrchk(cudaStreamCreate(&streams[d][i])); + } + } + // See DeviceSpecification's constructor for why this restore matters: + // otherwise the active device is left at NumDevices-1 here too. + gpuErrchk(cudaSetDevice(original_device)); + initialized = true; + } + } + + MGARDX_CONT + void Destroy() { + if (initialized) { + log::dbg("Calling DeviceQueues::Destroy"); + int original_device = 0; + cudaGetDevice(&original_device); + for (int d = 0; d < NumDevices; d++) { + gpuErrchk(cudaSetDevice(d)); + for (int i = 0; i < MGARDX_NUM_QUEUES; i++) { + gpuErrchk(cudaStreamDestroy(streams[d][i])); + } + delete[] streams[d]; + } + gpuErrchk(cudaSetDevice(original_device)); + delete[] streams; + streams = nullptr; + initialized = false; + } + } + + MGARDX_CONT + DeviceQueues() { Initialize(); } + + MGARDX_CONT cudaStream_t GetQueue(int dev_id, SIZE queue_id) { + Initialize(); + return streams[dev_id][queue_id]; + } + + MGARDX_CONT void SyncQueue(int dev_id, SIZE queue_id) { + Initialize(); + cudaStreamSynchronize(streams[dev_id][queue_id]); + } + + MGARDX_CONT void SyncAllQueues(int dev_id) { + Initialize(); + for (SIZE i = 0; i < MGARDX_NUM_QUEUES; i++) { + gpuErrchk(cudaStreamSynchronize(streams[dev_id][i])); + } + } + + MGARDX_CONT + ~DeviceQueues() {} + + int NumDevices; + cudaStream_t **streams = nullptr; + bool initialized = false; +}; + +extern int cuda_dev_id; +#pragma omp threadprivate(cuda_dev_id) + +// 32-lane warp sub-group (CUDA). Stateless: reads its lane from threadIdx. +template <> struct SubGroup { + using mask_t = unsigned; + static constexpr int size() { return 32; } + __device__ int lane() const { return threadIdx.x & 31; } + __device__ mask_t full_mask() const { return 0xffffffffu; } + template __device__ T shfl(T v, int src) const { + return __shfl_sync(0xffffffffu, v, src); + } + __device__ mask_t ballot(int pred) const { + return __ballot_sync(0xffffffffu, pred); + } + __device__ int ffs(mask_t m) const { return __ffs((int)m); } + __device__ void sync() const { __syncwarp(); } +}; + +template <> class DeviceRuntime { +public: + MGARDX_CONT + DeviceRuntime() {} + + MGARDX_CONT static void Initialize() { queues.Initialize(); } + + MGARDX_CONT static void Finalize() { queues.Destroy(); } + + MGARDX_CONT static int GetDeviceCount() { return DeviceSpecs.NumDevices; } + + MGARDX_CONT static void SelectDevice(SIZE dev_id) { + gpuErrchk(cudaSetDevice(dev_id)); + cuda_dev_id = dev_id; + } + + MGARDX_CONT static int GetDevice() { + gpuErrchk(cudaGetDevice(&cuda_dev_id)); + return cuda_dev_id; + } + + MGARDX_CONT static cudaStream_t GetQueue(SIZE queue_id) { + return queues.GetQueue(cuda_dev_id, queue_id); + } + + MGARDX_CONT static void SyncQueue(SIZE queue_id) { + log::dbg("Calling DeviceRuntime::SyncQueue: " + + std::to_string(queue_id)); + queues.SyncQueue(cuda_dev_id, queue_id); + } + + MGARDX_CONT static void SyncAllQueues() { queues.SyncAllQueues(cuda_dev_id); } + + MGARDX_CONT static void SyncDevice() { + log::dbg("Calling DeviceRuntime::SyncDevice: "); + gpuErrchk(cudaDeviceSynchronize()); + } + + MGARDX_CONT static std::string GetDeviceName() { + return DeviceSpecs.GetDeviceName(cuda_dev_id); + } + + MGARDX_CONT static int GetMaxSharedMemorySize() { + return DeviceSpecs.GetMaxSharedMemorySize(cuda_dev_id); + } + + MGARDX_CONT static int GetWarpSize() { + return DeviceSpecs.GetWarpSize(cuda_dev_id); + } + + MGARDX_CONT static int GetNumSMs() { + return DeviceSpecs.GetNumSMs(cuda_dev_id); + } + + MGARDX_CONT static int GetArchitectureGeneration() { + return DeviceSpecs.GetArchitectureGeneration(cuda_dev_id); + } + + MGARDX_CONT static int GetMaxNumThreadsPerSM() { + return DeviceSpecs.GetMaxNumThreadsPerSM(cuda_dev_id); + } + + MGARDX_CONT static int GetMaxNumThreadsPerTB() { + return DeviceSpecs.GetMaxNumThreadsPerTB(cuda_dev_id); + } + + MGARDX_CONT static size_t GetAvailableMemory() { + return DeviceSpecs.GetAvailableMemory(cuda_dev_id); + } + + MGARDX_CONT static bool SupportCG() { + return DeviceSpecs.SupportCG(cuda_dev_id); + } + + template + MGARDX_CONT static int + GetOccupancyMaxActiveBlocksPerSM(FunctorType functor, int blockSize, + size_t dynamicSMemSize) { + int numBlocks = 0; + + if constexpr (std::is_base_of, FunctorType>::value) { + gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &numBlocks, CudaKernel>, blockSize, + dynamicSMemSize)); + } else if constexpr (std::is_base_of, + FunctorType>::value) { + gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &numBlocks, CudaIterKernel>, blockSize, + dynamicSMemSize)); + } else if constexpr (std::is_base_of, + FunctorType>::value) { + gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &numBlocks, CudaHuffmanCLCustomizedKernel>, + blockSize, dynamicSMemSize)); + } else if constexpr (std::is_base_of, + FunctorType>::value) { + gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &numBlocks, CudaHuffmanCWCustomizedKernel>, + blockSize, dynamicSMemSize)); + } else { + throw std::runtime_error("GetOccupancyMaxActiveBlocksPerSM Error!"); + } + return numBlocks; + } + + template + MGARDX_CONT static void SetMaxDynamicSharedMemorySize(FunctorType functor, + int maxbytes) { + + if constexpr (std::is_base_of, FunctorType>::value) { + gpuErrchk(cudaFuncSetAttribute( + CudaKernel>, + cudaFuncAttributeMaxDynamicSharedMemorySize, maxbytes)); + } else if constexpr (std::is_base_of, + FunctorType>::value) { + gpuErrchk(cudaFuncSetAttribute( + CudaIterKernel>, + cudaFuncAttributeMaxDynamicSharedMemorySize, maxbytes)); + } else if constexpr (std::is_base_of, + FunctorType>::value) { + gpuErrchk(cudaFuncSetAttribute( + CudaHuffmanCLCustomizedKernel>, + cudaFuncAttributeMaxDynamicSharedMemorySize, maxbytes)); + } else if constexpr (std::is_base_of, + FunctorType>::value) { + gpuErrchk(cudaFuncSetAttribute( + CudaHuffmanCWCustomizedKernel>, + cudaFuncAttributeMaxDynamicSharedMemorySize, maxbytes)); + } else { + throw std::runtime_error("SetPreferredSharedMemoryCarveout Error!"); + } + } + + MGARDX_CONT + ~DeviceRuntime() {} + + static DeviceQueues queues; + static bool SyncAllKernelsAndCheckErrors; + static DeviceSpecification DeviceSpecs; + static bool TimingAllKernels; + static bool PrintKernelConfig; +}; + +template <> class MemoryManager { +public: + MGARDX_CONT + MemoryManager(){}; + + template + MGARDX_CONT static void Malloc1D(T *&ptr, SIZE n, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Malloc1D"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + gpuErrchk(cudaMalloc(&ptr, n * sizeof(T))); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void MallocND(T *&ptr, SIZE n1, SIZE n2, SIZE &ld, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MallocND"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + size_t pitch = 0; + gpuErrchk(cudaMallocPitch(&ptr, &pitch, n1 * sizeof(T), (size_t)n2)); + ld = pitch / sizeof(T); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void + MallocManaged1D(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MallocManaged1D"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + gpuErrchk(cudaMallocManaged(&ptr, n * sizeof(T))); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void Free(T *ptr, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Free"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (ptr == nullptr) + return; + gpuErrchk(cudaFree(ptr)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void Copy1D(T *dst_ptr, const T *src_ptr, SIZE n, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Copy1D"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); + gpuErrchk(cudaMemcpyAsync(dst_ptr, src_ptr, n * sizeof(T), + cudaMemcpyDefault, stream)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, + SIZE src_ld, SIZE n1, SIZE n2, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::CopyND"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); + gpuErrchk(cudaMemcpy2DAsync(dst_ptr, dst_ld * sizeof(T), src_ptr, + src_ld * sizeof(T), n1 * sizeof(T), n2, + cudaMemcpyDefault, stream)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void + MallocHost(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MallocHost"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + gpuErrchk(cudaMallocHost(&ptr, n * sizeof(T))); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void FreeHost(T *ptr, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::FreeHost"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (ptr == nullptr) + return; + gpuErrchk(cudaFreeHost(ptr)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void Memset1D(T *ptr, SIZE n, int value, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Memset1D"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); + gpuErrchk(cudaMemsetAsync(ptr, value, n * sizeof(T), stream)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void MemsetND(T *ptr, SIZE ld, SIZE n1, SIZE n2, int value, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MemsetND"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); + gpuErrchk(cudaMemset2DAsync(ptr, ld * sizeof(T), value, n1 * sizeof(T), n2, + stream)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template MGARDX_CONT static bool IsDevicePointer(T *ptr) { + log::dbg("Calling MemoryManager::IsDevicePointer"); + cudaPointerAttributes attr; + cudaPointerGetAttributes(&attr, ptr); + return attr.type == cudaMemoryTypeDevice; + } + + template MGARDX_CONT static int GetPointerDevice(T *ptr) { + log::dbg("Calling MemoryManager::GetPointerDevice"); + cudaPointerAttributes attr; + cudaPointerGetAttributes(&attr, ptr); + return attr.device; + } + + template MGARDX_CONT static bool CheckHostRegister(T *ptr) { + log::dbg("Calling MemoryManager::CheckHostRegister"); + unsigned int flags; + cudaHostGetFlags(&flags, (void *)ptr); + return cudaGetLastError() == cudaSuccess; + } + + template MGARDX_CONT static void HostRegister(T *ptr, SIZE n) { + log::dbg("Calling MemoryManager::HostRegister"); + if (!CheckHostRegister(ptr) && n > 0) { + gpuErrchk(cudaHostRegister((void *)ptr, n * sizeof(T), + cudaHostRegisterPortable)); + } + } + + template MGARDX_CONT static void HostUnregister(T *ptr) { + log::dbg("Calling MemoryManager::HostUnregister"); + if (CheckHostRegister(ptr)) { + gpuErrchk(cudaHostUnregister((void *)ptr)); + } + } +}; + +#define ALIGN_LEFT 0 // for encoding +#define ALIGN_RIGHT 1 // for decoding + +#define Sign_Encoding_Atomic 0 +#define Sign_Encoding_Reduce 1 +#define Sign_Encoding_Ballot 2 + +#define Sign_Decoding_Parallel 0 + +#define BINARY 0 +#define NEGABINARY 1 + +#define Bit_Transpose_Serial_All 0 +#define Bit_Transpose_Parallel_B_Serial_b 1 +#define Bit_Transpose_Parallel_B_Atomic_b 2 +#define Bit_Transpose_Parallel_B_Reduce_b 3 +#define Bit_Transpose_Parallel_B_Ballot_b 4 +#define Bit_Transpose_TCU 5 + +#define Warp_Bit_Transpose_Serial_All 0 +#define Warp_Bit_Transpose_Parallel_B_Serial_b 1 +#define Warp_Bit_Transpose_Serial_B_Atomic_b 2 +#define Warp_Bit_Transpose_Serial_B_Reduce_b 3 +#define Warp_Bit_Transpose_Serial_B_Ballot_b 4 +#define Warp_Bit_Transpose_TCU 5 + +#define Error_Collecting_Disable -1 +#define Error_Collecting_Serial_All 0 +#define Error_Collecting_Parallel_Bitplanes_Serial_Error 1 +#define Error_Collecting_Parallel_Bitplanes_Atomic_Error 2 +#define Error_Collecting_Parallel_Bitplanes_Reduce_Error 3 + +#define Warp_Error_Collecting_Disable -1 +#define Warp_Error_Collecting_Serial_All 0 +#define Warp_Error_Collecting_Parallel_Bitplanes_Serial_Error 1 +#define Warp_Error_Collecting_Serial_Bitplanes_Atomic_Error 2 +#define Warp_Error_Collecting_Serial_Bitplanes_Reduce_Error 3 + +typedef unsigned long long int uint64_cu; + +template +struct BlockReduce { + typedef cub::BlockReduce + BlockReduceType; + using TempStorageType = typename BlockReduceType::TempStorage; + + MGARDX_EXEC + static void Sum(T intput, T &output) { + __shared__ TempStorageType temp_storage; + BlockReduceType blockReduce(temp_storage); + output = blockReduce.Sum(intput); + } + + MGARDX_EXEC + static void Max(T intput, T &output) { + __shared__ TempStorageType temp_storage; + BlockReduceType blockReduce(temp_storage); +#if defined(CUB_VERSION) && CUB_VERSION >= 300000 + // cub::Max() was removed as of CCCL 3.0 (bundled with CUDA 13+). + output = blockReduce.Reduce(intput, ::cuda::maximum<>()); +#else + output = blockReduce.Reduce(intput, cub::Max()); +#endif + } +}; + +template +struct BlockBitTranspose { + + typedef cub::WarpReduce WarpReduceType; + using WarpReduceStorageType = typename WarpReduceType::TempStorage; + + typedef cub::BlockReduce + BlockReduceType; + using BlockReduceStorageType = typename BlockReduceType::TempStorage; + + MGARDX_EXEC + static void Serial_All(T_org *v, T_trans *tv, SIZE b, SIZE B, SIZE IdX, + SIZE IdY) { + if (IdX == 0 && IdY == 0) { + // printf("add-in: %llu %u\n", v, v[0]); + // for (int i = 0; i < b; i++) { + // printf("v: %u\n", v[i]); + // } + for (SIZE B_idx = 0; B_idx < B; B_idx++) { + T_trans buffer = 0; + for (SIZE b_idx = 0; b_idx < b; b_idx++) { + T_trans bit = (v[b_idx] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + if (ALIGN == ALIGN_LEFT) { + buffer += bit << (sizeof(T_trans) * 8 - 1 - b_idx); + // if (B_idx == 0) { + // printf("%u %u %u\n", B_idx, b_idx, bit); + // print_bits(buffer, sizeof(T_trans)*8, false); + // printf("\n"); + // } + } else if (ALIGN == ALIGN_RIGHT) { + buffer += bit << (b - 1 - b_idx); + // if (b_idx == 0) printf("%u %u %u\n", B_idx, b_idx, bit); + } else { + } + // if (j == 0 ) {printf("i %u j %u shift %u bit %u\n", i,j,b-1-j, + // bit); } + } + + // printf("buffer: %u\n", buffer); + + tv[B_idx] = buffer; + } + } + } + + MGARDX_EXEC + static void Parallel_B_Serial_b(T_org *v, T_trans *tv, SIZE b, SIZE B, + SIZE IdX, SIZE IdY) { + if (IdY == 0) { + for (SIZE B_idx = IdX; B_idx < B; B_idx += 32) { + T_trans buffer = 0; + for (SIZE b_idx = 0; b_idx < b; b_idx++) { + T_trans bit = (v[b_idx] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + if (ALIGN == ALIGN_LEFT) { + buffer += bit << sizeof(T_trans) * 8 - 1 - b_idx; + } else if (ALIGN == ALIGN_RIGHT) { + buffer += bit << (b - 1 - b_idx); + // if (b_idx == 0) printf("%u %u %u\n", B_idx, b_idx, bit); + } else { + } + } + tv[B_idx] = buffer; + } + } + } + + MGARDX_EXEC + static void Parallel_B_Atomic_b(T_org *v, T_trans *tv, SIZE b, SIZE B, + SIZE IdX, SIZE IdY) { + if (IdX < b && IdY < B) { + SIZE i = IdX; + for (SIZE B_idx = IdY; B_idx < B; B_idx += 32) { + for (SIZE b_idx = IdX; b_idx < b; b_idx += 32) { + T_trans bit = (v[b_idx] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + T_trans shifted_bit; + if (ALIGN == ALIGN_LEFT) { + shifted_bit = bit << sizeof(T_trans) * 8 - 1 - b_idx; + } else if (ALIGN == ALIGN_RIGHT) { + shifted_bit = bit << b - 1 - b_idx; + } else { + } + T_trans *sum = &(tv[B_idx]); + Atomic::Add( + sum, shifted_bit); + } + } + } + } + + MGARDX_EXEC + static void Parallel_B_Reduction_b(T_org *v, T_trans *tv, SIZE b, SIZE B, + SIZE IdX, SIZE IdY) { + + // __syncthreads(); long long start = clock64(); + + __shared__ WarpReduceStorageType warp_storage[32]; + + SIZE warp_idx = IdY; + SIZE lane_idx = IdX; + SIZE B_idx, b_idx; + T_trans bit = 0; + T_trans shifted_bit = 0; + T_trans sum = 0; + + // __syncthreads(); start = clock64() - start; + // if (threadIdx.y == 0 && threadIdx.x == 0) printf("time0: %llu\n", start); + // __syncthreads(); start = clock64(); + + for (SIZE B_idx = IdY; B_idx < B; B_idx += 32) { + sum = 0; + for (SIZE b_idx = IdX; b_idx < ((b - 1) / 32 + 1) * 32; b_idx += 32) { + shifted_bit = 0; + if (b_idx < b && B_idx < B) { + bit = (v[b_idx] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + if (ALIGN == ALIGN_LEFT) { + shifted_bit = bit << sizeof(T_trans) * 8 - 1 - b_idx; + } else if (ALIGN == ALIGN_RIGHT) { + shifted_bit = bit << b - 1 - b_idx; + } else { + } + } + + // __syncthreads(); start = clock64() - start; + // if (threadIdx.y == 0 && threadIdx.x == 0) printf("time1: %llu\n", + // start); + // __syncthreads(); start = clock64(); + + sum += WarpReduceType(warp_storage[warp_idx]).Sum(shifted_bit); + // if (B_idx == 32) printf("shifted_bit[%u] %u sum %u\n", b_idx, + // shifted_bit, sum); + } + if (lane_idx == 0) { + tv[B_idx] = sum; + } + } + + // __syncthreads(); start = clock64() - start; + // if (threadIdx.y == 0 && threadIdx.x == 0) printf("time2: %llu\n", start); + // __syncthreads(); start = clock64(); + } + + MGARDX_EXEC + static void Parallel_B_Ballot_b(T_org *v, T_trans *tv, SIZE b, SIZE B, + SIZE IdX, SIZE IdY) { + + SIZE warp_idx = IdY; + SIZE lane_idx = IdX; + SIZE B_idx, b_idx; + int bit = 0; + T_trans sum = 0; + + // __syncthreads(); start = clock64() - start; + // if (threadIdx.y == 0 && threadIdx.x == 0) printf("time0: %llu\n", start); + // __syncthreads(); start = clock64(); + + for (SIZE B_idx = IdY; B_idx < B; B_idx += 32) { + sum = 0; + SIZE shift = 0; + for (SIZE b_idx = IdX; b_idx < ((b - 1) / 32 + 1) * 32; b_idx += 32) { + bit = 0; + if (b_idx < b && B_idx < B) { + if (ALIGN == ALIGN_LEFT) { + bit = (v[sizeof(T_trans) * 8 - 1 - b_idx] >> + (sizeof(T_org) * 8 - 1 - B_idx)) & + 1u; + } else if (ALIGN == ALIGN_RIGHT) { + bit = (v[b - 1 - b_idx] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + } else { + } + } + + // __syncthreads(); start = clock64() - start; + // if (threadIdx.y == 0 && threadIdx.x == 0) printf("time1: %llu\n", + // start); + // __syncthreads(); start = clock64(); + sum += ((T_trans)__ballot_sync(0xffffffff, bit)) << shift; + // sum += WarpReduceType(warp_storage[warp_idx]).Sum(shifted_bit); + // if (B_idx == 32) printf("shifted_bit[%u] %u sum %u\n", b_idx, + // shifted_bit, sum); + shift += 32; + } + if (lane_idx == 0) { + tv[B_idx] = sum; + } + } + + // __syncthreads(); start = clock64() - start; + // if (threadIdx.y == 0 && threadIdx.x == 0) printf("time2: %llu\n", start); + // __syncthreads(); start = clock64(); + + // __syncthreads(); long long start = clock64(); + + // SIZE i = threadIdx.x; + // SIZE B_idx = threadIdx.y; + // SIZE b_idx = threadIdx.x; + + // __syncthreads(); start = clock64() - start; + // if (threadIdx.y == 0 && threadIdx.x == 0) printf("time0: %llu\n", start); + // __syncthreads(); start = clock64(); + + // int bit = (v[b-1-b_idx] >> (sizeof(T_org)*8 - 1 - B_idx)) & 1u; + + // __syncthreads(); + // start = clock64() - start; + // if (threadIdx.y == 0 && threadIdx.x == 0) printf("time1: %llu\n", start); + // __syncthreads(); start = clock64(); + + // printf("b_idx[%u]: bit %d\n", b_idx, bit); + // unsigned int sum = __ballot_sync (0xffffffff, bit); + // printf("b_idx[%u]: sum %u\n", b_idx, sum); + // if (b_idx) tv[B_idx] = sum; + + // __syncthreads(); start = clock64() - start; + // if (threadIdx.y == 0 && threadIdx.x == 0) printf("time2: %llu\n", start); + // __syncthreads(); start = clock64(); + } + + // MGARDX_EXEC + // static void TCU(T_org *v, T_trans *tv, SIZE b, SIZE B, SIZE IdX, SIZE IdY) + // { + // __syncthreads(); + // long long start = clock64(); + + // __shared__ half tile_a[16 * 16]; + // __shared__ half tile_b[32 * 32]; + // __shared__ float output[32 * 32]; + // uint8_t bit; + // half shifted_bit; + // SIZE i = threadIdx.x; + // SIZE B_idx = threadIdx.y; + // SIZE b_idx = threadIdx.x; + // SIZE warp_idx = threadIdx.y; + // SIZE lane_idx = threadIdx.x; + + // __syncthreads(); + // start = clock64() - start; + // if (IdY == 0 && IdX == 0) + // printf("time0: %llu\n", start); + + // __syncthreads(); + // start = clock64(); + // __syncthreads(); + + // if (IdX < B * b) { + // uint8_t bit = (v[sizeof(T_trans) * 8 - 1 - b_idx] >> + // (sizeof(T_org) * 8 - 1 - B_idx)) & + // 1u; + // shifted_bit = bit << (sizeof(T_trans) * 8 - 1 - b_idx) % 8; + // tile_b[b_idx * 32 + B_idx] = shifted_bit; + // if (i < 8) { + // tile_a[i] = 1u; + // tile_a[i + 8] = 1u << 8; + // } + // } + // __syncthreads(); + // start = clock64() - start; + // if (IdY == 0 && IdX == 0) + // printf("time1: %llu\n", start); + + // __syncthreads(); + // start = clock64(); + // __syncthreads(); + + // if (warp_idx < 4) { + // wmma::fragment + // a_frag; wmma::fragment b_frag; wmma::fragment c_frag; wmma::load_matrix_sync(a_frag, tile_a, 16); + // wmma::load_matrix_sync( + // b_frag, tile_b + (warp_idx / 2) * 16 + (warp_idx % 2) * 16, 32); + // wmma::fill_fragment(c_frag, 0.0f); + // wmma::mma_sync(c_frag, a_frag, b_frag, c_frag); + // wmma::store_matrix_sync(output + (warp_idx / 2) * 16 + + // (warp_idx % 2) * 16, + // c_frag, 32, wmma::mem_row_major); + // } + + // __syncthreads(); + // start = clock64() - start; + // if (IdY == 0 && IdX == 0) + // printf("time2: %llu\n", start); + // } + + MGARDX_EXEC + static void Transpose(T_org *v, T_trans *tv, SIZE b, SIZE B, SIZE IdX, + SIZE IdY) { + if (METHOD == Bit_Transpose_Serial_All) + Serial_All(v, tv, b, B, IdX, IdY); + else if (METHOD == Bit_Transpose_Parallel_B_Serial_b) + Parallel_B_Serial_b(v, tv, b, B, IdX, IdY); + else if (METHOD == Bit_Transpose_Parallel_B_Atomic_b) + Parallel_B_Atomic_b(v, tv, b, B, IdX, IdY); + else if (METHOD == Bit_Transpose_Parallel_B_Reduce_b) + Parallel_B_Reduction_b(v, tv, b, B, IdX, IdY); + else if (METHOD == Bit_Transpose_Parallel_B_Ballot_b) + Parallel_B_Ballot_b(v, tv, b, B, IdX, IdY); + // else { printf("Bit Transpose Wrong Algorithm Type!\n"); } + // else if (METHOD == 5) TCU(v, tv, b, B, IdX, IdY); + } +}; + +template +struct WarpBitTranspose { + + typedef cub::WarpReduce WarpReduceType; + using WarpReduceStorageType = typename WarpReduceType::TempStorage; + + MGARDX_EXEC static void Serial_All(T_org *v, SIZE inc_v, T_trans *tv, + SIZE inc_tv, SIZE LaneId) { + // long long start; + // if (threadIdx.x == 0 && threadIdx.y == 0) { start = clock64(); } + if (LaneId == 0) { + for (SIZE B_idx = 0; B_idx < B; B_idx++) { + T_trans buffer = 0; + for (SIZE b_idx = 0; b_idx < b; b_idx++) { + T_trans bit = + (v[b_idx * inc_v] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + // if (blockIdx.x == 0 )printf("bit: %u\n", bit); + if (ALIGN == ALIGN_LEFT) { + buffer += bit << (sizeof(T_trans) * 8 - 1 - b_idx); + } else if (ALIGN == ALIGN_RIGHT) { + buffer += bit << (b - 1 - b_idx); + } else { + } + } + tv[B_idx * inc_tv] = buffer; + } + } + // if (threadIdx.x == 0 && threadIdx.y == 0) { start = clock64() - start; + // printf("Serial_All time : %llu\n", start); } + } + + MGARDX_EXEC static void Parallel_B_Serial_b(T_org *v, SIZE inc_v, T_trans *tv, + SIZE inc_tv, SIZE LaneId) { + // long long start; + // if (threadIdx.x == 0 && threadIdx.y == 0) { start = clock64(); } + for (SIZE B_idx = LaneId; B_idx < B; B_idx += MGARDX_WARP_SIZE) { + T_trans buffer = 0; + for (SIZE b_idx = 0; b_idx < b; b_idx++) { + T_trans bit = + (v[b_idx * inc_v] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + // if (blockIdx.x == 0 )printf("bit: %u\n", bit); + if (ALIGN == ALIGN_LEFT) { + buffer += bit << (sizeof(T_trans) * 8 - 1 - b_idx); + } else if (ALIGN == ALIGN_RIGHT) { + buffer += bit << (b - 1 - b_idx); + } else { + } + } + tv[B_idx * inc_tv] = buffer; + } + // if (threadIdx.x == 0 && threadIdx.y == 0) { start = clock64() - start; + // printf("Parallel_B_Serial_b time : %llu\n", start); } + } + + MGARDX_EXEC static void Serial_B_Atomic_b(T_org *v, SIZE inc_v, T_trans *tv, + SIZE inc_tv, SIZE LaneId) { + // long long start; + // if (threadIdx.x == 0 && threadIdx.y == 0) { start = clock64(); } + for (SIZE B_idx = LaneId; B_idx < B; B_idx += MGARDX_WARP_SIZE) { + tv[B_idx * inc_tv] = 0; + } + for (SIZE B_idx = 0; B_idx < B; B_idx++) { + for (SIZE b_idx = LaneId; b_idx < b; b_idx += MGARDX_WARP_SIZE) { + T_trans bit = + (v[b_idx * inc_v] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + T_trans shifted_bit = 0; + if (ALIGN == ALIGN_LEFT) { + shifted_bit = bit << (sizeof(T_trans) * 8 - 1 - b_idx); + } else if (ALIGN == ALIGN_RIGHT) { + shifted_bit = bit << (b - 1 - b_idx); + } else { + } + T_trans *sum = &(tv[B_idx * inc_tv]); + Atomic::Add( + sum, shifted_bit); + } + } + // if (threadIdx.x == 0 && threadIdx.y == 0) { start = clock64() - start; + // printf("Serial_B_Atomic_b time : %llu\n", start); } + } + + MGARDX_EXEC static void Serial_B_Reduce_b(T_org *v, SIZE inc_v, T_trans *tv, + SIZE inc_tv, SIZE LaneId) { + // long long start; + // if (threadIdx.x == 0 && threadIdx.y == 0) { start = clock64(); } + __shared__ WarpReduceStorageType warp_storage; + + T_trans bit = 0; + T_trans shifted_bit = 0; + T_trans sum = 0; + for (SIZE B_idx = 0; B_idx < B; B_idx++) { + sum = 0; + for (SIZE b_idx = LaneId; + b_idx < ((b - 1) / MGARDX_WARP_SIZE + 1) * MGARDX_WARP_SIZE; + b_idx += MGARDX_WARP_SIZE) { + if (b_idx < b) { + bit = (v[b_idx * inc_v] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + } + shifted_bit = 0; + if (ALIGN == ALIGN_LEFT) { + shifted_bit = bit << (sizeof(T_trans) * 8 - 1 - b_idx); + } else if (ALIGN == ALIGN_RIGHT) { + shifted_bit = bit << (b - 1 - b_idx); + } else { + } + sum += WarpReduceType(warp_storage).Sum(shifted_bit); + } + if (LaneId == 0) { + tv[B_idx * inc_tv] = sum; + } + } + // if (threadIdx.x == 0 && threadIdx.y == 0) { start = clock64() - start; + // printf("Serial_B_Reduce_b time : %llu\n", start); } + } + + MGARDX_EXEC static void Serial_B_Ballot_b(T_org *v, SIZE inc_v, T_trans *tv, + SIZE inc_tv, SIZE LaneId) { + // long long start; + // if (threadIdx.x == 0 && threadIdx.y == 0) { start = clock64(); } + T_trans bit = 0; + T_trans sum = 0; + for (SIZE B_idx = 0; B_idx < B; B_idx++) { + sum = 0; + SIZE shift = 0; + for (SIZE b_idx = LaneId; + b_idx < ((b - 1) / MGARDX_WARP_SIZE + 1) * MGARDX_WARP_SIZE; + b_idx += MGARDX_WARP_SIZE) { + bit = 0; + if (b_idx < b) { + if (ALIGN == ALIGN_LEFT) { + bit = (v[(sizeof(T_trans) * 8 - 1 - b_idx) * inc_v] >> + (sizeof(T_org) * 8 - 1 - B_idx)) & + 1u; + } else if (ALIGN == ALIGN_RIGHT) { + bit = (v[(b - 1 - b_idx) * inc_v] >> + (sizeof(T_org) * 8 - 1 - B_idx)) & + 1u; + } else { + } + } + sum += ((T_trans)__ballot_sync(0xffffffff, bit)) << shift; + shift += MGARDX_WARP_SIZE; + } + if (LaneId == 0) { + tv[B_idx * inc_tv] = sum; + } + } + // if (threadIdx.x == 0 && threadIdx.y == 0) { start = clock64() - start; + // printf("Serial_B_Ballot_b time : %llu\n", start); } + } + + MGARDX_EXEC static void Transpose(T_org *v, SIZE inc_v, T_trans *tv, + SIZE inc_tv, SIZE LaneId) { + if constexpr (METHOD == Warp_Bit_Transpose_Serial_All) { + Serial_All(v, inc_v, tv, inc_tv, LaneId); + } else if constexpr (METHOD == Warp_Bit_Transpose_Parallel_B_Serial_b) { + Parallel_B_Serial_b(v, inc_v, tv, inc_tv, LaneId); + } else if constexpr (METHOD == Warp_Bit_Transpose_Serial_B_Atomic_b) { + Serial_B_Atomic_b(v, inc_v, tv, inc_tv, LaneId); + } else if constexpr (METHOD == Warp_Bit_Transpose_Serial_B_Reduce_b) { + Serial_B_Reduce_b(v, inc_v, tv, inc_tv, LaneId); + } else if constexpr (METHOD == Warp_Bit_Transpose_Serial_B_Ballot_b) { + Serial_B_Ballot_b(v, inc_v, tv, inc_tv, LaneId); + } + } +}; + +template +struct BlockErrorCollect { + + typedef cub::WarpReduce WarpReduceType; + using WarpReduceStorageType = typename WarpReduceType::TempStorage; + + typedef cub::BlockReduce + BlockReduceType; + using BlockReduceStorageType = typename BlockReduceType::TempStorage; + + MGARDX_EXEC + static void Serial_All(T *v, T_error *temp, T_error *errors, SIZE num_elems, + SIZE num_bitplanes, SIZE IdX, SIZE IdY) { + if (IdX == 0 && IdY == 0) { + for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx++) { + T data = v[elem_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + for (SIZE bitplane_idx = 0; bitplane_idx < num_bitplanes; + bitplane_idx++) { + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + } + errors[num_bitplanes - bitplane_idx] += diff * diff; + // if (blockIdx.x == 0 && num_bitplanes-bitplane_idx == 2) { + // printf("elem error[%u]: %f\n", elem_idx, diff * diff); + // } + } + errors[0] += data * data; + } + } + } + + MGARDX_EXEC + static void Parallel_Bitplanes_Serial_Error(T *v, T_error *temp, + T_error *errors, SIZE num_elems, + SIZE num_bitplanes, SIZE IdX, + SIZE IdY) { + SIZE bitplane_idx = IdY * nblockx + IdX; + if (bitplane_idx < num_bitplanes) { + for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx++) { + T data = v[elem_idx]; + T_fp fp_data = (T_fp)fabs(v[elem_idx]); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + } + errors[num_bitplanes - bitplane_idx] += diff * diff; + } + } + if (bitplane_idx == 0) { + for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx++) { + T data = v[elem_idx]; + errors[0] += data * data; + } + } + } + + MGARDX_EXEC + static void Parallel_Bitplanes_Atomic_Error(T *v, T_error *temp, + T_error *errors, SIZE num_elems, + SIZE num_bitplanes, SIZE IdX, + SIZE IdY) { + for (SIZE elem_idx = IdX; elem_idx < num_elems; elem_idx += nblockx) { + for (SIZE bitplane_idx = IdY; bitplane_idx < num_bitplanes; + bitplane_idx += nblocky) { + T data = v[elem_idx]; + T_fp fp_data = (T_fp)fabs(v[elem_idx]); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + } + temp[(num_bitplanes - bitplane_idx) * num_elems + elem_idx] = + diff * diff; + if (bitplane_idx == 0) { + temp[elem_idx] = data * data; + } + } + } + __syncthreads(); + // if (blockIdx.x == 0 && IdX == 0 && IdY == 0) { + // for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx += 1) { + // printf("elem error[%u]: %f\n", elem_idx, temp[(2) * num_elems + + // elem_idx]); + // } + // } + for (SIZE bitplane_idx = IdY; bitplane_idx < num_bitplanes + 1; + bitplane_idx += nblocky) { + for (SIZE elem_idx = IdX; elem_idx < ((num_elems - 1) / 32 + 1) * 32; + elem_idx += 32) { + T_error error = 0; + if (elem_idx < num_elems) { + error = temp[(num_bitplanes - bitplane_idx) * num_elems + elem_idx]; + } + T_error *sum = &(errors[num_bitplanes - bitplane_idx]); + Atomic::Add(sum, + error); + } + } + } + + MGARDX_EXEC + static void Parallel_Bitplanes_Reduce_Error(T *v, T_error *temp, + T_error *errors, SIZE num_elems, + SIZE num_bitplanes, SIZE IdX, + SIZE IdY) { + for (SIZE elem_idx = IdX; elem_idx < num_elems; elem_idx += nblockx) { + for (SIZE bitplane_idx = IdY; bitplane_idx < num_bitplanes; + bitplane_idx += nblocky) { + T data = v[elem_idx]; + T_fp fp_data = (T_fp)fabs(v[elem_idx]); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + } + temp[(num_bitplanes - bitplane_idx) * num_elems + elem_idx] = + diff * diff; + // if (blockIdx.x == 0 && num_bitplanes - bitplane_idx == 31) { + // printf("elem_idx: %u, data: %f, fp_data: %u, mask: %u, mantissa: + // %f, diff: %f\n", + // elem_idx, data, fp_data, mask, mantissa, diff); + // } + if (bitplane_idx == 0) { + temp[elem_idx] = data * data; + } + } + } + __syncthreads(); + + // if (blockIdx.x == 0 && IdX == 0 && IdY == 0) { + // for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx += 1) { + // printf("elem error[%u]: %f\n", elem_idx, temp[(31) * num_elems + + // elem_idx]); + // } + // } + + __shared__ WarpReduceStorageType warp_storage[nblocky]; + + for (SIZE bitplane_idx = IdY; bitplane_idx < num_bitplanes + 1; + bitplane_idx += nblocky) { + T error_sum = 0; + for (SIZE elem_idx = IdX; elem_idx < ((num_elems - 1) / 32 + 1) * 32; + elem_idx += 32) { + T_error error = 0; + if (elem_idx < num_elems) { + error = temp[(num_bitplanes - bitplane_idx) * num_elems + elem_idx]; + } + error_sum += WarpReduceType(warp_storage[IdY]).Sum(error); + // errors[num_bitplanes - bitplane_idx] += + // WarpReduceType(warp_storage[IdY]).Sum(error); + } + if (IdX == 0) { + errors[num_bitplanes - bitplane_idx] = error_sum; + } + } + // __syncthreads(); + // if (blockIdx.x == 0 && IdX == 0 && IdY == 0) { + // for (int i = 0; i < num_bitplanes + 1; i++) { + // printf("error[%d]: %f\n", i, errors[i]); + // } + // } + } + + MGARDX_EXEC + static void Collect(T *v, T_error *temp, T_error *errors, SIZE num_elems, + SIZE num_bitplanes, SIZE IdX, SIZE IdY) { + if constexpr (METHOD == Error_Collecting_Serial_All) + Serial_All(v, temp, errors, num_elems, num_bitplanes, IdX, IdY); + else if constexpr (METHOD == + Error_Collecting_Parallel_Bitplanes_Serial_Error) + Parallel_Bitplanes_Serial_Error(v, temp, errors, num_elems, num_bitplanes, + IdX, IdY); + else if constexpr (METHOD == + Error_Collecting_Parallel_Bitplanes_Atomic_Error) + Parallel_Bitplanes_Atomic_Error(v, temp, errors, num_elems, num_bitplanes, + IdX, IdY); + else if constexpr (METHOD == + Error_Collecting_Parallel_Bitplanes_Reduce_Error) + Parallel_Bitplanes_Reduce_Error(v, temp, errors, num_elems, num_bitplanes, + IdX, IdY); + // else if (METHOD == Error_Collecting_Disable) {} + // else { printf("Error Collecting Wrong Algorithm Type!\n"); } + } +}; + +template +struct WarpErrorCollect { + + typedef cub::WarpReduce WarpReduceType; + using WarpReduceStorageType = typename WarpReduceType::TempStorage; + + MGARDX_EXEC static void Serial_All(T *v, T_error *errors, SIZE LaneId) { + if (LaneId == 0) { + for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx++) { + T data = v[elem_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + + // printf("fp: %u error: \n", fp_data); + for (SIZE bitplane_idx = 0; bitplane_idx < num_bitplanes; + bitplane_idx++) { + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + } + errors[num_bitplanes - bitplane_idx] += diff * diff; + // printf("%f ", diff * diff); + } + errors[0] += data * data; + // printf("%f \n", data * data); + } + } + } + + MGARDX_EXEC static void Parallel_Bitplanes_Serial_Error(T *v, T_error *errors, + SIZE LaneId) { + + __shared__ WarpReduceStorageType warp_storage; + + for (SIZE bitplane_idx = LaneId; bitplane_idx < num_bitplanes; + bitplane_idx += MGARDX_WARP_SIZE) { + for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx++) { + T data = v[elem_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + } + errors[num_bitplanes - bitplane_idx] += diff * diff; + } + } + + T data = 0; + for (SIZE elem_idx = LaneId; + elem_idx < ((num_elems - 1) / MGARDX_WARP_SIZE + 1) * MGARDX_WARP_SIZE; + elem_idx += MGARDX_WARP_SIZE) { + if (elem_idx < num_elems) { + data = v[elem_idx]; + } + T_error error_sum = WarpReduceType(warp_storage).Sum(data * data); + if (LaneId == 0) + errors[0] += error_sum; + } + } + + MGARDX_EXEC static void Serial_Bitplanes_Atomic_Error(T *v, T_error *errors, + SIZE LaneId) { + T data = 0; + for (SIZE elem_idx = LaneId; + elem_idx < ((num_elems - 1) / MGARDX_WARP_SIZE + 1) * MGARDX_WARP_SIZE; + elem_idx += MGARDX_WARP_SIZE) { + if (elem_idx < num_elems) { + data = v[elem_idx]; + } else { + data = 0; + } + + T_fp fp_data = (T_fp)fabs(data); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + + // printf("fp: %u error: \n", fp_data); + for (SIZE bitplane_idx = 0; bitplane_idx < num_bitplanes; + bitplane_idx++) { + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + } + T_error *sum = &(errors[num_bitplanes - bitplane_idx]); + Atomic::Add( + sum, diff * diff); + } + T_error *sum = &(errors[0]); + Atomic::Add( + sum, data * data); + } + } + + MGARDX_EXEC static void Serial_Bitplanes_Reduce_Error(T *v, T_error *errors, + SIZE LaneId) { + + __shared__ WarpReduceStorageType warp_storage; + + T data = 0; + for (SIZE elem_idx = LaneId; + elem_idx < ((num_elems - 1) / MGARDX_WARP_SIZE + 1) * MGARDX_WARP_SIZE; + elem_idx += MGARDX_WARP_SIZE) { + if (elem_idx < num_elems) { + data = v[elem_idx]; + } else { + data = 0; + } + + T_fp fp_data = (T_fp)fabs(data); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + + // printf("fp: %u error: \n", fp_data); + for (SIZE bitplane_idx = 0; bitplane_idx < num_bitplanes; + bitplane_idx++) { + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + } + T_error error_sum = WarpReduceType(warp_storage).Sum(diff * diff); + if (LaneId == 0) + errors[num_bitplanes - bitplane_idx] += error_sum; + } + T_error error_sum = WarpReduceType(warp_storage).Sum(data * data); + if (LaneId == 0) + errors[0] += error_sum; + } + } + + MGARDX_EXEC static void Collect(T *v, T_error *errors, SIZE LaneId) { + if (METHOD == Warp_Error_Collecting_Serial_All) { + Serial_All(v, errors, LaneId); + } + if (METHOD == Warp_Error_Collecting_Parallel_Bitplanes_Serial_Error) { + Parallel_Bitplanes_Serial_Error(v, errors, LaneId); + } + if (METHOD == Warp_Error_Collecting_Serial_Bitplanes_Atomic_Error) { + Serial_Bitplanes_Atomic_Error(v, errors, LaneId); + } + if (METHOD == Warp_Error_Collecting_Serial_Bitplanes_Reduce_Error) { + Serial_Bitplanes_Reduce_Error(v, errors, LaneId); + } + } +}; + +template void CudaHuffmanCLCustomizedNoCGKernel(Task task) { + // std::cout << "calling HuffmanCLCustomizedNoCGKernel\n"; + dim3 threadsPerBlock(task.GetBlockDimX(), task.GetBlockDimY(), + task.GetBlockDimZ()); + dim3 blockPerGrid(task.GetGridDimX(), task.GetGridDimY(), task.GetGridDimZ()); + size_t sm_size = task.GetSharedMemorySize(); + cudaStream_t stream = DeviceRuntime::GetQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation1_Kernel\n"; + Single_Operation1_Kernel<<>>( + task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling LoopCondition1\n"; + while (task.GetFunctor().LoopCondition1()) { + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling Single_Operation2_Kernel\n"; + Single_Operation2_Kernel<<>>(task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling Single_Operation3_Kernel\n"; + Single_Operation3_Kernel<<>>(task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + Single_Operation4_Kernel<<>>(task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling Single_Operation4_Kernel\n"; + Single_Operation5_Kernel<<>>(task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling BranchCondition1\n"; + if (task.GetFunctor().BranchCondition1()) { + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling CudaParallelMergeKernel\n"; + CudaParallelMergeKernel<<>>(task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling Single_Operation10_Kernel\n"; + Single_Operation11_Kernel<<>>(task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + } + + // std::cout << "calling Single_Operation11_Kernel\n"; + Single_Operation12_Kernel<<>>(task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling Single_Operation12_Kernel\n"; + Single_Operation13_Kernel<<>>(task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling Single_Operation13_Kernel\n"; + Single_Operation14_Kernel<<>>(task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling Single_Operation14_Kernel\n"; + Single_Operation15_Kernel<<>>(task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + } +} + +template void CudaHuffmanCWCustomizedNoCGKernel(Task task) { + // std::cout << "calling HuffmanCWCustomizedNoCGKernel\n"; + dim3 threadsPerBlock(task.GetBlockDimX(), task.GetBlockDimY(), + task.GetBlockDimZ()); + dim3 blockPerGrid(task.GetGridDimX(), task.GetGridDimY(), task.GetGridDimZ()); + size_t sm_size = task.GetSharedMemorySize(); + cudaStream_t stream = DeviceRuntime::GetQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation1_Kernel\n"; + Single_Operation1_Kernel<<>>( + task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + // std::cout << "calling Single_Operation2_Kernel\n"; + Single_Operation2_Kernel<<>>( + task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + // std::cout << "calling Single_Operation3_Kernel\n"; + Single_Operation3_Kernel<<>>( + task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling LoopCondition1\n"; + while (task.GetFunctor().LoopCondition1()) { + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling Single_Operation4_Kernel\n"; + Single_Operation4_Kernel<<>>(task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling Single_Operation5_Kernel\n"; + Single_Operation5_Kernel<<>>(task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling Single_Operation6_Kernel\n"; + Single_Operation6_Kernel<<>>(task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling Single_Operation7_Kernel\n"; + Single_Operation7_Kernel<<>>(task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling Single_Operation8_Kernel\n"; + Single_Operation8_Kernel<<>>(task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + } + + // std::cout << "calling Single_Operation9_Kernel\n"; + Single_Operation9_Kernel<<>>( + task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + + // std::cout << "calling Single_Operation10_Kernel\n"; + Single_Operation10_Kernel<<>>( + task); + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); +} + +#define MGARD_CUDA_MAX_GRID_X 2147483647 +#define MGARD_CUDA_MAX_GRID_Y 65535 +#define MGARD_CUDA_MAX_GRID_Z 65535 + +// template class DeviceAdapter { +// public: +// MGARDX_CONT +// DeviceAdapter(){}; + +// MGARDX_CONT +// int IsResourceEnough(TaskType &task) { +// if (task.GetBlockDimX() * task.GetBlockDimY() * task.GetBlockDimZ() > +// DeviceRuntime::GetMaxNumThreadsPerTB()) { +// return THREADBLOCK_TOO_LARGE; +// } +// if (task.GetSharedMemorySize() > +// DeviceRuntime::GetMaxSharedMemorySize()) { +// return SHARED_MEMORY_TOO_LARGE; +// } +// return RESOURCE_ENOUGH; +// } + +// MGARDX_CONT +// ExecutionReturn Execute(TaskType &task) { +// dim3 threadsPerBlock(task.GetBlockDimX(), task.GetBlockDimY(), +// task.GetBlockDimZ()); +// dim3 blockPerGrid(std::min(task.GetGridDimX(), +// (IDX)MGARD_CUDA_MAX_GRID_X), +// std::min(task.GetGridDimY(), +// (IDX)MGARD_CUDA_MAX_GRID_Y), +// std::min(task.GetGridDimZ(), +// (IDX)MGARD_CUDA_MAX_GRID_Z)); +// size_t sm_size = task.GetSharedMemorySize(); + +// cudaStream_t stream = DeviceRuntime::GetQueue(task.GetQueueIdx()); + +// if (DeviceRuntime::PrintKernelConfig) { +// std::cout << log::log_info << task.GetFunctorName() << ": <" +// << threadsPerBlock.x << ", " << threadsPerBlock.y << ", " +// << threadsPerBlock.z << "> <" << blockPerGrid.x << ", " +// << blockPerGrid.y << ", " << blockPerGrid.z << ">\n"; +// } + +// ExecutionReturn ret; +// if (IsResourceEnough(task) != RESOURCE_ENOUGH) { +// if (DeviceRuntime::PrintKernelConfig) { +// if (IsResourceEnough(task) == THREADBLOCK_TOO_LARGE) { +// log::info("threadblock too large."); +// } +// if (IsResourceEnough(task) == SHARED_MEMORY_TOO_LARGE) { +// log::info("shared memory too large."); +// } +// } +// ret.success = false; +// ret.execution_time = std::numeric_limits::max(); +// return ret; +// } + +// Timer timer; +// if (task.GetQueueIdx() == MGARDX_SYNCHRONIZED_QUEUE || +// DeviceRuntime::TimingAllKernels || +// AutoTuner::ProfileKernels) { +// DeviceRuntime::SyncDevice(); +// timer.start(); +// } + +// // if constexpr evaluate at compile time otherwise this does not compile +// if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) { +// for (THREAD_IDX blockz_offset = 0; blockz_offset < task.GetGridDimZ(); +// blockz_offset += MGARD_CUDA_MAX_GRID_Z) { +// for (THREAD_IDX blocky_offset = 0; blocky_offset < +// task.GetGridDimY(); +// blocky_offset += MGARD_CUDA_MAX_GRID_Y) { +// for (THREAD_IDX blockx_offset = 0; blockx_offset < +// task.GetGridDimX(); +// blockx_offset += MGARD_CUDA_MAX_GRID_X) { +// CudaKernel<<>>( +// task, blockz_offset, blocky_offset, blockx_offset); +// } +// } +// } +// } else if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) +// { +// for (THREAD_IDX blockz_offset = 0; blockz_offset < task.GetGridDimZ(); +// blockz_offset += MGARD_CUDA_MAX_GRID_Z) { +// for (THREAD_IDX blocky_offset = 0; blocky_offset < +// task.GetGridDimY(); +// blocky_offset += MGARD_CUDA_MAX_GRID_Y) { +// for (THREAD_IDX blockx_offset = 0; blockx_offset < +// task.GetGridDimX(); +// blockx_offset += MGARD_CUDA_MAX_GRID_X) { +// CudaIterKernel<<>>( +// task, blockz_offset, blocky_offset, blockx_offset); +// } +// } +// } +// } else if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) +// { +// if (task.GetFunctor().use_CG && DeviceRuntime::SupportCG()) { +// void *Args[] = {(void *)&task}; +// cudaLaunchCooperativeKernel( +// (void *)CudaHuffmanCLCustomizedKernel, blockPerGrid, +// threadsPerBlock, Args, sm_size, stream); +// } else { +// CudaHuffmanCLCustomizedNoCGKernel(task); +// } +// } else if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) +// { +// if (task.GetFunctor().use_CG && DeviceRuntime::SupportCG()) { +// void *Args[] = {(void *)&task}; +// cudaLaunchCooperativeKernel( +// (void *)CudaHuffmanCWCustomizedKernel, blockPerGrid, +// threadsPerBlock, Args, sm_size, stream); +// } else { +// CudaHuffmanCWCustomizedNoCGKernel(task); +// } +// } +// ErrorAsyncCheckTask(cudaGetLastError(), task); +// gpuErrchk(cudaGetLastError()); +// if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { +// ErrorSyncCheckTask(cudaDeviceSynchronize(), task); +// } + +// if (task.GetQueueIdx() == MGARDX_SYNCHRONIZED_QUEUE || +// DeviceRuntime::TimingAllKernels || +// AutoTuner::ProfileKernels) { +// DeviceRuntime::SyncDevice(); +// timer.end(); +// if (DeviceRuntime::TimingAllKernels) { +// timer.print(task.GetFunctorName()); +// } +// if (AutoTuner::ProfileKernels) { +// ret.success = true; +// ret.execution_time = timer.get(); +// } +// } +// return ret; +// } +// }; + +template <> class DeviceLauncher { +public: + template + MGARDX_CONT int static IsResourceEnough(TaskType &task) { + if (task.GetBlockDimX() * task.GetBlockDimY() * task.GetBlockDimZ() > + DeviceRuntime::GetMaxNumThreadsPerTB()) { + return THREADBLOCK_TOO_LARGE; + } + if (task.GetSharedMemorySize() > + DeviceRuntime::GetMaxSharedMemorySize()) { + return SHARED_MEMORY_TOO_LARGE; + } + return RESOURCE_ENOUGH; + } + + template + MGARDX_CONT ExecutionReturn static Execute(TaskType &task) { + + dim3 threadsPerBlock(task.GetBlockDimX(), task.GetBlockDimY(), + task.GetBlockDimZ()); + dim3 blockPerGrid(std::min(task.GetGridDimX(), (IDX)MGARD_CUDA_MAX_GRID_X), + std::min(task.GetGridDimY(), (IDX)MGARD_CUDA_MAX_GRID_Y), + std::min(task.GetGridDimZ(), (IDX)MGARD_CUDA_MAX_GRID_Z)); + size_t sm_size = task.GetSharedMemorySize(); + + cudaStream_t stream = DeviceRuntime::GetQueue(task.GetQueueIdx()); + + if (DeviceRuntime::PrintKernelConfig) { + std::cout << log::log_info << task.GetFunctorName() << ": <" + << threadsPerBlock.x << ", " << threadsPerBlock.y << ", " + << threadsPerBlock.z << "> <" << blockPerGrid.x << ", " + << blockPerGrid.y << ", " << blockPerGrid.z + << "> sm: " << sm_size << "\n"; + } + + ExecutionReturn ret; + if (IsResourceEnough(task) != RESOURCE_ENOUGH) { + if (DeviceRuntime::PrintKernelConfig) { + if (IsResourceEnough(task) == THREADBLOCK_TOO_LARGE) { + log::info("threadblock too large."); + } + if (IsResourceEnough(task) == SHARED_MEMORY_TOO_LARGE) { + log::info("shared memory too large."); + } + } + ret.success = false; + ret.execution_time = std::numeric_limits::max(); + return ret; + } + + Timer timer; + if (task.GetQueueIdx() == MGARDX_SYNCHRONIZED_QUEUE || + DeviceRuntime::TimingAllKernels || + AutoTuner::ProfileKernels) { + DeviceRuntime::SyncDevice(); + timer.start(); + } + + // if constexpr evaluate at compile time otherwise this does not compile + if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + for (THREAD_IDX blockz_offset = 0; blockz_offset < task.GetGridDimZ(); + blockz_offset += MGARD_CUDA_MAX_GRID_Z) { + for (THREAD_IDX blocky_offset = 0; blocky_offset < task.GetGridDimY(); + blocky_offset += MGARD_CUDA_MAX_GRID_Y) { + for (THREAD_IDX blockx_offset = 0; blockx_offset < task.GetGridDimX(); + blockx_offset += MGARD_CUDA_MAX_GRID_X) { + CudaKernel<<>>( + task, blockz_offset, blocky_offset, blockx_offset); + } + } + } + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + for (THREAD_IDX blockz_offset = 0; blockz_offset < task.GetGridDimZ(); + blockz_offset += MGARD_CUDA_MAX_GRID_Z) { + for (THREAD_IDX blocky_offset = 0; blocky_offset < task.GetGridDimY(); + blocky_offset += MGARD_CUDA_MAX_GRID_Y) { + for (THREAD_IDX blockx_offset = 0; blockx_offset < task.GetGridDimX(); + blockx_offset += MGARD_CUDA_MAX_GRID_X) { + CudaIterKernel<<>>( + task, blockz_offset, blocky_offset, blockx_offset); + } + } + } + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + if (task.GetFunctor().use_CG && DeviceRuntime::SupportCG()) { + void *Args[] = {(void *)&task}; + cudaLaunchCooperativeKernel( + (void *)CudaHuffmanCLCustomizedKernel, blockPerGrid, + threadsPerBlock, Args, sm_size, stream); + } else { + CudaHuffmanCLCustomizedNoCGKernel(task); + } + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + if (task.GetFunctor().use_CG && DeviceRuntime::SupportCG()) { + void *Args[] = {(void *)&task}; + cudaLaunchCooperativeKernel( + (void *)CudaHuffmanCWCustomizedKernel, blockPerGrid, + threadsPerBlock, Args, sm_size, stream); + } else { + CudaHuffmanCWCustomizedNoCGKernel(task); + } + } + ErrorAsyncCheckTask(cudaGetLastError(), task); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheckTask(cudaDeviceSynchronize(), task); + } + + if (task.GetQueueIdx() == MGARDX_SYNCHRONIZED_QUEUE || + DeviceRuntime::TimingAllKernels || + AutoTuner::ProfileKernels) { + DeviceRuntime::SyncDevice(); + timer.end(); + if (DeviceRuntime::TimingAllKernels) { + timer.print(task.GetFunctorName()); + } + if (AutoTuner::ProfileKernels) { + ret.success = true; + ret.execution_time = timer.get(); + } + } + return ret; + } + + template + MGARDX_CONT static void ConfigTask(TaskType task) { + typename TaskType::Functor functor; + int maxbytes = DeviceRuntime::GetMaxSharedMemorySize(); + DeviceRuntime::SetMaxDynamicSharedMemorySize(functor, maxbytes); + } + + template + MGARDX_CONT static void AutoTune(KernelType kernel, int queue_idx) { +#if MGARD_ENABLE_AUTO_TUNING + double min_time = std::numeric_limits::max(); + int min_config = 0; + ExecutionReturn ret; +#define RUN_CONFIG(CONFIG_IDX) \ + { \ + constexpr ExecutionConfig config = \ + GetExecutionConfig(CONFIG_IDX); \ + auto task = \ + kernel.template GenTask(queue_idx); \ + if constexpr (KernelType::EnableConfig()) { \ + ConfigTask(task); \ + } \ + ret = Execute(task); \ + if (ret.success && min_time > ret.execution_time) { \ + min_time = ret.execution_time; \ + min_config = CONFIG_IDX; \ + } \ + } + RUN_CONFIG(0) + RUN_CONFIG(1) + RUN_CONFIG(2) + RUN_CONFIG(3) + RUN_CONFIG(4) + RUN_CONFIG(5) + RUN_CONFIG(6) +#undef RUN_CONFIG + if (AutoTuner::WriteToTable) { + FillAutoTunerTable(std::string(KernelType::Name), min_config); + } +#else + throw std::runtime_error("MGARD is not built with auto tuning enabled."); +#endif + } + + template + MGARDX_CONT static void Execute(KernelType kernel, int queue_idx) { + if constexpr (KernelType::EnableAutoTuning()) { + constexpr ExecutionConfig config = + GetExecutionConfig(KernelType::Name); + auto task = + kernel.template GenTask(queue_idx); + if constexpr (KernelType::EnableConfig()) { + ConfigTask(task); + } + Execute(task); + + if (AutoTuner::ProfileKernels) { + AutoTune(kernel, queue_idx); + } + } else { + auto task = kernel.GenTask(queue_idx); + if constexpr (KernelType::EnableConfig()) { + ConfigTask(task); + } + Execute(task); + } + } +}; + +struct AbsMaxOp { + template + __device__ __forceinline__ T operator()(const T &a, const T &b) const { + return (fabs(b) > fabs(a)) ? fabs(b) : fabs(a); + } +}; + +struct AbsMinOp { + template + __device__ __forceinline__ T operator()(const T &a, const T &b) const { + return (fabs(b) > fabs(a)) ? fabs(a) : fabs(b); + } +}; + +struct SquareOp { + template + __device__ __forceinline__ T operator()(const T &a) const { + return a * a; + } +}; + +template <> class DeviceCollective { +public: + MGARDX_CONT + DeviceCollective(){}; + + template + MGARDX_CONT static void Sum(SIZE n, SubArray<1, T, CUDA> v, + SubArray<1, T, CUDA> result, + Array<1, Byte, CUDA> &workspace, + bool workspace_allocated, int queue_idx) { + + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); + cub::DeviceReduce::Sum(d_temp_storage, temp_storage_bytes, v.data(), + result.data(), n, stream); + ErrorAsyncCheck(cudaGetLastError(), "DeviceCollective::Sum"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(cudaDeviceSynchronize(), "DeviceCollective::Sum"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + + template + MGARDX_CONT static void AbsMax(SIZE n, SubArray<1, T, CUDA> v, + SubArray<1, T, CUDA> result, + Array<1, Byte, CUDA> &workspace, + bool workspace_allocated, int queue_idx) { + + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + AbsMaxOp absMaxOp; + cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); + cub::DeviceReduce::Reduce(d_temp_storage, temp_storage_bytes, v.data(), + result.data(), n, absMaxOp, static_cast(0), + stream); + ErrorAsyncCheck(cudaGetLastError(), "DeviceCollective::AbsMax"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(cudaDeviceSynchronize(), "DeviceCollective::AbsMax"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + + template + MGARDX_CONT static void AbsMin(SIZE n, SubArray<1, T, CUDA> v, + SubArray<1, T, CUDA> result, + Array<1, Byte, CUDA> &workspace, + bool workspace_allocated, int queue_idx) { + + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + AbsMinOp absMinOp; + cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); + cub::DeviceReduce::Reduce(d_temp_storage, temp_storage_bytes, v.data(), + result.data(), n, absMinOp, + std::numeric_limits::max(), stream); + ErrorAsyncCheck(cudaGetLastError(), "DeviceCollective::AbsMin"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(cudaDeviceSynchronize(), "DeviceCollective::AbsMin"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + + template + MGARDX_CONT static void SquareSum(SIZE n, SubArray<1, T, CUDA> v, + SubArray<1, T, CUDA> result, + Array<1, Byte, CUDA> &workspace, + bool workspace_allocated, int queue_idx) { + + SquareOp squareOp; + auto transformed_input_iter = + thrust::make_transform_iterator(v.data(), squareOp); + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); + cub::DeviceReduce::Sum(d_temp_storage, temp_storage_bytes, + transformed_input_iter, result.data(), n, stream); + ErrorAsyncCheck(cudaGetLastError(), "DeviceCollective::SquareSum"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(cudaDeviceSynchronize(), + "DeviceCollective::SquareSum"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + + template + MGARDX_CONT static void + ScanSumInclusive(SIZE n, SubArray<1, T, CUDA> v, SubArray<1, T, CUDA> result, + Array<1, Byte, CUDA> &workspace, bool workspace_allocated, + int queue_idx) { + + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); + cub::DeviceScan::InclusiveSum(d_temp_storage, temp_storage_bytes, v.data(), + result.data(), n, stream); + ErrorAsyncCheck(cudaGetLastError(), + "DeviceCollective::ScanSumInclusive"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(cudaDeviceSynchronize(), + "DeviceCollective::ScanSumInclusive"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + + template + MGARDX_CONT static void + ScanSumExclusive(SIZE n, SubArray<1, T, CUDA> v, SubArray<1, T, CUDA> result, + Array<1, Byte, CUDA> &workspace, bool workspace_allocated, + int queue_idx) { + + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); + cub::DeviceScan::ExclusiveSum(d_temp_storage, temp_storage_bytes, v.data(), + result.data(), n, stream); + ErrorAsyncCheck(cudaGetLastError(), + "DeviceCollective::ScanSumExclusive"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(cudaDeviceSynchronize(), + "DeviceCollective::ScanSumExclusive"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + + template + MGARDX_CONT static void + ScanSumExtended(SIZE n, SubArray<1, T, CUDA> v, SubArray<1, T, CUDA> result, + Array<1, Byte, CUDA> &workspace, bool workspace_allocated, + int queue_idx) { + + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); + cub::DeviceScan::InclusiveSum(d_temp_storage, temp_storage_bytes, v.data(), + result.data() + 1, n, stream); + ErrorAsyncCheck(cudaGetLastError(), + "DeviceCollective::ScanSumExtended"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(cudaDeviceSynchronize(), + "DeviceCollective::ScanSumExtended"); + } + if (workspace_allocated) { + T zero = 0; + MemoryManager().Copy1D(result.data(), &zero, 1, queue_idx); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + + template + MGARDX_CONT static void SortByKey(SIZE n, SubArray<1, KeyT, CUDA> in_keys, + SubArray<1, ValueT, CUDA> in_values, + SubArray<1, KeyT, CUDA> out_keys, + SubArray<1, ValueT, CUDA> out_values, + Array<1, Byte, CUDA> &workspace, + bool workspace_allocated, int queue_idx) { + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); + cub::DeviceRadixSort::SortPairs( + d_temp_storage, temp_storage_bytes, in_keys.data(), out_keys.data(), + in_values.data(), out_values.data(), n, 0, sizeof(KeyT) * 8, stream); + ErrorAsyncCheck(cudaGetLastError(), "DeviceCollective::SortByKey"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(cudaDeviceSynchronize(), + "DeviceCollective::SortByKey"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + + template + MGARDX_CONT static void + ScanOpInclusiveByKey(SubArray<1, SIZE, CUDA> &key, + SubArray<1, ValueT, CUDA> &v, + SubArray<1, ValueT, CUDA> &result, int queue_idx) { + + thrust::equal_to binary_pred; + + struct ThrustBinaryOp { + MGARDX_CONT_EXEC + ValueT operator()(ValueT x, ValueT y) { + BinaryOpType op; + return op(x, y); + } + }; + + ThrustBinaryOp binary_op; + + thrust::inclusive_scan_by_key( + thrust::device, thrust::device_ptr(key.data()), + thrust::device_ptr(key.data() + key.shape(0)), + thrust::device_ptr(v.data()), + thrust::device_ptr(result.data()), binary_pred, binary_op); + ErrorAsyncCheck(cudaGetLastError(), + "DeviceCollective::ScanOpInclusiveByKey"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(cudaDeviceSynchronize(), + "DeviceCollective::ScanOpInclusiveByKey"); + } + } +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterHip.h b/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterHip.h new file mode 100644 index 000000000..997a45184 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterHip.h @@ -0,0 +1,2580 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "../Utilities/Exceptions.h" +#include "DeviceAdapter.h" + +#define HIP_ENABLE_WARP_SYNC_BUILTINS +#include +#include +#include +#include +// #include +#include + +// using namespace nvhip; +namespace cg = cooperative_groups; + +#ifndef MGARD_X_160_DEVICE_ADAPTER_HIP_H +#define MGARD_X_160_DEVICE_ADAPTER_HIP_H + +template struct SharedMemory { + MGARDX_EXEC operator T *() { + extern __shared__ int __smem[]; + return (T *)__smem; + } + + MGARDX_EXEC operator const T *() const { + extern __shared__ int __smem[]; + return (T *)__smem; + } +}; + +static __device__ __inline__ uint32_t __mywarpid() { + uint32_t warpid = threadIdx.y; + // asm volatile("mov.u32 %0, %%warpid;" : "=r"(warpid)); + return warpid; +} + +static __device__ __inline__ uint32_t __mylaneid() { + uint32_t laneid = threadIdx.x; + // asm volatile("mov.u32 %0, %%laneid;" : "=r"(laneid)); + return laneid; +} + +namespace mgard_x_160 { + +template +inline void ErrorAsyncCheckTask(hipError_t code, TaskType &task, + bool abort = true) { + if (code != hipSuccess) { + std::string msg = std::string(hipGetErrorString(code)) + + " while executing " + task.GetFunctorName().c_str() + + " with HIP (Async-check)"; + if (abort) + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; + } +} + +template +inline void ErrorSyncCheckTask(hipError_t code, TaskType &task, + bool abort = true) { + if (code != hipSuccess) { + std::string msg = std::string(hipGetErrorString(code)) + + " while executing " + task.GetFunctorName().c_str() + + " with HIP (Sync-check)"; + if (abort) + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; + } +} + +inline void ErrorAsyncCheck(hipError_t code, std::string task, + bool abort = true) { + if (code != hipSuccess) { + std::string msg = std::string(hipGetErrorString(code)) + + " while executing " + task.c_str() + + " with HIP (Async-check)"; + if (abort) + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; + } +} + +inline void ErrorSyncCheck(hipError_t code, std::string task, + bool abort = true) { + if (code != hipSuccess) { + std::string msg = std::string(hipGetErrorString(code)) + + " while executing " + task.c_str() + + " with HIP (Sync-check)"; + if (abort) + throw ProcessingException(msg); + else + std::cerr << msg << std::endl; + } +} + +#define gpuErrchk(ans) \ + { gpuAssert((ans), __FILE__, __LINE__); } + +inline void gpuAssert(hipError_t code, const char *file, int line, + bool abort = true) { + if (code != hipSuccess) { + if (abort) + throw ProcessingException(std::string("GPUassert: ") + + hipGetErrorString(code) + " " + file + " " + + std::to_string(line)); + else + fprintf(stderr, "GPUassert: %s %s %d\n", hipGetErrorString(code), file, + line); + } +} + +template <> struct SyncBlock { + MGARDX_EXEC static void Sync() { __syncthreads(); } +}; + +template <> struct SyncGrid { + MGARDX_EXEC static void Sync() { cg::this_grid().sync(); } +}; + +template +struct Atomic { + MGARDX_EXEC static T Min(T *result, T value) { + if constexpr (Scope == AtomicSystemScope) { + return atomicMin(result, value); + } else if constexpr (Scope == AtomicDeviceScope) { + return atomicMin(result, value); + } else { + return atomicMin(result, value); + } + } + MGARDX_EXEC static T Max(T *result, T value) { + if constexpr (Scope == AtomicSystemScope) { + return atomicMax(result, value); + } else if constexpr (Scope == AtomicDeviceScope) { + return atomicMax(result, value); + } else { + return atomicMax_(result, value); + } + } + MGARDX_EXEC static T Add(T *result, T value) { + if constexpr (Scope == AtomicSystemScope) { + return atomicAdd(result, value); + } else if constexpr (Scope == AtomicDeviceScope) { + return atomicAdd(result, value); + } else { + return atomicAdd(result, value); + } + } + MGARDX_EXEC static T Or(T *result, T value) { + // Cast to the matching-width unsigned type for portability across + // H = uint32/64 (atomicOr is only defined for 32-/64-bit unsigned ints). + if constexpr (sizeof(T) == 8) { + using U = unsigned long long int; + return static_cast( + atomicOr(reinterpret_cast(result), static_cast(value))); + } else { + using U = unsigned int; + return static_cast( + atomicOr(reinterpret_cast(result), static_cast(value))); + } + } +}; + +template <> struct Math { + template MGARDX_EXEC static T Min(T a, T b) { return min(a, b); } + template MGARDX_EXEC static T Max(T a, T b) { return max(a, b); } + MGARDX_EXEC static int ffs(unsigned int a) { return __ffs(a); } + MGARDX_EXEC static int ffsll(long long unsigned int a) { return __ffsll(a); } + MGARDX_EXEC + static uint64_t binary2negabinary(const int64_t x) { + return (x + (uint64_t)0xaaaaaaaaaaaaaaaaull) ^ + (uint64_t)0xaaaaaaaaaaaaaaaaull; + } + + MGARDX_EXEC + static uint32_t binary2negabinary(const int32_t x) { + return (x + (uint32_t)0xaaaaaaaau) ^ (uint32_t)0xaaaaaaaau; + } + + MGARDX_EXEC + static int64_t negabinary2binary(const uint64_t x) { + return (x ^ 0xaaaaaaaaaaaaaaaaull) - 0xaaaaaaaaaaaaaaaaull; + } + + MGARDX_EXEC + static int32_t negabinary2binary(const uint32_t x) { + return (x ^ 0xaaaaaaaau) - 0xaaaaaaaau; + } +}; + +template MGARDX_KERL void HipKernel(Task task) { + Byte *shared_memory = SharedMemory(); + task.GetFunctor().Init(gridDim.z, gridDim.y, gridDim.x, blockDim.z, + blockDim.y, blockDim.x, blockIdx.z, blockIdx.y, + blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, + shared_memory); + + task.GetFunctor().Operation1(); + SyncBlock::Sync(); + task.GetFunctor().Operation2(); + SyncBlock::Sync(); + task.GetFunctor().Operation3(); + SyncBlock::Sync(); + task.GetFunctor().Operation4(); + SyncBlock::Sync(); + task.GetFunctor().Operation5(); + SyncBlock::Sync(); + task.GetFunctor().Operation6(); + SyncBlock::Sync(); + task.GetFunctor().Operation7(); + SyncBlock::Sync(); + task.GetFunctor().Operation8(); + SyncBlock::Sync(); + task.GetFunctor().Operation9(); + SyncBlock::Sync(); + task.GetFunctor().Operation10(); + SyncBlock::Sync(); + task.GetFunctor().Operation11(); + SyncBlock::Sync(); + task.GetFunctor().Operation12(); + SyncBlock::Sync(); + task.GetFunctor().Operation13(); + SyncBlock::Sync(); + task.GetFunctor().Operation14(); + SyncBlock::Sync(); + task.GetFunctor().Operation15(); + SyncBlock::Sync(); + task.GetFunctor().Operation16(); + SyncBlock::Sync(); + task.GetFunctor().Operation17(); + SyncBlock::Sync(); + task.GetFunctor().Operation18(); + SyncBlock::Sync(); + task.GetFunctor().Operation19(); + SyncBlock::Sync(); + task.GetFunctor().Operation20(); +} + +template MGARDX_KERL void HipIterKernel(Task task) { + Byte *shared_memory = SharedMemory(); + + task.GetFunctor().Init(gridDim.z, gridDim.y, gridDim.x, blockDim.z, + blockDim.y, blockDim.x, blockIdx.z, blockIdx.y, + blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, + shared_memory); + + task.GetFunctor().Operation1(); + SyncBlock::Sync(); + + task.GetFunctor().Operation2(); + SyncBlock::Sync(); + + while (task.GetFunctor().LoopCondition1()) { + task.GetFunctor().Operation3(); + SyncBlock::Sync(); + task.GetFunctor().Operation4(); + SyncBlock::Sync(); + task.GetFunctor().Operation5(); + SyncBlock::Sync(); + task.GetFunctor().Operation6(); + SyncBlock::Sync(); + } + + task.GetFunctor().Operation7(); + SyncBlock::Sync(); + task.GetFunctor().Operation8(); + SyncBlock::Sync(); + task.GetFunctor().Operation9(); + SyncBlock::Sync(); + task.GetFunctor().Operation10(); + SyncBlock::Sync(); + + while (task.GetFunctor().LoopCondition2()) { + task.GetFunctor().Operation11(); + SyncBlock::Sync(); + task.GetFunctor().Operation12(); + SyncBlock::Sync(); + task.GetFunctor().Operation13(); + SyncBlock::Sync(); + task.GetFunctor().Operation14(); + SyncBlock::Sync(); + } + + task.GetFunctor().Operation15(); + SyncBlock::Sync(); + task.GetFunctor().Operation16(); + SyncBlock::Sync(); + task.GetFunctor().Operation17(); + SyncBlock::Sync(); +} + +template +MGARDX_KERL void HipHuffmanCLCustomizedKernel(Task task) { + Byte *shared_memory = SharedMemory(); + + task.GetFunctor().Init(gridDim.z, gridDim.y, gridDim.x, blockDim.z, + blockDim.y, blockDim.x, blockIdx.z, blockIdx.y, + blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, + shared_memory); + + task.GetFunctor().Operation1(); + SyncGrid::Sync(); + while (task.GetFunctor().LoopCondition1()) { + task.GetFunctor().Operation2(); + SyncGrid::Sync(); + task.GetFunctor().Operation3(); + SyncGrid::Sync(); + task.GetFunctor().Operation4(); + SyncGrid::Sync(); + task.GetFunctor().Operation5(); + SyncGrid::Sync(); + task.GetFunctor().Operation6(); + SyncBlock::Sync(); + if (task.GetFunctor().BranchCondition1()) { + while (task.GetFunctor().LoopCondition2()) { + task.GetFunctor().Operation7(); + SyncBlock::Sync(); + task.GetFunctor().Operation8(); + SyncBlock::Sync(); + task.GetFunctor().Operation9(); + SyncBlock::Sync(); + } + task.GetFunctor().Operation10(); + SyncGrid::Sync(); + task.GetFunctor().Operation11(); + SyncGrid::Sync(); + } + task.GetFunctor().Operation12(); + SyncGrid::Sync(); + task.GetFunctor().Operation13(); + SyncGrid::Sync(); + task.GetFunctor().Operation14(); + SyncGrid::Sync(); + task.GetFunctor().Operation15(); + SyncGrid::Sync(); + } +} + +#define SINGLE_KERNEL(OPERATION) \ + template \ + MGARDX_KERL void Single_##OPERATION##_Kernel(Task task) { \ + Byte *shared_memory = SharedMemory(); \ + task.GetFunctor().Init(gridDim.z, gridDim.y, gridDim.x, blockDim.z, \ + blockDim.y, blockDim.x, blockIdx.z, blockIdx.y, \ + blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, \ + shared_memory); \ + task.GetFunctor().OPERATION(); \ + } + +SINGLE_KERNEL(Operation1); +SINGLE_KERNEL(Operation2); +SINGLE_KERNEL(Operation3); +SINGLE_KERNEL(Operation4); +SINGLE_KERNEL(Operation5); +SINGLE_KERNEL(Operation6); +SINGLE_KERNEL(Operation7); +SINGLE_KERNEL(Operation8); +SINGLE_KERNEL(Operation9); +SINGLE_KERNEL(Operation10); +SINGLE_KERNEL(Operation11); +SINGLE_KERNEL(Operation12); +SINGLE_KERNEL(Operation13); +SINGLE_KERNEL(Operation14); +SINGLE_KERNEL(Operation15); + +#undef SINGLE_KERNEL + +template MGARDX_KERL void HipParallelMergeKernel(Task task) { + Byte *shared_memory = SharedMemory(); + + task.GetFunctor().Init(gridDim.z, gridDim.y, gridDim.x, blockDim.z, + blockDim.y, blockDim.x, blockIdx.z, blockIdx.y, + blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, + shared_memory); + + task.GetFunctor().Operation6(); + SyncBlock::Sync(); + while (task.GetFunctor().LoopCondition2()) { + task.GetFunctor().Operation7(); + SyncBlock::Sync(); + task.GetFunctor().Operation8(); + SyncBlock::Sync(); + task.GetFunctor().Operation9(); + SyncBlock::Sync(); + } + task.GetFunctor().Operation10(); +} + +template +MGARDX_KERL void HipHuffmanCWCustomizedKernel(Task task) { + Byte *shared_memory = SharedMemory(); + + task.GetFunctor().Init(gridDim.z, gridDim.y, gridDim.x, blockDim.z, + blockDim.y, blockDim.x, blockIdx.z, blockIdx.y, + blockIdx.x, threadIdx.z, threadIdx.y, threadIdx.x, + shared_memory); + + task.GetFunctor().Operation1(); + SyncGrid::Sync(); + task.GetFunctor().Operation2(); + SyncGrid::Sync(); + task.GetFunctor().Operation3(); + SyncGrid::Sync(); + + while (task.GetFunctor().LoopCondition1()) { + task.GetFunctor().Operation4(); + SyncGrid::Sync(); + task.GetFunctor().Operation5(); + SyncGrid::Sync(); + task.GetFunctor().Operation6(); + SyncGrid::Sync(); + task.GetFunctor().Operation7(); + SyncGrid::Sync(); + task.GetFunctor().Operation8(); + SyncGrid::Sync(); + } + task.GetFunctor().Operation9(); + SyncGrid::Sync(); + task.GetFunctor().Operation10(); + SyncGrid::Sync(); +} + +template <> class DeviceSpecification { +public: + MGARDX_CONT + DeviceSpecification() { + int original_device = 0; + hipGetDevice(&original_device); + hipGetDeviceCount(&NumDevices); + MaxSharedMemorySize = new int[NumDevices]; + WarpSize = new int[NumDevices]; + NumSMs = new int[NumDevices]; + ArchitectureGeneration = new int[NumDevices]; + MaxNumThreadsPerSM = new int[NumDevices]; + MaxNumThreadsPerTB = new int[NumDevices]; + AvailableMemory = new size_t[NumDevices]; + SupportCooperativeGroups = new bool[NumDevices]; + DeviceNames = new std::string[NumDevices]; + + for (int d = 0; d < NumDevices; d++) { + gpuErrchk(hipSetDevice(d)); + int maxbytes; + hipDeviceGetAttribute(&maxbytes, + hipDeviceAttributeMaxSharedMemoryPerBlock, d); + MaxSharedMemorySize[d] = maxbytes; + hipDeviceGetAttribute(&WarpSize[d], hipDeviceAttributeWarpSize, d); + hipDeviceGetAttribute(&NumSMs[d], hipDeviceAttributeMultiprocessorCount, + d); + hipDeviceGetAttribute(&MaxNumThreadsPerSM[d], + hipDeviceAttributeMaxThreadsPerMultiProcessor, d); + hipDeviceGetAttribute(&MaxNumThreadsPerTB[d], + hipDeviceAttributeMaxThreadsPerBlock, d); + SupportCooperativeGroups[d] = true; + hipDeviceProp_t prop; + hipGetDeviceProperties(&prop, d); + // WarpSize[d] holds the real hardware wavefront size queried above + // (64 on CDNA, 32 on RDNA). Code that launches warp-width thread + // blocks or partitions work by warp (Histogram, GenerateCL) must read + // this value at runtime rather than assuming MGARDX_WARP_SIZE (a + // CUDA-oriented compile-time constant) to stay correct and efficient + // across architectures. + // DeviceNames[d] = std::string(prop.name); // Not working in HIP + DeviceNames[d] = std::string("AMD GPU"); + } + // This loop's per-device hipSetDevice(d) calls otherwise leave the + // active HIP device at NumDevices-1 for the rest of the process, + // silently out of sync with DeviceRuntime::hip_dev_id (which + // stays at its default of 0 unless explicitly changed). Streams and + // allocations made before the mismatch is next resolved end up tied to + // the wrong device's context, which can surface much later as an + // unrelated illegal memory access. Restore the device that was active + // before this query loop ran. (Same bug as the CUDA backend's + // DeviceSpecification, confirmed and fixed there first.) + gpuErrchk(hipSetDevice(original_device)); + } + + MGARDX_CONT int GetNumDevices() { return NumDevices; } + + MGARDX_CONT int GetMaxSharedMemorySize(int dev_id) { + return MaxSharedMemorySize[dev_id]; + } + + MGARDX_CONT int GetWarpSize(int dev_id) { return WarpSize[dev_id]; } + + MGARDX_CONT int GetNumSMs(int dev_id) { return NumSMs[dev_id]; } + + MGARDX_CONT int GetArchitectureGeneration(int dev_id) { + return ArchitectureGeneration[dev_id]; + } + + MGARDX_CONT int GetMaxNumThreadsPerSM(int dev_id) { + return MaxNumThreadsPerSM[dev_id]; + } + + MGARDX_CONT int GetMaxNumThreadsPerTB(int dev_id) { + return MaxNumThreadsPerTB[dev_id]; + } + + MGARDX_CONT size_t GetAvailableMemory(int dev_id) { + gpuErrchk(hipSetDevice(dev_id)); + size_t free, total; + hipMemGetInfo(&free, &total); + AvailableMemory[dev_id] = total; + return AvailableMemory[dev_id]; + } + + MGARDX_CONT bool SupportCG(int dev_id) { + return SupportCooperativeGroups[dev_id]; + } + + MGARDX_CONT std::string GetDeviceName(int dev_id) { + return DeviceNames[dev_id]; + } + + MGARDX_CONT + ~DeviceSpecification() { + delete[] MaxSharedMemorySize; + delete[] WarpSize; + delete[] NumSMs; + delete[] ArchitectureGeneration; + delete[] MaxNumThreadsPerSM; + delete[] MaxNumThreadsPerTB; + delete[] AvailableMemory; + delete[] SupportCooperativeGroups; + delete[] DeviceNames; + } + + int NumDevices; + int *MaxSharedMemorySize; + int *WarpSize; + int *NumSMs; + int *ArchitectureGeneration; + int *MaxNumThreadsPerSM; + int *MaxNumThreadsPerTB; + size_t *AvailableMemory; + bool *SupportCooperativeGroups; + std::string *DeviceNames; +}; + +template <> class DeviceQueues { +public: + MGARDX_CONT + void Initialize() { + if (!initialized) { + log::dbg("Calling DeviceQueues::Initialize"); + int original_device = 0; + hipGetDevice(&original_device); + hipGetDeviceCount(&NumDevices); + streams = new hipStream_t *[NumDevices]; + for (int d = 0; d < NumDevices; d++) { + gpuErrchk(hipSetDevice(d)); + streams[d] = new hipStream_t[MGARDX_NUM_QUEUES]; + for (SIZE i = 0; i < MGARDX_NUM_QUEUES; i++) { + gpuErrchk(hipStreamCreate(&streams[d][i])); + } + } + // See DeviceSpecification's constructor for why this restore matters: + // otherwise the active device is left at NumDevices-1 here too. + gpuErrchk(hipSetDevice(original_device)); + initialized = true; + } + } + + MGARDX_CONT + void Destroy() { + if (initialized) { + log::dbg("Calling DeviceQueues::Destroy"); + int original_device = 0; + hipGetDevice(&original_device); + for (int d = 0; d < NumDevices; d++) { + gpuErrchk(hipSetDevice(d)); + for (int i = 0; i < MGARDX_NUM_QUEUES; i++) { + gpuErrchk(hipStreamDestroy(streams[d][i])); + } + delete[] streams[d]; + } + gpuErrchk(hipSetDevice(original_device)); + delete[] streams; + streams = nullptr; + initialized = false; + } + } + + MGARDX_CONT + DeviceQueues() { Initialize(); } + + MGARDX_CONT hipStream_t GetQueue(int dev_id, SIZE queue_id) { + Initialize(); + return streams[dev_id][queue_id]; + } + + MGARDX_CONT void SyncQueue(int dev_id, SIZE queue_id) { + Initialize(); + hipStreamSynchronize(streams[dev_id][queue_id]); + } + + MGARDX_CONT void SyncAllQueues(int dev_id) { + Initialize(); + for (SIZE i = 0; i < MGARDX_NUM_QUEUES; i++) { + gpuErrchk(hipStreamSynchronize(streams[dev_id][i])); + } + } + + MGARDX_CONT + ~DeviceQueues() {} + + bool initialized = false; + int NumDevices; + hipStream_t **streams = nullptr; +}; + +extern int hip_dev_id; +#pragma omp threadprivate(hip_dev_id) + +// AMD wavefront sub-group (HIP). UNTESTED -- no AMD hardware available here. +// See the verification notes at the original definition: width via +// __AMDGCN_WAVEFRONT_SIZE__ (64 CDNA / 32 RDNA), 64-bit ballot mask, +// __ballot/__shfl/__ffsll/__syncwarp. If the project's "64 triggers a bug" note +// bites, force size()=32 and launch half-wavefront blocks. +template <> struct SubGroup { + using mask_t = unsigned long long; +#ifdef __AMDGCN_WAVEFRONT_SIZE__ + static constexpr int size() { return __AMDGCN_WAVEFRONT_SIZE__; } +#else + static constexpr int size() { return 64; } +#endif + __device__ int lane() const { return threadIdx.x & (size() - 1); } + __device__ mask_t full_mask() const { + return size() == 64 ? ~0ull : ((1ull << size()) - 1); + } + template __device__ T shfl(T v, int src) const { + return __shfl(v, src, size()); + } + __device__ mask_t ballot(int pred) const { return (mask_t)__ballot(pred); } + __device__ int ffs(mask_t m) const { return __ffsll((long long)m); } + // __syncwarp() is opt-in (behind HIP_ENABLE_WARP_SYNC_BUILTINS) before ROCm + // 7.0 and default-on from 7.0 onward; see the HIP find_package version + // floor in the top-level CMakeLists.txt. + __device__ void sync() const { __syncwarp(); } +}; + +template <> class DeviceRuntime { +public: + MGARDX_CONT + DeviceRuntime() {} + + MGARDX_CONT static void Initialize() { queues.Initialize(); } + + MGARDX_CONT static void Finalize() { queues.Destroy(); } + + MGARDX_CONT static int GetDeviceCount() { return DeviceSpecs.NumDevices; } + + MGARDX_CONT static void SelectDevice(SIZE dev_id) { + gpuErrchk(hipSetDevice(dev_id)); + hip_dev_id = dev_id; + } + + MGARDX_CONT static int GetDevice() { + gpuErrchk(hipGetDevice(&hip_dev_id)); + return hip_dev_id; + } + + MGARDX_CONT static hipStream_t GetQueue(SIZE queue_id) { + return queues.GetQueue(hip_dev_id, queue_id); + } + + MGARDX_CONT static void SyncQueue(SIZE queue_id) { + log::dbg("Calling DeviceRuntime::SyncQueue: " + + std::to_string(queue_id)); + queues.SyncQueue(hip_dev_id, queue_id); + } + + MGARDX_CONT static void SyncAllQueues() { queues.SyncAllQueues(hip_dev_id); } + + MGARDX_CONT static void SyncDevice() { + log::dbg("Calling DeviceRuntime::SyncDevice: "); + gpuErrchk(hipDeviceSynchronize()); + } + + MGARDX_CONT static std::string GetDeviceName() { + return DeviceSpecs.GetDeviceName(hip_dev_id); + } + + MGARDX_CONT static int GetMaxSharedMemorySize() { + return DeviceSpecs.GetMaxSharedMemorySize(hip_dev_id); + } + + MGARDX_CONT static int GetWarpSize() { + return DeviceSpecs.GetWarpSize(hip_dev_id); + } + + MGARDX_CONT static int GetNumSMs() { + return DeviceSpecs.GetNumSMs(hip_dev_id); + } + + MGARDX_CONT static int GetArchitectureGeneration() { + return DeviceSpecs.GetArchitectureGeneration(hip_dev_id); + } + + MGARDX_CONT static int GetMaxNumThreadsPerSM() { + return DeviceSpecs.GetMaxNumThreadsPerSM(hip_dev_id); + } + + MGARDX_CONT static int GetMaxNumThreadsPerTB() { + return DeviceSpecs.GetMaxNumThreadsPerTB(hip_dev_id); + } + + MGARDX_CONT static size_t GetAvailableMemory() { + return DeviceSpecs.GetAvailableMemory(hip_dev_id); + } + + MGARDX_CONT static bool SupportCG() { + return DeviceSpecs.SupportCG(hip_dev_id); + } + + template + MGARDX_CONT static int + GetOccupancyMaxActiveBlocksPerSM(FunctorType functor, int blockSize, + size_t dynamicSMemSize) { + int numBlocks = 0; + + if constexpr (std::is_base_of, FunctorType>::value) { + gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( + &numBlocks, HipKernel>, blockSize, + dynamicSMemSize)); + } else if constexpr (std::is_base_of, + FunctorType>::value) { + gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( + &numBlocks, HipIterKernel>, blockSize, + dynamicSMemSize)); + } else if constexpr (std::is_base_of, + FunctorType>::value) { + gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( + &numBlocks, HipHuffmanCLCustomizedKernel>, + blockSize, dynamicSMemSize)); + } else if constexpr (std::is_base_of, + FunctorType>::value) { + gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( + &numBlocks, HipHuffmanCWCustomizedKernel>, + blockSize, dynamicSMemSize)); + } else { + throw std::runtime_error("GetOccupancyMaxActiveBlocksPerSM Error!"); + } + // HIP tends to over estimate this value + numBlocks /= 2; + return numBlocks; + } + + template + MGARDX_CONT static void SetMaxDynamicSharedMemorySize(FunctorType functor, + int maxbytes) { + + if constexpr (std::is_base_of, FunctorType>::value) { + gpuErrchk(hipFuncSetAttribute((const void *)HipKernel>, + hipFuncAttributeMaxDynamicSharedMemorySize, + maxbytes)); + } else if constexpr (std::is_base_of, + FunctorType>::value) { + gpuErrchk(hipFuncSetAttribute( + (const void *)HipIterKernel>, + hipFuncAttributeMaxDynamicSharedMemorySize, maxbytes)); + } else if constexpr (std::is_base_of, + FunctorType>::value) { + gpuErrchk(hipFuncSetAttribute( + (const void *)HipHuffmanCLCustomizedKernel>, + hipFuncAttributeMaxDynamicSharedMemorySize, maxbytes)); + } else if constexpr (std::is_base_of, + FunctorType>::value) { + gpuErrchk(hipFuncSetAttribute( + (const void *)HipHuffmanCWCustomizedKernel>, + hipFuncAttributeMaxDynamicSharedMemorySize, maxbytes)); + } else { + throw std::runtime_error("SetPreferredSharedMemoryCarveout Error!"); + } + } + + MGARDX_CONT + ~DeviceRuntime() {} + + static DeviceQueues queues; + static bool SyncAllKernelsAndCheckErrors; + static DeviceSpecification DeviceSpecs; + static bool TimingAllKernels; + static bool PrintKernelConfig; +}; + +template <> class MemoryManager { +public: + MGARDX_CONT + MemoryManager(){}; + + template + MGARDX_CONT static void Malloc1D(T *&ptr, SIZE n, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Malloc1D"); + if (n == 0) + return; + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + using converted_T = + typename std::conditional::value, Byte, T>::type; + gpuErrchk(hipMalloc(&ptr, n * sizeof(converted_T))); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void MallocND(T *&ptr, SIZE n1, SIZE n2, SIZE &ld, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MallocND"); + if (n1 == 0 || n2 == 0) + return; + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + using converted_T = + typename std::conditional::value, Byte, T>::type; + size_t pitch = 0; + gpuErrchk(hipMallocPitch((void **)&ptr, &pitch, n1 * sizeof(converted_T), + (size_t)n2)); + ld = pitch / sizeof(converted_T); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void + MallocManaged1D(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MallocManaged1D"); + if (n == 0) + return; + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + gpuErrchk(hipMallocManaged(&ptr, n * sizeof(T))); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void Free(T *ptr, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Free"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (ptr == nullptr) + return; + gpuErrchk(hipFree(ptr)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void Copy1D(T *dst_ptr, const T *src_ptr, SIZE n, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Copy1D"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + using converted_T = + typename std::conditional::value, Byte, T>::type; + hipStream_t stream = DeviceRuntime::GetQueue(queue_idx); + gpuErrchk(hipMemcpyAsync(dst_ptr, src_ptr, n * sizeof(converted_T), + hipMemcpyDefault, stream)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, + SIZE src_ld, SIZE n1, SIZE n2, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::CopyND"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + using converted_T = + typename std::conditional::value, Byte, T>::type; + hipStream_t stream = DeviceRuntime::GetQueue(queue_idx); + // HIP seesm to have bug when n2 = 1 + if (n2 != 1) { + gpuErrchk(hipMemcpy2DAsync(dst_ptr, dst_ld * sizeof(converted_T), src_ptr, + src_ld * sizeof(converted_T), + n1 * sizeof(converted_T), n2, hipMemcpyDefault, + stream)); + } else { + gpuErrchk(hipMemcpyAsync(dst_ptr, src_ptr, n1 * sizeof(converted_T), + hipMemcpyDefault, stream)); + } + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void + MallocHost(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MallocHost"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + using converted_T = + typename std::conditional::value, Byte, T>::type; + gpuErrchk(hipMallocHost((void **)&ptr, n * sizeof(converted_T))); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void FreeHost(T *ptr, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::FreeHost"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (ptr == nullptr) + return; + gpuErrchk(hipFreeHost(ptr)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void Memset1D(T *ptr, SIZE n, int value, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Memset1D"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + using converted_T = + typename std::conditional::value, Byte, T>::type; + hipStream_t stream = DeviceRuntime::GetQueue(queue_idx); + gpuErrchk(hipMemsetAsync(ptr, value, n * sizeof(converted_T), stream)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void MemsetND(T *ptr, SIZE ld, SIZE n1, SIZE n2, int value, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MemsetND"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + using converted_T = + typename std::conditional::value, Byte, T>::type; + hipStream_t stream = DeviceRuntime::GetQueue(queue_idx); + gpuErrchk(hipMemset2DAsync(ptr, ld * sizeof(converted_T), value, + n1 * sizeof(converted_T), n2, stream)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template MGARDX_CONT static bool IsDevicePointer(T *ptr) { + log::dbg("Calling MemoryManager::IsDevicePointer"); + hipPointerAttribute_t attr; + hipPointerGetAttributes(&attr, ptr); + return attr.type == hipMemoryTypeDevice; + } + + template MGARDX_CONT static int GetPointerDevice(T *ptr) { + log::dbg("Calling MemoryManager::GetPointerDevice"); + hipPointerAttribute_t attr; + hipPointerGetAttributes(&attr, ptr); + return attr.device; + } + + template MGARDX_CONT static bool CheckHostRegister(T *ptr) { + log::dbg("Calling MemoryManager::CheckHostRegister"); + // Disabled since it is not working correctly + // unsigned int flags; + // hipHostGetFlags(&flags, (void *)ptr); + // return hipGetLastError() == hipSuccess; + return true; + } + + template MGARDX_CONT static void HostRegister(T *ptr, SIZE n) { + log::dbg("Calling MemoryManager::HostRegister"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + // if (!CheckHostRegister(ptr)) { + if (n > 0) { + gpuErrchk(hipHostRegister((void *)ptr, n * sizeof(converted_T), + hipHostRegisterPortable)); + } + //} + } + + template MGARDX_CONT static void HostUnregister(T *ptr) { + log::dbg("Calling MemoryManager::HostUnregister"); + // if (CheckHostRegister(ptr)) { + gpuErrchk(hipHostUnregister((void *)ptr)); + //} + } +}; + +#define ALIGN_LEFT 0 // for encoding +#define ALIGN_RIGHT 1 // for decoding + +#define Sign_Encoding_Atomic 0 +#define Sign_Encoding_Reduce 1 +#define Sign_Encoding_Ballot 2 + +#define Sign_Decoding_Parallel 0 + +#define BINARY 0 +#define NEGABINARY 1 + +#define Bit_Transpose_Serial_All 0 +#define Bit_Transpose_Parallel_B_Serial_b 1 +#define Bit_Transpose_Parallel_B_Atomic_b 2 +#define Bit_Transpose_Parallel_B_Reduce_b 3 +#define Bit_Transpose_Parallel_B_Ballot_b 4 +#define Bit_Transpose_TCU 5 + +#define Warp_Bit_Transpose_Serial_All 0 +#define Warp_Bit_Transpose_Parallel_B_Serial_b 1 +#define Warp_Bit_Transpose_Serial_B_Atomic_b 2 +#define Warp_Bit_Transpose_Serial_B_Reduce_b 3 +#define Warp_Bit_Transpose_Serial_B_Ballot_b 4 +#define Warp_Bit_Transpose_TCU 5 + +#define Error_Collecting_Disable -1 +#define Error_Collecting_Serial_All 0 +#define Error_Collecting_Parallel_Bitplanes_Serial_Error 1 +#define Error_Collecting_Parallel_Bitplanes_Atomic_Error 2 +#define Error_Collecting_Parallel_Bitplanes_Reduce_Error 3 + +#define Warp_Error_Collecting_Disable -1 +#define Warp_Error_Collecting_Serial_All 0 +#define Warp_Error_Collecting_Parallel_Bitplanes_Serial_Error 1 +#define Warp_Error_Collecting_Serial_Bitplanes_Atomic_Error 2 +#define Warp_Error_Collecting_Serial_Bitplanes_Reduce_Error 3 + +typedef unsigned long long int uint64_cu; + +template +struct BlockReduce { + typedef hipcub::BlockReduce + BlockReduceType; + using TempStorageType = typename BlockReduceType::TempStorage; + + MGARDX_EXEC + static void Sum(T intput, T &output) { + __shared__ TempStorageType temp_storage; + BlockReduceType blockReduce(temp_storage); + return blockReduce.Sum(intput); + } + + MGARDX_EXEC + static void Max(T intput, T &output) { + __shared__ TempStorageType temp_storage; + BlockReduceType blockReduce(temp_storage); + return blockReduce.Reduce(intput, hipcub::Max()); + } +}; + +template +struct BlockBitTranspose { + + typedef hipcub::WarpReduce WarpReduceType; + using WarpReduceStorageType = typename WarpReduceType::TempStorage; + + typedef hipcub::BlockReduce< + T_trans, nblockx, hipcub::BLOCK_REDUCE_WARP_REDUCTIONS, nblocky, nblockz> + BlockReduceType; + using BlockReduceStorageType = typename BlockReduceType::TempStorage; + + MGARDX_EXEC + static void Serial_All(T_org *v, T_trans *tv, SIZE b, SIZE B, SIZE IdX, + SIZE IdY) { + if (IdX == 0 && IdY == 0) { + // printf("add-in: %llu %u\n", v, v[0]); + // for (int i = 0; i < b; i++) { + // printf("v: %u\n", v[i]); + // } + for (SIZE B_idx = 0; B_idx < B; B_idx++) { + T_trans buffer = 0; + for (SIZE b_idx = 0; b_idx < b; b_idx++) { + T_trans bit = (v[b_idx] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + if (ALIGN == ALIGN_LEFT) { + buffer += bit << (sizeof(T_trans) * 8 - 1 - b_idx); + // if (B_idx == 0) { + // printf("%u %u %u\n", B_idx, b_idx, bit); + // print_bits(buffer, sizeof(T_trans)*8, false); + // printf("\n"); + // } + } else if (ALIGN == ALIGN_RIGHT) { + buffer += bit << (b - 1 - b_idx); + // if (b_idx == 0) printf("%u %u %u\n", B_idx, b_idx, bit); + } else { + } + // if (j == 0 ) {printf("i %u j %u shift %u bit %u\n", i,j,b-1-j, + // bit); } + } + + // printf("buffer: %u\n", buffer); + + tv[B_idx] = buffer; + } + } + } + + MGARDX_EXEC + static void Parallel_B_Serial_b(T_org *v, T_trans *tv, SIZE b, SIZE B, + SIZE IdX, SIZE IdY) { + if (IdY == 0) { + for (SIZE B_idx = IdX; B_idx < B; B_idx += 32) { + T_trans buffer = 0; + for (SIZE b_idx = 0; b_idx < b; b_idx++) { + T_trans bit = (v[b_idx] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + if (ALIGN == ALIGN_LEFT) { + buffer += bit << sizeof(T_trans) * 8 - 1 - b_idx; + } else if (ALIGN == ALIGN_RIGHT) { + buffer += bit << (b - 1 - b_idx); + // if (b_idx == 0) printf("%u %u %u\n", B_idx, b_idx, bit); + } else { + } + } + tv[B_idx] = buffer; + } + } + } + + MGARDX_EXEC + static void Parallel_B_Atomic_b(T_org *v, T_trans *tv, SIZE b, SIZE B, + SIZE IdX, SIZE IdY) { + if (IdX < b && IdY < B) { + SIZE i = IdX; + for (SIZE B_idx = IdY; B_idx < B; B_idx += 32) { + for (SIZE b_idx = IdX; b_idx < b; b_idx += 32) { + T_trans bit = (v[b_idx] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + T_trans shifted_bit; + if (ALIGN == ALIGN_LEFT) { + shifted_bit = bit << sizeof(T_trans) * 8 - 1 - b_idx; + } else if (ALIGN == ALIGN_RIGHT) { + shifted_bit = bit << b - 1 - b_idx; + } else { + } + T_trans *sum = &(tv[B_idx]); + Atomic::Add( + sum, shifted_bit); + } + } + } + } + + MGARDX_EXEC + static void Parallel_B_Reduction_b(T_org *v, T_trans *tv, SIZE b, SIZE B, + SIZE IdX, SIZE IdY) { + + // __syncthreads(); long long start = clock64(); + + __shared__ WarpReduceStorageType warp_storage[32]; + + SIZE warp_idx = IdY; + SIZE lane_idx = IdX; + SIZE B_idx, b_idx; + T_trans bit = 0; + T_trans shifted_bit = 0; + T_trans sum = 0; + + // __syncthreads(); start = clock64() - start; + // if (IdY == 0 && IdX == 0) printf("time0: %llu\n", start); + // __syncthreads(); start = clock64(); + + for (SIZE B_idx = IdY; B_idx < B; B_idx += 32) { + sum = 0; + for (SIZE b_idx = IdX; b_idx < ((b - 1) / 32 + 1) * 32; b_idx += 32) { + shifted_bit = 0; + if (b_idx < b && B_idx < B) { + bit = (v[b_idx] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + if (ALIGN == ALIGN_LEFT) { + shifted_bit = bit << sizeof(T_trans) * 8 - 1 - b_idx; + } else if (ALIGN == ALIGN_RIGHT) { + shifted_bit = bit << b - 1 - b_idx; + } else { + } + } + + // __syncthreads(); start = clock64() - start; + // if (IdY == 0 && IdX == 0) printf("time1: %llu\n", + // start); + // __syncthreads(); start = clock64(); + + sum += WarpReduceType(warp_storage[warp_idx]).Sum(shifted_bit); + // if (B_idx == 32) printf("shifted_bit[%u] %u sum %u\n", b_idx, + // shifted_bit, sum); + } + if (lane_idx == 0) { + tv[B_idx] = sum; + } + } + + // __syncthreads(); start = clock64() - start; + // if (IdY == 0 && IdX == 0) printf("time2: %llu\n", start); + // __syncthreads(); start = clock64(); + } + + MGARDX_EXEC + static void Parallel_B_Ballot_b(T_org *v, T_trans *tv, SIZE b, SIZE B, + SIZE IdX, SIZE IdY) { + + SIZE warp_idx = IdY; + SIZE lane_idx = IdX; + SIZE B_idx, b_idx; + int bit = 0; + T_trans sum = 0; + + // __syncthreads(); start = clock64() - start; + // if (IdY == 0 && IdX == 0) printf("time0: %llu\n", start); + // __syncthreads(); start = clock64(); + + for (SIZE B_idx = IdY; B_idx < B; B_idx += 32) { + sum = 0; + SIZE shift = 0; + for (SIZE b_idx = IdX; b_idx < ((b - 1) / 32 + 1) * 32; b_idx += 32) { + bit = 0; + if (b_idx < b && B_idx < B) { + if (ALIGN == ALIGN_LEFT) { + bit = (v[sizeof(T_trans) * 8 - 1 - b_idx] >> + (sizeof(T_org) * 8 - 1 - B_idx)) & + 1u; + } else if (ALIGN == ALIGN_RIGHT) { + bit = (v[b - 1 - b_idx] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + } else { + } + } + + // __syncthreads(); start = clock64() - start; + // if (IdY == 0 && IdX == 0) printf("time1: %llu\n", + // start); + // __syncthreads(); start = clock64(); + sum += ((T_trans)__ballot(bit)) << shift; + // sum += WarpReduceType(warp_storage[warp_idx]).Sum(shifted_bit); + // if (B_idx == 32) printf("shifted_bit[%u] %u sum %u\n", b_idx, + // shifted_bit, sum); + shift += 32; + } + if (lane_idx == 0) { + tv[B_idx] = sum; + } + } + + // __syncthreads(); start = clock64() - start; + // if (IdY == 0 && IdX == 0) printf("time2: %llu\n", start); + // __syncthreads(); start = clock64(); + + // __syncthreads(); long long start = clock64(); + + // SIZE i = IdX; + // SIZE B_idx = IdY; + // SIZE b_idx = IdX; + + // __syncthreads(); start = clock64() - start; + // if (IdY == 0 && IdX == 0) printf("time0: %llu\n", start); + // __syncthreads(); start = clock64(); + + // int bit = (v[b-1-b_idx] >> (sizeof(T_org)*8 - 1 - B_idx)) & 1u; + + // __syncthreads(); + // start = clock64() - start; + // if (IdY == 0 && IdX == 0) printf("time1: %llu\n", start); + // __syncthreads(); start = clock64(); + + // printf("b_idx[%u]: bit %d\n", b_idx, bit); + // unsigned int sum = __ballot_sync (0xffffffff, bit); + // printf("b_idx[%u]: sum %u\n", b_idx, sum); + // if (b_idx) tv[B_idx] = sum; + + // __syncthreads(); start = clock64() - start; + // if (IdY == 0 && IdX == 0) printf("time2: %llu\n", start); + // __syncthreads(); start = clock64(); + } + + MGARDX_EXEC + static void Transpose(T_org *v, T_trans *tv, SIZE b, SIZE B, SIZE IdX, + SIZE IdY) { + if (METHOD == Bit_Transpose_Serial_All) + Serial_All(v, tv, b, B, IdX, IdY); + else if (METHOD == Bit_Transpose_Parallel_B_Serial_b) + Parallel_B_Serial_b(v, tv, b, B, IdX, IdY); + else if (METHOD == Bit_Transpose_Parallel_B_Atomic_b) + Parallel_B_Atomic_b(v, tv, b, B, IdX, IdY); + else if (METHOD == Bit_Transpose_Parallel_B_Reduce_b) + Parallel_B_Reduction_b(v, tv, b, B, IdX, IdY); + else if (METHOD == Bit_Transpose_Parallel_B_Ballot_b) + Parallel_B_Ballot_b(v, tv, b, B, IdX, IdY); + // else { printf("Bit Transpose Wrong Algorithm Type!\n"); } + // else if (METHOD == 5) TCU(v, tv, b, B, IdX, IdY); + } +}; + +template +struct BlockErrorCollect { + + typedef hipcub::WarpReduce WarpReduceType; + using WarpReduceStorageType = typename WarpReduceType::TempStorage; + + typedef hipcub::BlockReduce< + T_error, nblockx, hipcub::BLOCK_REDUCE_WARP_REDUCTIONS, nblocky, nblockz> + BlockReduceType; + using BlockReduceStorageType = typename BlockReduceType::TempStorage; + + MGARDX_EXEC + static void Serial_All(T *v, T_error *temp, T_error *errors, SIZE num_elems, + SIZE num_bitplanes, SIZE IdX, SIZE IdY) { + if (IdX == 0 && IdY == 0) { + for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx++) { + T data = v[elem_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + for (SIZE bitplane_idx = 0; bitplane_idx < num_bitplanes; + bitplane_idx++) { + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + } + errors[num_bitplanes - bitplane_idx] += diff * diff; + // if (blockIdx.x == 0 && num_bitplanes-bitplane_idx == 2) { + // printf("elem error[%u]: %f\n", elem_idx, diff * diff); + // } + } + errors[0] += data * data; + } + } + } + + MGARDX_EXEC + static void Parallel_Bitplanes_Serial_Error(T *v, T_error *temp, + T_error *errors, SIZE num_elems, + SIZE num_bitplanes, SIZE IdX, + SIZE IdY) { + SIZE bitplane_idx = IdY * nblockx + IdX; + if (bitplane_idx < num_bitplanes) { + for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx++) { + T data = v[elem_idx]; + T_fp fp_data = (T_fp)fabs(v[elem_idx]); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + mantissa; + } + errors[num_bitplanes - bitplane_idx] += diff * diff; + } + } + if (bitplane_idx == 0) { + for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx++) { + T data = v[elem_idx]; + errors[0] += data * data; + } + } + } + + MGARDX_EXEC + static void Parallel_Bitplanes_Atomic_Error(T *v, T_error *temp, + T_error *errors, SIZE num_elems, + SIZE num_bitplanes, SIZE IdX, + SIZE IdY) { + for (SIZE elem_idx = IdX; elem_idx < num_elems; elem_idx += nblockx) { + for (SIZE bitplane_idx = IdY; bitplane_idx < num_bitplanes; + bitplane_idx += nblocky) { + T data = v[elem_idx]; + T_fp fp_data = (T_fp)fabs(v[elem_idx]); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + mantissa; + } + temp[(num_bitplanes - bitplane_idx) * num_elems + elem_idx] = + diff * diff; + if (bitplane_idx == 0) { + temp[elem_idx] = data * data; + } + } + } + __syncthreads(); + // if (blockIdx.x == 0 && IdX == 0 && IdY == 0) { + // for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx += 1) { + // printf("elem error[%u]: %f\n", elem_idx, temp[(2) * num_elems + + // elem_idx]); + // } + // } + for (SIZE bitplane_idx = IdY; bitplane_idx < num_bitplanes + 1; + bitplane_idx += nblocky) { + for (SIZE elem_idx = IdX; elem_idx < ((num_elems - 1) / 32 + 1) * 32; + elem_idx += 32) { + T_error error = 0; + if (elem_idx < num_elems) { + error = temp[(num_bitplanes - bitplane_idx) * num_elems + elem_idx]; + } + T_error *sum = &(errors[num_bitplanes - bitplane_idx]); + Atomic::Add(sum, + error); + } + } + } + + MGARDX_EXEC + static void Parallel_Bitplanes_Reduce_Error(T *v, T_error *temp, + T_error *errors, SIZE num_elems, + SIZE num_bitplanes, SIZE IdX, + SIZE IdY) { + for (SIZE elem_idx = IdX; elem_idx < num_elems; elem_idx += nblockx) { + for (SIZE bitplane_idx = IdY; bitplane_idx < num_bitplanes; + bitplane_idx += nblocky) { + T data = v[elem_idx]; + T_fp fp_data = (T_fp)fabs(v[elem_idx]); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + mantissa; + } + temp[(num_bitplanes - bitplane_idx) * num_elems + elem_idx] = + diff * diff; + // if (blockIdx.x == 0 && num_bitplanes - bitplane_idx == 31) { + // printf("elem_idx: %u, data: %f, fp_data: %u, mask: %u, mantissa: + // %f, diff: %f\n", + // elem_idx, data, fp_data, mask, mantissa, diff); + // } + if (bitplane_idx == 0) { + temp[elem_idx] = data * data; + } + } + } + __syncthreads(); + + // if (blockIdx.x == 0 && IdX == 0 && IdY == 0) { + // for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx += 1) { + // printf("elem error[%u]: %f\n", elem_idx, temp[(31) * num_elems + + // elem_idx]); + // } + // } + + __shared__ WarpReduceStorageType warp_storage[nblocky]; + + for (SIZE bitplane_idx = IdY; bitplane_idx < num_bitplanes + 1; + bitplane_idx += nblocky) { + T error_sum = 0; + for (SIZE elem_idx = IdX; elem_idx < ((num_elems - 1) / 32 + 1) * 32; + elem_idx += 32) { + T_error error = 0; + if (elem_idx < num_elems) { + error = temp[(num_bitplanes - bitplane_idx) * num_elems + elem_idx]; + } + error_sum += WarpReduceType(warp_storage[IdY]).Sum(error); + // errors[num_bitplanes - bitplane_idx] += + // WarpReduceType(warp_storage[IdY]).Sum(error); + } + if (IdX == 0) { + errors[num_bitplanes - bitplane_idx] = error_sum; + } + } + // __syncthreads(); + // if (blockIdx.x == 0 && IdX == 0 && IdY == 0) { + // for (int i = 0; i < num_bitplanes + 1; i++) { + // printf("error[%d]: %f\n", i, errors[i]); + // } + // } + } + + MGARDX_EXEC + static void Collect(T *v, T_error *temp, T_error *errors, SIZE num_elems, + SIZE num_bitplanes, SIZE IdX, SIZE IdY) { + if (METHOD == Error_Collecting_Serial_All) + Serial_All(v, temp, errors, num_elems, num_bitplanes, IdX, IdY); + else if (METHOD == Error_Collecting_Parallel_Bitplanes_Serial_Error) + Parallel_Bitplanes_Serial_Error(v, temp, errors, num_elems, num_bitplanes, + IdX, IdY); + else if (METHOD == Error_Collecting_Parallel_Bitplanes_Atomic_Error) + Parallel_Bitplanes_Atomic_Error(v, temp, errors, num_elems, num_bitplanes, + IdX, IdY); + else if (METHOD == Error_Collecting_Parallel_Bitplanes_Reduce_Error) + Parallel_Bitplanes_Reduce_Error(v, temp, errors, num_elems, num_bitplanes, + IdX, IdY); + // else if (METHOD == Error_Collecting_Disable) {} + // else { printf("Error Collecting Wrong Algorithm Type!\n"); } + } +}; + +template +struct WarpBitTranspose { + + typedef hipcub::WarpReduce WarpReduceType; + using WarpReduceStorageType = typename WarpReduceType::TempStorage; + + MGARDX_EXEC static void Serial_All(T_org *v, SIZE inc_v, T_trans *tv, + SIZE inc_tv, SIZE LaneId) { + // long long start; + // if (IdX == 0 && IdY == 0) { start = clock64(); } + if (LaneId == 0) { + for (SIZE B_idx = 0; B_idx < B; B_idx++) { + T_trans buffer = 0; + for (SIZE b_idx = 0; b_idx < b; b_idx++) { + T_trans bit = + (v[b_idx * inc_v] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + // if (blockIdx.x == 0 )printf("bit: %u\n", bit); + if (ALIGN == ALIGN_LEFT) { + buffer += bit << (sizeof(T_trans) * 8 - 1 - b_idx); + } else if (ALIGN == ALIGN_RIGHT) { + buffer += bit << (b - 1 - b_idx); + } else { + } + } + tv[B_idx * inc_tv] = buffer; + } + } + // if (IdX == 0 && IdY == 0) { start = clock64() - start; + // printf("Serial_All time : %llu\n", start); } + } + + MGARDX_EXEC static void Parallel_B_Serial_b(T_org *v, SIZE inc_v, T_trans *tv, + SIZE inc_tv, SIZE LaneId) { + // long long start; + // if (IdX == 0 && IdY == 0) { start = clock64(); } + for (SIZE B_idx = LaneId; B_idx < B; B_idx += MGARDX_WARP_SIZE) { + T_trans buffer = 0; + for (SIZE b_idx = 0; b_idx < b; b_idx++) { + T_trans bit = + (v[b_idx * inc_v] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + // if (blockIdx.x == 0 )printf("bit: %u\n", bit); + if (ALIGN == ALIGN_LEFT) { + buffer += bit << (sizeof(T_trans) * 8 - 1 - b_idx); + } else if (ALIGN == ALIGN_RIGHT) { + buffer += bit << (b - 1 - b_idx); + } else { + } + } + tv[B_idx * inc_tv] = buffer; + } + // if (IdX == 0 && IdY == 0) { start = clock64() - start; + // printf("Parallel_B_Serial_b time : %llu\n", start); } + } + + MGARDX_EXEC static void Serial_B_Atomic_b(T_org *v, SIZE inc_v, T_trans *tv, + SIZE inc_tv, SIZE LaneId) { + // long long start; + // if (IdX == 0 && IdY == 0) { start = clock64(); } + for (SIZE B_idx = LaneId; B_idx < B; B_idx += MGARDX_WARP_SIZE) { + tv[B_idx * inc_tv] = 0; + } + for (SIZE B_idx = 0; B_idx < B; B_idx++) { + for (SIZE b_idx = LaneId; b_idx < b; b_idx += MGARDX_WARP_SIZE) { + T_trans bit = + (v[b_idx * inc_v] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + T_trans shifted_bit = 0; + if (ALIGN == ALIGN_LEFT) { + shifted_bit = bit << (sizeof(T_trans) * 8 - 1 - b_idx); + } else if (ALIGN == ALIGN_RIGHT) { + shifted_bit = bit << (b - 1 - b_idx); + } else { + } + T_trans *sum = &(tv[B_idx * inc_tv]); + Atomic::Add( + sum, shifted_bit); + } + } + // if (IdX == 0 && IdY == 0) { start = clock64() - start; + // printf("Serial_B_Atomic_b time : %llu\n", start); } + } + + MGARDX_EXEC static void Serial_B_Reduce_b(T_org *v, SIZE inc_v, T_trans *tv, + SIZE inc_tv, SIZE LaneId) { + // long long start; + // if (IdX == 0 && IdY == 0) { start = clock64(); } + __shared__ WarpReduceStorageType warp_storage; + + T_trans bit = 0; + T_trans shifted_bit = 0; + T_trans sum = 0; + for (SIZE B_idx = 0; B_idx < B; B_idx++) { + sum = 0; + for (SIZE b_idx = LaneId; + b_idx < ((b - 1) / MGARDX_WARP_SIZE + 1) * MGARDX_WARP_SIZE; + b_idx += MGARDX_WARP_SIZE) { + if (b_idx < b) { + bit = (v[b_idx * inc_v] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + } + shifted_bit = 0; + if (ALIGN == ALIGN_LEFT) { + shifted_bit = bit << (sizeof(T_trans) * 8 - 1 - b_idx); + } else if (ALIGN == ALIGN_RIGHT) { + shifted_bit = bit << (b - 1 - b_idx); + } else { + } + sum += WarpReduceType(warp_storage).Sum(shifted_bit); + } + if (LaneId == 0) { + tv[B_idx * inc_tv] = sum; + } + } + // if (IdX == 0 && IdY == 0) { start = clock64() - start; + // printf("Serial_B_Reduce_b time : %llu\n", start); } + } + + MGARDX_EXEC static void Serial_B_Ballot_b(T_org *v, SIZE inc_v, T_trans *tv, + SIZE inc_tv, SIZE LaneId) { + // long long start; + // if (IdX == 0 && IdY == 0) { start = clock64(); } + T_trans bit = 0; + T_trans sum = 0; + for (SIZE B_idx = 0; B_idx < B; B_idx++) { + sum = 0; + SIZE shift = 0; + for (SIZE b_idx = LaneId; + b_idx < ((b - 1) / MGARDX_WARP_SIZE + 1) * MGARDX_WARP_SIZE; + b_idx += MGARDX_WARP_SIZE) { + bit = 0; + if (b_idx < b) { + if (ALIGN == ALIGN_LEFT) { + bit = (v[(sizeof(T_trans) * 8 - 1 - b_idx) * inc_v] >> + (sizeof(T_org) * 8 - 1 - B_idx)) & + 1u; + } else if (ALIGN == ALIGN_RIGHT) { + bit = (v[(b - 1 - b_idx) * inc_v] >> + (sizeof(T_org) * 8 - 1 - B_idx)) & + 1u; + } else { + } + } + sum += ((T_trans)__ballot(bit)) << shift; + shift += MGARDX_WARP_SIZE; + } + if (LaneId == 0) { + tv[B_idx * inc_tv] = sum; + } + } + // if (IdX == 0 && IdY == 0) { start = clock64() - start; + // printf("Serial_B_Ballot_b time : %llu\n", start); } + } + + MGARDX_EXEC static void Transpose(T_org *v, SIZE inc_v, T_trans *tv, + SIZE inc_tv, SIZE LaneId) { + if (METHOD == Warp_Bit_Transpose_Serial_All) { + Serial_All(v, inc_v, tv, inc_tv, LaneId); + } else if (METHOD == Warp_Bit_Transpose_Parallel_B_Serial_b) { + Parallel_B_Serial_b(v, inc_v, tv, inc_tv, LaneId); + } else if (METHOD == Warp_Bit_Transpose_Serial_B_Atomic_b) { + Serial_B_Atomic_b(v, inc_v, tv, inc_tv, LaneId); + } else if (METHOD == Warp_Bit_Transpose_Serial_B_Reduce_b) { + Serial_B_Reduce_b(v, inc_v, tv, inc_tv, LaneId); + } else if (METHOD == Warp_Bit_Transpose_Serial_B_Ballot_b) { + Serial_B_Ballot_b(v, inc_v, tv, inc_tv, LaneId); + } + } +}; + +template +struct WarpErrorCollect { + + typedef hipcub::WarpReduce WarpReduceType; + using WarpReduceStorageType = typename WarpReduceType::TempStorage; + + MGARDX_EXEC static void Serial_All(T *v, T_error *errors, SIZE LaneId) { + if (LaneId == 0) { + for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx++) { + T data = v[elem_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + + // printf("fp: %u error: \n", fp_data); + for (SIZE bitplane_idx = 0; bitplane_idx < num_bitplanes; + bitplane_idx++) { + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + } + errors[num_bitplanes - bitplane_idx] += diff * diff; + // printf("%f ", diff * diff); + } + errors[0] += data * data; + // printf("%f \n", data * data); + } + } + } + + MGARDX_EXEC static void Parallel_Bitplanes_Serial_Error(T *v, T_error *errors, + SIZE LaneId) { + + __shared__ WarpReduceStorageType warp_storage; + + for (SIZE bitplane_idx = LaneId; bitplane_idx < num_bitplanes; + bitplane_idx += MGARDX_WARP_SIZE) { + for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx++) { + T data = v[elem_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + mantissa; + } + errors[num_bitplanes - bitplane_idx] += diff * diff; + } + } + + T data = 0; + for (SIZE elem_idx = LaneId; + elem_idx < ((num_elems - 1) / MGARDX_WARP_SIZE + 1) * MGARDX_WARP_SIZE; + elem_idx += MGARDX_WARP_SIZE) { + if (elem_idx < num_elems) { + data = v[elem_idx]; + } + T_error error_sum = WarpReduceType(warp_storage).Sum(data * data); + if (LaneId == 0) + errors[0] += error_sum; + } + } + + MGARDX_EXEC static void Serial_Bitplanes_Atomic_Error(T *v, T_error *errors, + SIZE LaneId) { + T data = 0; + for (SIZE elem_idx = LaneId; + elem_idx < ((num_elems - 1) / MGARDX_WARP_SIZE + 1) * MGARDX_WARP_SIZE; + elem_idx += MGARDX_WARP_SIZE) { + if (elem_idx < num_elems) { + data = v[elem_idx]; + } else { + data = 0; + } + + T_fp fp_data = (T_fp)fabs(data); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + + // printf("fp: %u error: \n", fp_data); + for (SIZE bitplane_idx = 0; bitplane_idx < num_bitplanes; + bitplane_idx++) { + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + mantissa; + } + T_error *sum = &(errors[num_bitplanes - bitplane_idx]); + Atomic::Add( + sum, diff * diff); + } + T_error *sum = &(errors[0]); + Atomic::Add( + sum, data * data); + } + } + + MGARDX_EXEC static void Serial_Bitplanes_Reduce_Error(T *v, T_error *errors, + SIZE LaneId) { + + __shared__ WarpReduceStorageType warp_storage; + + T data = 0; + for (SIZE elem_idx = LaneId; + elem_idx < ((num_elems - 1) / MGARDX_WARP_SIZE + 1) * MGARDX_WARP_SIZE; + elem_idx += MGARDX_WARP_SIZE) { + if (elem_idx < num_elems) { + data = v[elem_idx]; + } else { + data = 0; + } + + T_fp fp_data = (T_fp)fabs(data); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + + // printf("fp: %u error: \n", fp_data); + for (SIZE bitplane_idx = 0; bitplane_idx < num_bitplanes; + bitplane_idx++) { + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = + (T_error)Math::negabinary2binary(ngb_data & mask) + mantissa; + } + T_error error_sum = WarpReduceType(warp_storage).Sum(diff * diff); + if (LaneId == 0) + errors[num_bitplanes - bitplane_idx] += error_sum; + } + T_error error_sum = WarpReduceType(warp_storage).Sum(data * data); + if (LaneId == 0) + errors[0] += error_sum; + } + } + + MGARDX_EXEC static void Collect(T *v, T_error *errors, SIZE LaneId) { + if (METHOD == Warp_Error_Collecting_Serial_All) { + Serial_All(v, errors, LaneId); + } + if (METHOD == Warp_Error_Collecting_Parallel_Bitplanes_Serial_Error) { + Parallel_Bitplanes_Serial_Error(v, errors, LaneId); + } + if (METHOD == Warp_Error_Collecting_Serial_Bitplanes_Atomic_Error) { + Serial_Bitplanes_Atomic_Error(v, errors, LaneId); + } + if (METHOD == Warp_Error_Collecting_Serial_Bitplanes_Reduce_Error) { + Serial_Bitplanes_Reduce_Error(v, errors, LaneId); + } + } +}; + +template void HipHuffmanCLCustomizedNoCGKernel(Task task) { + // std::cout << "calling HipHuffmanCLCustomizedNoCGKernel\n"; + dim3 threadsPerBlock(task.GetBlockDimX(), task.GetBlockDimY(), + task.GetBlockDimZ()); + dim3 blockPerGrid(task.GetGridDimX(), task.GetGridDimY(), task.GetGridDimZ()); + size_t sm_size = task.GetSharedMemorySize(); + hipStream_t stream = DeviceRuntime::GetQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation1_Kernel\n"; + Single_Operation1_Kernel<<>>( + task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling LoopCondition1\n"; + while (task.GetFunctor().LoopCondition1()) { + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation2_Kernel\n"; + Single_Operation2_Kernel<<>>(task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation3_Kernel\n"; + Single_Operation3_Kernel<<>>(task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation4_Kernel\n"; + Single_Operation4_Kernel<<>>(task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation5_Kernel\n"; + Single_Operation5_Kernel<<>>(task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling BranchCondition1\n"; + if (task.GetFunctor().BranchCondition1()) { + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling HipParallelMergeKernel\n"; + HipParallelMergeKernel<<>>(task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation10_Kernel\n"; + Single_Operation11_Kernel<<>>(task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + } + + // std::cout << "calling Single_Operation11_Kernel\n"; + Single_Operation12_Kernel<<>>(task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation12_Kernel\n"; + Single_Operation13_Kernel<<>>(task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation13_Kernel\n"; + Single_Operation14_Kernel<<>>(task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation14_Kernel\n"; + Single_Operation15_Kernel<<>>(task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + } +} + +template void HipHuffmanCWCustomizedNoCGKernel(Task task) { + // std::cout << "calling HipHuffmanCWCustomizedNoCGKernel\n"; + dim3 threadsPerBlock(task.GetBlockDimX(), task.GetBlockDimY(), + task.GetBlockDimZ()); + dim3 blockPerGrid(task.GetGridDimX(), task.GetGridDimY(), task.GetGridDimZ()); + size_t sm_size = task.GetSharedMemorySize(); + hipStream_t stream = DeviceRuntime::GetQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation1_Kernel\n"; + Single_Operation1_Kernel<<>>( + task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + // std::cout << "calling Single_Operation2_Kernel\n"; + Single_Operation2_Kernel<<>>( + task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + // std::cout << "calling Single_Operation3_Kernel\n"; + Single_Operation3_Kernel<<>>( + task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling LoopCondition1\n"; + while (task.GetFunctor().LoopCondition1()) { + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation4_Kernel\n"; + Single_Operation4_Kernel<<>>(task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation5_Kernel\n"; + Single_Operation5_Kernel<<>>(task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation6_Kernel\n"; + Single_Operation6_Kernel<<>>(task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation7_Kernel\n"; + Single_Operation7_Kernel<<>>(task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation8_Kernel\n"; + Single_Operation8_Kernel<<>>(task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + } + + // std::cout << "calling Single_Operation9_Kernel\n"; + Single_Operation9_Kernel<<>>( + task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation10_Kernel\n"; + Single_Operation10_Kernel<<>>( + task); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); +} + +// template class DeviceAdapter { +// public: +// MGARDX_CONT +// DeviceAdapter(){}; + +// MGARDX_CONT +// int IsResourceEnough(TaskType &task) { +// if (task.GetBlockDimX() * task.GetBlockDimY() * task.GetBlockDimZ() > +// DeviceRuntime::GetMaxNumThreadsPerTB()) { +// return THREADBLOCK_TOO_LARGE; +// } +// if (task.GetSharedMemorySize() > +// DeviceRuntime::GetMaxSharedMemorySize()) { +// return SHARED_MEMORY_TOO_LARGE; +// } +// return RESOURCE_ENOUGH; +// } + +// MGARDX_CONT +// ExecutionReturn Execute(TaskType &task) { +// dim3 threadsPerBlock(task.GetBlockDimX(), task.GetBlockDimY(), +// task.GetBlockDimZ()); +// dim3 blockPerGrid(task.GetGridDimX(), task.GetGridDimY(), +// task.GetGridDimZ()); +// size_t sm_size = task.GetSharedMemorySize(); +// // printf("exec config (%d %d %d) (%d %d %d) sm_size: %llu\n", +// // threadsPerBlock.x, threadsPerBlock.y, threadsPerBlock.z, +// // blockPerGrid.x, blockPerGrid.y, blockPerGrid.z, +// sm_size); hipStream_t stream = +// DeviceRuntime::GetQueue(task.GetQueueIdx()); + +// if (DeviceRuntime::PrintKernelConfig) { +// std::cout << log::log_info << task.GetFunctorName() << ": <" +// << task.GetBlockDimX() << ", " << task.GetBlockDimY() << ", " +// << task.GetBlockDimZ() << "> <" << task.GetGridDimX() << ", " +// << task.GetGridDimY() << ", " << task.GetGridDimZ() << ">\n"; +// } + +// ExecutionReturn ret; +// if (IsResourceEnough(task) != RESOURCE_ENOUGH) { +// if (DeviceRuntime::PrintKernelConfig) { +// if (IsResourceEnough(task) == THREADBLOCK_TOO_LARGE) { +// log::info("threadblock too large."); +// } +// if (IsResourceEnough(task) == SHARED_MEMORY_TOO_LARGE) { +// log::info("shared memory too large."); +// } +// } +// ret.success = false; +// ret.execution_time = std::numeric_limits::max(); +// return ret; +// } + +// Timer timer; +// if (task.GetQueueIdx() == MGARDX_SYNCHRONIZED_QUEUE || +// DeviceRuntime::TimingAllKernels || +// AutoTuner::ProfileKernels) { +// DeviceRuntime::SyncDevice(); +// timer.start(); +// } + +// // if constexpr evaluate at compile time otherwise this does not compile +// if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) { +// HipKernel<<>>(task); +// } else if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) +// { +// HipIterKernel<<>>(task); +// } else if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) +// { +// if (task.GetFunctor().use_CG && DeviceRuntime::SupportCG()) { +// void *Args[] = {(void *)&task}; +// hipLaunchCooperativeKernel( +// (void *)HipHuffmanCLCustomizedKernel, blockPerGrid, +// threadsPerBlock, Args, sm_size, stream); +// } else { +// HipHuffmanCLCustomizedNoCGKernel(task); +// } +// } else if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) +// { +// if (task.GetFunctor().use_CG && DeviceRuntime::SupportCG()) { +// void *Args[] = {(void *)&task}; +// hipLaunchCooperativeKernel( +// (void *)HipHuffmanCWCustomizedKernel, blockPerGrid, +// threadsPerBlock, Args, sm_size, stream); +// } else { +// HipHuffmanCWCustomizedNoCGKernel(task); +// } +// } +// ErrorAsyncCheckTask(hipGetLastError(), task); +// gpuErrchk(hipGetLastError()); +// if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { +// ErrorSyncCheckTask(hipDeviceSynchronize(), task); +// } + +// if (task.GetQueueIdx() == MGARDX_SYNCHRONIZED_QUEUE || +// DeviceRuntime::TimingAllKernels || +// AutoTuner::ProfileKernels) { +// DeviceRuntime::SyncDevice(); +// timer.end(); +// if (DeviceRuntime::TimingAllKernels) { +// timer.print(task.GetFunctorName()); +// } +// if (AutoTuner::ProfileKernels) { +// ret.success = true; +// ret.execution_time = timer.get(); +// } +// } +// return ret; +// } +// }; + +template <> class DeviceLauncher { +public: + template + MGARDX_CONT int static IsResourceEnough(TaskType &task) { + if (task.GetBlockDimX() * task.GetBlockDimY() * task.GetBlockDimZ() > + DeviceRuntime::GetMaxNumThreadsPerTB()) { + return THREADBLOCK_TOO_LARGE; + } + if (task.GetSharedMemorySize() > + DeviceRuntime::GetMaxSharedMemorySize()) { + return SHARED_MEMORY_TOO_LARGE; + } + return RESOURCE_ENOUGH; + } + + template + MGARDX_CONT ExecutionReturn static Execute(TaskType &task) { + + dim3 threadsPerBlock(task.GetBlockDimX(), task.GetBlockDimY(), + task.GetBlockDimZ()); + dim3 blockPerGrid(task.GetGridDimX(), task.GetGridDimY(), + task.GetGridDimZ()); + size_t sm_size = task.GetSharedMemorySize(); + // printf("exec config (%d %d %d) (%d %d %d) sm_size: %llu\n", + // threadsPerBlock.x, threadsPerBlock.y, threadsPerBlock.z, + // blockPerGrid.x, blockPerGrid.y, blockPerGrid.z, sm_size); + hipStream_t stream = DeviceRuntime::GetQueue(task.GetQueueIdx()); + + if (DeviceRuntime::PrintKernelConfig) { + std::cout << log::log_info << task.GetFunctorName() << ": <" + << task.GetBlockDimX() << ", " << task.GetBlockDimY() << ", " + << task.GetBlockDimZ() << "> <" << task.GetGridDimX() << ", " + << task.GetGridDimY() << ", " << task.GetGridDimZ() << ">\n"; + } + + ExecutionReturn ret; + if (IsResourceEnough(task) != RESOURCE_ENOUGH) { + if (DeviceRuntime::PrintKernelConfig) { + if (IsResourceEnough(task) == THREADBLOCK_TOO_LARGE) { + log::info("threadblock too large."); + } + if (IsResourceEnough(task) == SHARED_MEMORY_TOO_LARGE) { + log::info("shared memory too large."); + } + } + ret.success = false; + ret.execution_time = std::numeric_limits::max(); + return ret; + } + + Timer timer; + if (task.GetQueueIdx() == MGARDX_SYNCHRONIZED_QUEUE || + DeviceRuntime::TimingAllKernels || + AutoTuner::ProfileKernels) { + DeviceRuntime::SyncDevice(); + timer.start(); + } + + // if constexpr evaluate at compile time otherwise this does not compile + if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + HipKernel<<>>(task); + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + HipIterKernel<<>>(task); + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + if (task.GetFunctor().use_CG && DeviceRuntime::SupportCG()) { + void *Args[] = {(void *)&task}; + hipLaunchCooperativeKernel( + (void *)HipHuffmanCLCustomizedKernel, blockPerGrid, + threadsPerBlock, Args, sm_size, stream); + } else { + HipHuffmanCLCustomizedNoCGKernel(task); + } + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + if (task.GetFunctor().use_CG && DeviceRuntime::SupportCG()) { + void *Args[] = {(void *)&task}; + hipLaunchCooperativeKernel( + (void *)HipHuffmanCWCustomizedKernel, blockPerGrid, + threadsPerBlock, Args, sm_size, stream); + } else { + HipHuffmanCWCustomizedNoCGKernel(task); + } + } + ErrorAsyncCheckTask(hipGetLastError(), task); + gpuErrchk(hipGetLastError()); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheckTask(hipDeviceSynchronize(), task); + } + + if (task.GetQueueIdx() == MGARDX_SYNCHRONIZED_QUEUE || + DeviceRuntime::TimingAllKernels || + AutoTuner::ProfileKernels) { + DeviceRuntime::SyncDevice(); + timer.end(); + if (DeviceRuntime::TimingAllKernels) { + timer.print(task.GetFunctorName()); + } + if (AutoTuner::ProfileKernels) { + ret.success = true; + ret.execution_time = timer.get(); + } + } + return ret; + } + + template + MGARDX_CONT static void ConfigTask(TaskType task) { + typename TaskType::Functor functor; + int maxbytes = DeviceRuntime::GetMaxSharedMemorySize(); + DeviceRuntime::SetMaxDynamicSharedMemorySize(functor, maxbytes); + } + + template + MGARDX_CONT static void AutoTune(KernelType kernel, int queue_idx) { +#if MGARD_ENABLE_AUTO_TUNING + double min_time = std::numeric_limits::max(); + int min_config = 0; + ExecutionReturn ret; +#define RUN_CONFIG(CONFIG_IDX) \ + { \ + constexpr ExecutionConfig config = \ + GetExecutionConfig(CONFIG_IDX); \ + auto task = \ + kernel.template GenTask(queue_idx); \ + if constexpr (KernelType::EnableConfig()) { \ + ConfigTask(task); \ + } \ + ret = Execute(task); \ + if (ret.success && min_time > ret.execution_time) { \ + min_time = ret.execution_time; \ + min_config = CONFIG_IDX; \ + } \ + } + RUN_CONFIG(0) + RUN_CONFIG(1) + RUN_CONFIG(2) + RUN_CONFIG(3) + RUN_CONFIG(4) + RUN_CONFIG(5) + RUN_CONFIG(6) +#undef RUN_CONFIG + if (AutoTuner::WriteToTable) { + FillAutoTunerTable(std::string(KernelType::Name), min_config); + } +#else + throw std::runtime_error("MGARD is not built with auto tuning enabled."); +#endif + } + + template + MGARDX_CONT static void Execute(KernelType kernel, int queue_idx) { + if constexpr (KernelType::EnableAutoTuning()) { + constexpr ExecutionConfig config = + GetExecutionConfig(KernelType::Name); + auto task = + kernel.template GenTask(queue_idx); + if constexpr (KernelType::EnableConfig()) { + ConfigTask(task); + } + Execute(task); + + if (AutoTuner::ProfileKernels) { + AutoTune(kernel, queue_idx); + } + } else { + auto task = kernel.GenTask(queue_idx); + if constexpr (KernelType::EnableConfig()) { + ConfigTask(task); + } + Execute(task); + } + } +}; + +struct AbsMaxOp { + template + __device__ __forceinline__ T operator()(const T &a, const T &b) const { + return (fabs(b) > fabs(a)) ? fabs(b) : fabs(a); + } +}; + +struct AbsMinOp { + template + __device__ __forceinline__ T operator()(const T &a, const T &b) const { + return (fabs(b) > fabs(a)) ? fabs(a) : fabs(b); + } +}; + +struct SquareOp { + template + __device__ __forceinline__ T operator()(const T &a) const { + return a * a; + } +}; + +template <> class DeviceCollective { +public: + MGARDX_CONT + DeviceCollective(){}; + + template + MGARDX_CONT static void + Sum(SIZE n, SubArray<1, T, HIP> v, SubArray<1, T, HIP> result, + Array<1, Byte, HIP> &workspace, bool workspace_allocated, int queue_idx) { + + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + hipStream_t stream = DeviceRuntime::GetQueue(queue_idx); + hipcub::DeviceReduce::Sum(d_temp_storage, temp_storage_bytes, v.data(), + result.data(), n, stream); + ErrorAsyncCheck(hipGetLastError(), "DeviceCollective::Sum"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(hipDeviceSynchronize(), "DeviceCollective::Sum"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + + template + MGARDX_CONT static void AbsMax(SIZE n, SubArray<1, T, HIP> v, + SubArray<1, T, HIP> result, + Array<1, Byte, HIP> &workspace, + bool workspace_allocated, int queue_idx) { + + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + AbsMaxOp absMaxOp; + hipStream_t stream = DeviceRuntime::GetQueue(queue_idx); + hipcub::DeviceReduce::Reduce(d_temp_storage, temp_storage_bytes, v.data(), + result.data(), n, absMaxOp, static_cast(0), + stream); + ErrorAsyncCheck(hipGetLastError(), "DeviceCollective::AbsMax"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(hipDeviceSynchronize(), "DeviceCollective::AbsMax"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + + template + MGARDX_CONT static void AbsMin(SIZE n, SubArray<1, T, HIP> v, + SubArray<1, T, HIP> result, + Array<1, Byte, HIP> &workspace, + bool workspace_allocated, int queue_idx) { + + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + AbsMinOp absMinOp; + hipStream_t stream = DeviceRuntime::GetQueue(queue_idx); + hipcub::DeviceReduce::Reduce(d_temp_storage, temp_storage_bytes, v.data(), + result.data(), n, absMinOp, + std::numeric_limits::max(), stream); + ErrorAsyncCheck(hipGetLastError(), "DeviceCollective::AbsMax"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(hipDeviceSynchronize(), "DeviceCollective::AbsMax"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + + template + MGARDX_CONT static void SquareSum(SIZE n, SubArray<1, T, HIP> v, + SubArray<1, T, HIP> result, + Array<1, Byte, HIP> &workspace, + bool workspace_allocated, int queue_idx) { + + SquareOp squareOp; + hipcub::TransformInputIterator transformed_input_iter( + v.data(), squareOp); + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + hipStream_t stream = DeviceRuntime::GetQueue(queue_idx); + hipcub::DeviceReduce::Sum(d_temp_storage, temp_storage_bytes, + transformed_input_iter, result.data(), n, stream); + ErrorAsyncCheck(hipGetLastError(), "DeviceCollective::SquareSum"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(hipDeviceSynchronize(), + "DeviceCollective::SquareSum"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + + template + MGARDX_CONT static void + ScanSumInclusive(SIZE n, SubArray<1, T, HIP> v, SubArray<1, T, HIP> result, + Array<1, Byte, HIP> &workspace, bool workspace_allocated, + int queue_idx) { + + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + hipStream_t stream = DeviceRuntime::GetQueue(queue_idx); + hipcub::DeviceScan::InclusiveSum(d_temp_storage, temp_storage_bytes, + v.data(), result.data(), n, stream); + ErrorAsyncCheck(hipGetLastError(), + "DeviceCollective::ScanSumInclusive"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(hipDeviceSynchronize(), + "DeviceCollective::ScanSumInclusive"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + + template + MGARDX_CONT static void + ScanSumExclusive(SIZE n, SubArray<1, T, HIP> v, SubArray<1, T, HIP> result, + Array<1, Byte, HIP> &workspace, bool workspace_allocated, + int queue_idx) { + + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + hipStream_t stream = DeviceRuntime::GetQueue(queue_idx); + hipcub::DeviceScan::ExclusiveSum(d_temp_storage, temp_storage_bytes, + v.data(), result.data(), n, stream); + ErrorAsyncCheck(hipGetLastError(), + "DeviceCollective::ScanSumExclusive"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(hipDeviceSynchronize(), + "DeviceCollective::ScanSumExclusive"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + + template + MGARDX_CONT static void + ScanSumExtended(SIZE n, SubArray<1, T, HIP> v, SubArray<1, T, HIP> result, + Array<1, Byte, HIP> &workspace, bool workspace_allocated, + int queue_idx) { + + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + hipStream_t stream = DeviceRuntime::GetQueue(queue_idx); + hipcub::DeviceScan::InclusiveSum(d_temp_storage, temp_storage_bytes, + v.data(), result.data() + 1, n, stream); + ErrorAsyncCheck(hipGetLastError(), + "DeviceCollective::ScanSumExtended"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(hipDeviceSynchronize(), + "DeviceCollective::ScanSumExtended"); + } + if (workspace_allocated) { + T zero = 0; + MemoryManager().Copy1D(result.data(), &zero, 1, queue_idx); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } + + template + MGARDX_CONT static void + SortByKey(SIZE n, SubArray<1, KeyT, HIP> in_keys, + SubArray<1, ValueT, HIP> in_values, SubArray<1, KeyT, HIP> out_keys, + SubArray<1, ValueT, HIP> out_values, Array<1, Byte, HIP> &workspace, + bool workspace_allocated, int queue_idx) { + + Byte *d_temp_storage = workspace_allocated ? workspace.data() : nullptr; + size_t temp_storage_bytes = workspace_allocated ? workspace.shape(0) : 0; + hipStream_t stream = DeviceRuntime::GetQueue(queue_idx); + hipcub::DeviceRadixSort::SortPairs( + d_temp_storage, temp_storage_bytes, in_keys.data(), out_keys.data(), + in_values.data(), out_values.data(), n, 0, sizeof(KeyT) * 8, stream); + ErrorAsyncCheck(hipGetLastError(), "DeviceCollective::SortByKey"); + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + ErrorSyncCheck(hipDeviceSynchronize(), + "DeviceCollective::SortByKey"); + } + if (!workspace_allocated) { + workspace.resize({(SIZE)temp_storage_bytes}, queue_idx); + } + } +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h b/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h new file mode 100644 index 000000000..5e0d4f9bc --- /dev/null +++ b/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h @@ -0,0 +1,382 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_DEVICE_ADAPTER_KOKKOS_H +#define MGARD_X_160_DEVICE_ADAPTER_KOKKOS_H + +#include "../Utilities/Exceptions.h" +#include "DeviceAdapter.h" +#include "Kokkos_Core.hpp" + +namespace mgard_x_160 { + +#ifdef KOKKOS_ENABLE_CUDA +#define gpuErrchk(ans) \ + { gpuAssert((ans), __FILE__, __LINE__); } + +inline void gpuAssert(cudaError_t code, const char *file, int line, + bool abort = true) { + if (code != cudaSuccess) { + if (abort) + throw ProcessingException(std::string("GPUassert: ") + + cudaGetErrorString(code) + " " + file + " " + + std::to_string(line)); + else + fprintf(stderr, "GPUassert: %s %s %d\n", cudaGetErrorString(code), file, + line); + } +} +#endif + +#ifdef KOKKOS_ENABLE_HIP + +#define gpuErrchk(ans) \ + { gpuAssert((ans), __FILE__, __LINE__); } + +inline void gpuAssert(hipError_t code, const char *file, int line, + bool abort = true) { + if (code != hipSuccess) { + if (abort) + throw ProcessingException(std::string("GPUassert: ") + + hipGetErrorString(code) + " " + file + " " + + std::to_string(line)); + else + fprintf(stderr, "GPUassert: %s %s %d\n", hipGetErrorString(code), file, + line); + } +} + +#endif + +template MGARDX_KERL void KOKKOSKernel(TaskType task) { + typedef Kokkos::TeamPolicy::member_type member_type; + typedef Kokkos::KOKKOS::scratch_memory_space ScratchSpace; + typedef Kokkos::View> + ShareMemoryView; + SIZE league_size = + task.GetGridDimZ() * task.GetGridDimY() * task.GetGridDimX(); + SIZE team_size = + task.GetBlockDimZ() * task.GetBlockDimY() * task.GetBlockDimX(); + Kokkos::TeamPolicy policy(league_size, team_size); + + Kokkos::parallel_for( + task.GetFunctorName(), + policy.set_scratch_size(0, task.GetSharedMemorySize()), + KOKKOS_LAMBDA(member_type team_member) { + IDX threadx = team_member.team_rank() % task.GetBlockDimX(); + IDX thready = team_member.team_rank() / task.GetBlockDimX(); + IDX threadz = team_member.team_rank() / + (task.GetBlockDimX() * task.GetBlockDimY()); + IDX blockx = team_member.league_rank() % task.GetGridDimX(); + IDX blocky = team_member.league_rank() / task.GetGridDimX(); + IDX blockz = team_member.league_rank() / + (task.GetGridDimX() * task.GetGridDimY()); + ShareMemoryView shared_memory(team_member.team_scratch(0), + team_member.team_size()); + task.GetFunctor().Init(task.GetGridDimZ(), task.GetGridDimY(), + task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX(), blockz, + blocky, blockx, threadz, thready, threadx, + shared_memory.data()); + task.Operations1(); + team_member.team_barrier(); + task.Operations2(); + team_member.team_barrier(); + task.Operations3(); + team_member.team_barrier(); + task.Operations4(); + team_member.team_barrier(); + task.Operations5(); + team_member.team_barrier(); + task.Operations6(); + team_member.team_barrier(); + task.Operations7(); + team_member.team_barrier(); + task.Operations8(); + team_member.team_barrier(); + task.Operations9(); + team_member.team_barrier(); + task.Operations10(); + team_member.team_barrier(); + }); +} + +template <> class DeviceSpecification { +public: + MGARDX_CONT + DeviceSpecification() { + + Kokkos::InitArguments args; + args.device_id = 0; + Kokkos::initialize(args); + + NumDevices = 1; + MaxSharedMemorySize = new int[NumDevices]; + WarpSize = new int[NumDevices]; + NumSMs = new int[NumDevices]; + ArchitectureGeneration = new int[NumDevices]; + MaxNumThreadsPerSM = new int[NumDevices]; + + for (int d = 0; d < NumDevices; d++) { + MaxSharedMemorySize[d] = 1e6; + WarpSize[d] = 32; + NumSMs[d] = 80; + MaxNumThreadsPerSM[d] = 1024; + ArchitectureGeneration[d] = 1; + } + } + + MGARDX_CONT int GetNumDevices() { return NumDevices; } + + MGARDX_CONT int GetMaxSharedMemorySize(int dev_id) { + return MaxSharedMemorySize[dev_id]; + } + + MGARDX_CONT int GetWarpSize(int dev_id) { return WarpSize[dev_id]; } + + MGARDX_CONT int GetNumSMs(int dev_id) { return NumSMs[dev_id]; } + + MGARDX_CONT int GetArchitectureGeneration(int dev_id) { + return ArchitectureGeneration[dev_id]; + } + + MGARDX_CONT int GetMaxNumThreadsPerSM(int dev_id) { + return MaxNumThreadsPerSM[dev_id]; + } + + MGARDX_CONT + ~DeviceSpecification() { + delete[] MaxSharedMemorySize; + delete[] WarpSize; + delete[] NumSMs; + delete[] ArchitectureGeneration; + } + + int NumDevices; + int *MaxSharedMemorySize; + int *WarpSize; + int *NumSMs; + int *ArchitectureGeneration; + int *MaxNumThreadsPerSM; +}; + +template <> class DeviceRuntime { +public: + MGARDX_CONT + DeviceRuntime() {} + + MGARDX_CONT static void SelectDevice(SIZE dev_id) { + // do not support for now + curr_dev_id = dev_id; + } + + MGARDX_CONT static int GetQueue(SIZE queue_id) { + // do not support for now + return 0; + } + + MGARDX_CONT static void SyncQueue(SIZE queue_id) { + // do not support for now + // queues.SyncQueue(curr_dev_id, queue_id); + } + + MGARDX_CONT static void SyncAllQueues() { + // do not support for now + // queues.SyncAllQueues(curr_dev_id); + } + + MGARDX_CONT static void SyncDevice() { KOKKOS::impl_static_fence(); } + + MGARDX_CONT static int GetMaxSharedMemorySize() { + return DeviceSpecs.GetMaxSharedMemorySize(curr_dev_id); + } + + MGARDX_CONT static int GetWarpSize() { + return DeviceSpecs.GetWarpSize(curr_dev_id); + } + + MGARDX_CONT static int GetNumSMs() { + return DeviceSpecs.GetNumSMs(curr_dev_id); + } + + MGARDX_CONT static int GetArchitectureGeneration() { + return DeviceSpecs.GetArchitectureGeneration(curr_dev_id); + } + + MGARDX_CONT static int GetMaxNumThreadsPerSM() { + return DeviceSpecs.GetMaxNumThreadsPerSM(curr_dev_id); + } + + template + MGARDX_CONT static int + GetOccupancyMaxActiveBlocksPerSM(FunctorType functor, int blockSize, + size_t dynamicSMemSize) { + int numBlocks = 0; + // Task task = Task(functor, 1, 1, 1, 1, 1, + // blockSize, dynamicSMemSize, 0); if constexpr (std::is_same::value) { + // if constexpr (std::is_base_of, FunctorType>::value) { + // gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, Kernel>, blockSize, + // dynamicSMemSize)); + // } else if constexpr (std::is_base_of, + // FunctorType>::value) { + // gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, IterKernel>, blockSize, + // dynamicSMemSize)); + // } else if constexpr + // (std::is_base_of, + // FunctorType>::value) { + // gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, HuffmanCLCustomizedKernel>, + // blockSize, dynamicSMemSize)); + // } else if constexpr + // (std::is_base_of, + // FunctorType>::value) { + // gpuErrchk(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, HuffmanCWCustomizedKernel>, + // blockSize, dynamicSMemSize)); + // } else { + // std::cout << log::log_err << "GetOccupancyMaxActiveBlocksPerSM + // Error!\n"; + // } + // } else if (std::is_same::value) { + // if constexpr (std::is_base_of, FunctorType>::value) { + // gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, Kernel>, blockSize, + // dynamicSMemSize)); + // } else if constexpr (std::is_base_of, + // FunctorType>::value) { + // gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, IterKernel>, blockSize, + // dynamicSMemSize)); + // } else if constexpr (std::is_base_of, + // FunctorType>::value) { + // gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, HuffmanCLCustomizedKernel>, + // blockSize, dynamicSMemSize)); + // } else if constexpr (std::is_base_of, + // FunctorType>::value) { + // gpuErrchk(hipOccupancyMaxActiveBlocksPerMultiprocessor( + // &numBlocks, HuffmanCWCustomizedKernel>, + // blockSize, dynamicSMemSize)); + // } else { + // std::cout << log::log_err << "GetOccupancyMaxActiveBlocksPerSM + // Error!\n"; + // } + // } + + return numBlocks; + } + + template + MGARDX_CONT static void SetMaxDynamicSharedMemorySize(FunctorType functor, + int maxbytes) { + // skip for now + } + + MGARDX_CONT + ~DeviceRuntime() {} + + static int curr_dev_id; + // static DeviceQueues queues; + static bool SyncAllKernelsAndCheckErrors; + static DeviceSpecification DeviceSpecs; +}; + +template <> class MemoryManager { +public: + MGARDX_CONT + MemoryManager(){}; + + template + MGARDX_CONT static void Malloc1D(T *&ptr, SIZE n, int queue_idx) { + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = (T *)Kokkos::kokkos_malloc(n * + sizeof(converted_T)); + if (ptr == NULL) { + std::cout << log::log_err << "MemoryManager::Malloc1D error.\n"; + } + } + + template + MGARDX_CONT static void MallocND(T *&ptr, SIZE n1, SIZE n2, SIZE &ld, + int queue_idx) { + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = (T *)Kokkos::kokkos_malloc(n1 * n2 * + sizeof(converted_T)); + ld = n1; + if (ptr == NULL) { + std::cout << log::log_err << "MemoryManager::MallocND error.\n"; + } + } + + template MGARDX_CONT static void Free(T *ptr) { + if (ptr == NULL) + return; + Kokkos::kokkos_free(ptr); + } + + template + MGARDX_CONT static void Copy1D(T *dst_ptr, const T *src_ptr, SIZE n, + int queue_idx) { + using converted_T = + typename std::conditional::value, Byte, T>::type; + Kokkos::Impl::DeepCopy( + dst_ptr, src_ptr, n * sizeof(converted_T)); + } + + template + MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, + SIZE src_ld, SIZE n1, SIZE n2, int queue_idx) { + using converted_T = + typename std::conditional::value, Byte, T>::type; + Kokkos::Impl::DeepCopy( + dst_ptr, src_ptr, n1 * n2 * sizeof(converted_T)); + } + + template + MGARDX_CONT static void MallocHost(T *&ptr, SIZE n, int queue_idx) { + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = (T *)std::malloc(n * sizeof(converted_T)); + if (ptr == NULL) { + std::cout << log::log_err << "MemoryManager::MallocHost error.\n"; + } + } + + template MGARDX_CONT static void FreeHost(T *ptr) { + if (ptr == NULL) + return; + std::free(ptr); + } + + template + MGARDX_CONT static void Memset1D(T *ptr, SIZE n, int value) { + for (SIZE i = 0; i < n; ++i) + ptr[i] = value; + } + + template + MGARDX_CONT static void MemsetND(T *ptr, SIZE ld, SIZE n1, SIZE n2, + int value) { + for (SIZE i = 0; i < n1 * n2; ++i) + ptr[i] = value; + } + + template MGARDX_CONT static bool IsDevicePointer(T *ptr) { + return true; + } + + static bool ReduceMemoryFootprint; +}; +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h b/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h new file mode 100644 index 000000000..465f6d501 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h @@ -0,0 +1,1575 @@ +#ifndef MGARD_X_160_DEVICE_ADAPTER_OPENMP_H +#define MGARD_X_160_DEVICE_ADAPTER_OPENMP_H + +#include "DeviceAdapter.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace mgard_x_160 { + +template <> struct SyncBlock { + MGARDX_EXEC static void Sync() { + // do nothing + } +}; + +template <> struct SyncGrid { + MGARDX_EXEC static void Sync() { + // do nothing + } +}; + +template +struct Atomic { + MGARDX_EXEC static T Min(T *result, T value) { + T old = 0; +#pragma omp critical + { + old = *result; + *result = std::min(*result, value); + } + return old; + } + MGARDX_EXEC static T Max(T *result, T value) { + T old = 0; +#pragma omp critical + { + old = *result; + *result = std::max(*result, value); + } + return old; + } + MGARDX_EXEC static T Add(T *result, T value) { + T old = 0; +#pragma omp atomic capture + { + old = *result; + *result += value; + } + return old; + } + MGARDX_EXEC static T Or(T *result, T value) { + T old = 0; +#pragma omp atomic capture + { + old = *result; + *result |= value; + } + return old; + } +}; + +template <> struct Math { + template MGARDX_EXEC static T Min(T a, T b) { + return std::min(a, b); + } + template MGARDX_EXEC static T Max(T a, T b) { + return std::max(a, b); + } + + MGARDX_EXEC static int ffs(unsigned int a) { return ffs(a); } + MGARDX_EXEC static int ffsll(long long unsigned int a) { + // return ffsll(a); + int pos = 0; + if (a == 0) + return pos; + pos = MultiplyDeBruijnBitPosition + [((long long unsigned int)((a & -a) * 151050438420815295)) >> 58]; + // while (!(a & 1)) + // { + // a >>= 1; + // ++pos; + // } + // if (pos2 != pos) {printf("mismatch: %d %d", pos2, pos);} + return pos + 1; + } + MGARDX_EXEC + static uint64_t binary2negabinary(const int64_t x) { + return (x + (uint64_t)0xaaaaaaaaaaaaaaaaull) ^ + (uint64_t)0xaaaaaaaaaaaaaaaaull; + } + + MGARDX_EXEC + static uint32_t binary2negabinary(const int32_t x) { + return (x + (uint32_t)0xaaaaaaaau) ^ (uint32_t)0xaaaaaaaau; + } + + MGARDX_EXEC + static int64_t negabinary2binary(const uint64_t x) { + return (x ^ 0xaaaaaaaaaaaaaaaaull) - 0xaaaaaaaaaaaaaaaaull; + } + + MGARDX_EXEC + static int32_t negabinary2binary(const uint32_t x) { + return (x ^ 0xaaaaaaaau) - 0xaaaaaaaau; + } +}; + +template <> class DeviceSpecification { +public: + MGARDX_CONT + DeviceSpecification() { + // Get the number of cores + NumDevices = 1; + MaxSharedMemorySize = new int[NumDevices]; + WarpSize = new int[NumDevices]; + NumSMs = new int[NumDevices]; + ArchitectureGeneration = new int[NumDevices]; + MaxNumThreadsPerSM = new int[NumDevices]; + MaxNumThreadsPerTB = new int[NumDevices]; + AvailableMemory = new size_t[NumDevices]; + SupportCooperativeGroups = new bool[NumDevices]; + DeviceNames = new std::string[NumDevices]; + + for (int d = 0; d < NumDevices; d++) { + MaxSharedMemorySize[d] = 1e6; + WarpSize[d] = MGARDX_WARP_SIZE; + MaxNumThreadsPerSM[d] = 1024; + MaxNumThreadsPerTB[d] = 1024; + ArchitectureGeneration[d] = 1; + SupportCooperativeGroups[d] = true; + AvailableMemory[d] = std::numeric_limits::max(); // unlimited + } + } + + MGARDX_CONT int GetNumDevices() { return NumDevices; } + + MGARDX_CONT int GetMaxSharedMemorySize(int dev_id) { + return MaxSharedMemorySize[dev_id]; + } + + MGARDX_CONT int GetWarpSize(int dev_id) { return WarpSize[dev_id]; } + + MGARDX_CONT int GetNumSMs(int dev_id) { +#pragma omp parallel + { +#pragma omp single + { NumSMs[dev_id] = omp_get_num_threads(); } + } + return NumSMs[dev_id]; + } + + MGARDX_CONT int GetArchitectureGeneration(int dev_id) { + return ArchitectureGeneration[dev_id]; + } + + MGARDX_CONT int GetMaxNumThreadsPerSM(int dev_id) { + return MaxNumThreadsPerSM[dev_id]; + } + + MGARDX_CONT int GetMaxNumThreadsPerTB(int dev_id) { + return MaxNumThreadsPerTB[dev_id]; + } + + MGARDX_CONT int GetAvailableMemory(int dev_id) { + AvailableMemory[dev_id] = 8e9; + return AvailableMemory[dev_id]; + } + + MGARDX_CONT bool SupportCG(int dev_id) { + return SupportCooperativeGroups[dev_id]; + } + + MGARDX_CONT std::string GetDeviceName(int dev_id) { + return "CPU (OpenMP) " + std::to_string(GetNumSMs(dev_id)) + " core(s)"; + } + + MGARDX_CONT + ~DeviceSpecification() { + delete[] MaxSharedMemorySize; + delete[] WarpSize; + delete[] NumSMs; + delete[] ArchitectureGeneration; + delete[] MaxNumThreadsPerSM; + delete[] MaxNumThreadsPerTB; + delete[] AvailableMemory; + delete[] SupportCooperativeGroups; + delete[] DeviceNames; + } + + int NumDevices; + int *MaxSharedMemorySize; + int *WarpSize; + int *NumSMs; + int *ArchitectureGeneration; + int *MaxNumThreadsPerSM; + int *MaxNumThreadsPerTB; + size_t *AvailableMemory; + bool *SupportCooperativeGroups; + std::string *DeviceNames; +}; + +template <> class DeviceQueues { +public: + MGARDX_CONT + void Initialize() { + log::dbg("Calling DeviceQueues::Initialize"); + initialized = true; + } + + MGARDX_CONT + void Destroy() { + log::dbg("Calling DeviceQueues::Destroy"); + initialized = false; + } + + MGARDX_CONT + DeviceQueues() { + // do nothing + } + + MGARDX_CONT int GetQueue(int dev_id, SIZE queue_id) { return 0; } + + MGARDX_CONT void SyncQueue(int dev_id, SIZE queue_id) { + // do nothing + } + + MGARDX_CONT void SyncAllQueues(int dev_id) { + // do nothing + } + + MGARDX_CONT + ~DeviceQueues() { + // do nothing + } + bool initialized = false; +}; + +extern int openmp_dev_id; +#pragma omp threadprivate(openmp_dev_id) + +// OpenMP parallelism is across the grid (work-groups over cores), not within a +// sub-group, so its sub-group is the size-1 scalar group. +template <> struct SubGroup : SubGroupScalar {}; + +template <> class DeviceRuntime { +public: + MGARDX_CONT + DeviceRuntime() {} + + MGARDX_CONT static void Initialize() { queues.Initialize(); } + + MGARDX_CONT static void Finalize() { queues.Destroy(); } + + MGARDX_CONT static int GetDeviceCount() { return DeviceSpecs.NumDevices; } + + MGARDX_CONT static void SelectDevice(SIZE dev_id) { + // do nothing + } + + MGARDX_CONT static int GetDevice() { return openmp_dev_id; } + + MGARDX_CONT static int GetQueue(SIZE queue_id) { return 0; } + + MGARDX_CONT static void SyncQueue(SIZE queue_id) { + // do nothing + } + + MGARDX_CONT static void SyncAllQueues() { + // do nothing + } + + MGARDX_CONT static void SyncDevice() { + // do nothing + } + + MGARDX_CONT static std::string GetDeviceName() { + return DeviceSpecs.GetDeviceName(openmp_dev_id); + } + + MGARDX_CONT static int GetMaxSharedMemorySize() { + return DeviceSpecs.GetMaxSharedMemorySize(openmp_dev_id); + } + + MGARDX_CONT static int GetWarpSize() { + return DeviceSpecs.GetWarpSize(openmp_dev_id); + } + + MGARDX_CONT static int GetNumSMs() { + return DeviceSpecs.GetNumSMs(openmp_dev_id); + } + + MGARDX_CONT static int GetArchitectureGeneration() { + return DeviceSpecs.GetArchitectureGeneration(openmp_dev_id); + } + + MGARDX_CONT static int GetMaxNumThreadsPerSM() { + return DeviceSpecs.GetMaxNumThreadsPerSM(openmp_dev_id); + } + + MGARDX_CONT static int GetMaxNumThreadsPerTB() { + return DeviceSpecs.GetMaxNumThreadsPerTB(openmp_dev_id); + } + + MGARDX_CONT static size_t GetAvailableMemory() { + return DeviceSpecs.GetAvailableMemory(openmp_dev_id); + } + + MGARDX_CONT static bool SupportCG() { + return DeviceSpecs.SupportCG(openmp_dev_id); + } + + template + MGARDX_CONT static int + GetOccupancyMaxActiveBlocksPerSM(FunctorType functor, int blockSize, + size_t dynamicSMemSize) { + return 32; + } + + template + MGARDX_CONT static void SetMaxDynamicSharedMemorySize(FunctorType functor, + int maxbytes) { + // do nothing + } + + MGARDX_CONT + ~DeviceRuntime() {} + + static DeviceQueues queues; + static bool SyncAllKernelsAndCheckErrors; + static DeviceSpecification DeviceSpecs; + static bool TimingAllKernels; + static bool PrintKernelConfig; +}; + +template <> class MemoryManager { +public: + MGARDX_CONT + MemoryManager(){}; + + template + MGARDX_CONT static void Malloc1D(T *&ptr, SIZE n, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Malloc1D"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = (T *)std::malloc(n * sizeof(converted_T)); + if (ptr == nullptr) { + throw std::runtime_error("MemoryManager::Malloc1D error."); + } + } + + template + MGARDX_CONT static void MallocND(T *&ptr, SIZE n1, SIZE n2, SIZE &ld, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MallocND"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = (T *)std::malloc(n1 * n2 * sizeof(converted_T)); + ld = n1; + if (ptr == nullptr) { + throw std::runtime_error("MemoryManager::MallocND error."); + } + } + + template + MGARDX_CONT static void + MallocManaged1D(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MallocManaged1D"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = (T *)std::malloc(n * sizeof(converted_T)); + if (ptr == nullptr) { + throw std::runtime_error("MemoryManager::MallocManaged1D error."); + } + } + + template + MGARDX_CONT static void Free(T *ptr, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Free"); + if (ptr == nullptr) + return; + std::free(ptr); + } + + template + MGARDX_CONT static void Copy1D(T *dst_ptr, const T *src_ptr, SIZE n, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Copy1D"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + std::memcpy(dst_ptr, src_ptr, sizeof(converted_T) * n); + } + + template + MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, + SIZE src_ld, SIZE n1, SIZE n2, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::CopyND"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + if ((dst_ld == n1) && (src_ld == n1)) { + // Can copy all rows at once + std::memcpy(dst_ptr, src_ptr, sizeof(converted_T) * n1 * n2); + } else { + // There is padding in between rows in memory. Must copy each row. + for (SIZE row = 0; row < n2; ++row) { + std::memcpy((converted_T *)dst_ptr + row * dst_ld, + (converted_T *)src_ptr + row * src_ld, + sizeof(converted_T) * n1); + } + } + } + + template + MGARDX_CONT static void + MallocHost(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MallocHost"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = (T *)std::malloc(n * sizeof(converted_T)); + if (ptr == nullptr) { + throw std::runtime_error("MemoryManager::MallocHost error."); + } + } + + template + MGARDX_CONT static void FreeHost(T *ptr, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::FreeHost"); + if (ptr == nullptr) + return; + std::free(ptr); + } + + template + MGARDX_CONT static void Memset1D(T *ptr, SIZE n, int value, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Memset1D"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + memset(ptr, value, n * sizeof(converted_T)); + } + + template + MGARDX_CONT static void MemsetND(T *ptr, SIZE ld, SIZE n1, SIZE n2, int value, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MemsetND"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + memset(ptr, value, n1 * n2 * sizeof(converted_T)); + } + + template MGARDX_CONT static bool IsDevicePointer(T *ptr) { + log::dbg("Calling MemoryManager::IsDevicePointer"); + return true; + } + + template MGARDX_CONT static int GetPointerDevice(T *ptr) { + log::dbg("Calling MemoryManager::GetPointerDevice"); + return 0; + } + + template MGARDX_CONT static bool CheckHostRegister(T *ptr) { + log::dbg("Calling MemoryManager::CheckHostRegister"); + return false; + } + + template MGARDX_CONT static void HostRegister(T *ptr, SIZE n) { + log::dbg("Calling MemoryManager::HostRegister"); + } + + template MGARDX_CONT static void HostUnregister(T *ptr) { + log::dbg("Calling MemoryManager::HostUnregister"); + } +}; + +#define ALIGN_LEFT 0 // for encoding +#define ALIGN_RIGHT 1 // for decoding + +#define Sign_Encoding_Atomic 0 +#define Sign_Encoding_Reduce 1 +#define Sign_Encoding_Ballot 2 + +#define Sign_Decoding_Parallel 0 + +#define BINARY 0 +#define NEGABINARY 1 + +#define Bit_Transpose_Serial_All 0 +#define Bit_Transpose_Parallel_B_Serial_b 1 +#define Bit_Transpose_Parallel_B_Atomic_b 2 +#define Bit_Transpose_Parallel_B_Reduce_b 3 +#define Bit_Transpose_Parallel_B_Ballot_b 4 +#define Bit_Transpose_TCU 5 + +#define Warp_Bit_Transpose_Serial_All 0 +#define Warp_Bit_Transpose_Parallel_B_Serial_b 1 +#define Warp_Bit_Transpose_Serial_B_Atomic_b 2 +#define Warp_Bit_Transpose_Serial_B_Reduce_b 3 +#define Warp_Bit_Transpose_Serial_B_Ballot_b 4 +#define Warp_Bit_Transpose_TCU 5 + +#define Error_Collecting_Disable -1 +#define Error_Collecting_Serial_All 0 +#define Error_Collecting_Parallel_Bitplanes_Serial_Error 1 +#define Error_Collecting_Parallel_Bitplanes_Atomic_Error 2 +#define Error_Collecting_Parallel_Bitplanes_Reduce_Error 3 + +#define Warp_Error_Collecting_Disable -1 +#define Warp_Error_Collecting_Serial_All 0 +#define Warp_Error_Collecting_Parallel_Bitplanes_Serial_Error 1 +#define Warp_Error_Collecting_Serial_Bitplanes_Atomic_Error 2 +#define Warp_Error_Collecting_Serial_Bitplanes_Reduce_Error 3 + +typedef unsigned long long int uint64_cu; + +template +struct BlockBitTranspose { + + MGARDX_EXEC + static void Serial_All(T_org *v, T_trans *tv, SIZE b, SIZE B, SIZE IdX, + SIZE IdY) { + if (IdX == 0 && IdY == 0) { + // printf("add-in: %llu %u\n", v, v[0]); + // for (int i = 0; i < b; i++) { + // printf("v: %u\n", v[i]); + // } + for (SIZE B_idx = 0; B_idx < B; B_idx++) { + T_trans buffer = 0; + for (SIZE b_idx = 0; b_idx < b; b_idx++) { + T_trans bit = (v[b_idx] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + if (ALIGN == ALIGN_LEFT) { + buffer += bit << (sizeof(T_trans) * 8 - 1 - b_idx); + // if (B_idx == 0) { + // printf("%u %u %u\n", B_idx, b_idx, bit); + // print_bits(buffer, sizeof(T_trans)*8, false); + // printf("\n"); + // } + } else if (ALIGN == ALIGN_RIGHT) { + buffer += bit << (b - 1 - b_idx); + // if (b_idx == 0) printf("%u %u %u\n", B_idx, b_idx, bit); + } else { + } + // if (j == 0 ) {printf("i %u j %u shift %u bit %u\n", i,j,b-1-j, + // bit); } + } + + // printf("buffer: %u\n", buffer); + + tv[B_idx] = buffer; + } + } + } + + MGARDX_EXEC + static void Transpose(T_org *v, T_trans *tv, SIZE b, SIZE B, SIZE IdX, + SIZE IdY) { + Serial_All(v, tv, b, B, IdX, IdY); + } +}; + +template +struct BlockErrorCollect { + + MGARDX_EXEC + static void Serial_All(T *v, T_error *temp, T_error *errors, SIZE num_elems, + SIZE num_bitplanes, SIZE IdX, SIZE IdY) { + if (IdX == 0 && IdY == 0) { + for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx++) { + T data = v[elem_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + for (SIZE bitplane_idx = 0; bitplane_idx < num_bitplanes; + bitplane_idx++) { + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + } + errors[num_bitplanes - bitplane_idx] += diff * diff; + // if (blockIdx.x == 0 && num_bitplanes-bitplane_idx == 2) { + // printf("elem error[%u]: %f\n", elem_idx, diff * diff); + // } + } + errors[0] += data * data; + } + } + } + + MGARDX_EXEC + static void Collect(T *v, T_error *temp, T_error *errors, SIZE num_elems, + SIZE num_bitplanes, SIZE IdX, SIZE IdY) { + Serial_All(v, temp, errors, num_elems, num_bitplanes, IdX, IdY); + } +}; + +#define ALLOC_BLOCK \ + Byte **shared_memory = new Byte *[DeviceRuntime::GetNumSMs()]; \ + for (SIZE core = 0; core < DeviceRuntime::GetNumSMs(); core++) { \ + shared_memory[core] = new Byte[task.GetSharedMemorySize()]; \ + } \ + TaskType ****threads = new TaskType ***[DeviceRuntime::GetNumSMs()]; \ + for (SIZE core = 0; core < DeviceRuntime::GetNumSMs(); core++) { \ + threads[core] = new TaskType **[task.GetBlockDimZ()]; \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + threads[core][threadz] = new TaskType *[task.GetBlockDimY()]; \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + threads[core][threadz][thready] = new TaskType[task.GetBlockDimX()]; \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + TaskType &thread = threads[core][threadz][thready][threadx]; \ + thread = task; \ + thread.GetFunctor().InitConfig( \ + task.GetGridDimZ(), task.GetGridDimY(), task.GetGridDimX(), \ + task.GetBlockDimZ(), task.GetBlockDimY(), task.GetBlockDimX()); \ + thread.GetFunctor().InitSharedMemory(shared_memory[core]); \ + thread.GetFunctor().InitThreadId(threadz, thready, threadx); \ + } \ + } \ + } \ + } + +#define DEALLOC_BLOCK \ + for (SIZE core = 0; core < DeviceRuntime::GetNumSMs(); core++) { \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + delete[] threads[core][threadz][thready]; \ + } \ + delete[] threads[core][threadz]; \ + } \ + delete[] threads[core]; \ + delete[] shared_memory[core]; \ + } \ + delete[] threads; \ + delete[] shared_memory; + +// #define INIT_BLOCK \ +// for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) {\ +// for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) {\ +// for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) {\ +// TaskType &thread = threads[threadz][thready][threadx];\ +// thread = task;\ +// threads[threadz][thready][threadx].GetFunctor().Init(\ +// task.GetGridDimZ(), task.GetGridDimY(), task.GetGridDimX(),\ +// task.GetBlockDimZ(), task.GetBlockDimY(), task.GetBlockDimX(),\ +// blockz, blocky, blockx, threadz, thready, threadx,\ +// shared_memory);\ +// }\ +// }\ +// } + +#define INIT_BLOCK \ + int core = omp_get_thread_num(); \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + threads[core][threadz][thready][threadx].GetFunctor().InitBlockId( \ + blockz, blocky, blockx); \ + } \ + } \ + } + +#define COMPUTE_BLOCK(OPERATION) \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + threads[core][threadz][thready][threadx].GetFunctor().OPERATION(); \ + } \ + } \ + } + +template MGARDX_KERL void OpenmpKernel(TaskType task) { + // Timer timer_op, timer_op1, timer_op2, timer_op3, timer_alloc, timer_init, + // timer_dealloc; timer_op.clear(); + // timer_alloc.clear();timer_init.clear();timer_dealloc.clear(); + // timer_op1.clear(); timer_op2.clear(); timer_op3.clear(); + // omp_set_num_threads(DeviceRuntime::GetNumSMs()); + // timer_alloc.start(); + ALLOC_BLOCK; +// timer_alloc.end(); +#pragma omp parallel for collapse(3) + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { + // timer_init.start(); + INIT_BLOCK; + COMPUTE_BLOCK(Operation1); + COMPUTE_BLOCK(Operation2); + COMPUTE_BLOCK(Operation3); + COMPUTE_BLOCK(Operation4); + COMPUTE_BLOCK(Operation5); + COMPUTE_BLOCK(Operation6); + COMPUTE_BLOCK(Operation7); + COMPUTE_BLOCK(Operation8); + COMPUTE_BLOCK(Operation9); + COMPUTE_BLOCK(Operation10); + // COMPUTE_BLOCK(Operation11); + // COMPUTE_BLOCK(Operation12); + // COMPUTE_BLOCK(Operation13); + // COMPUTE_BLOCK(Operation14); + // COMPUTE_BLOCK(Operation15); + // COMPUTE_BLOCK(Operation16); + // COMPUTE_BLOCK(Operation17); + // COMPUTE_BLOCK(Operation18); + // COMPUTE_BLOCK(Operation19); + // COMPUTE_BLOCK(Operation20); + } + } + } + + // timer_dealloc.start(); + DEALLOC_BLOCK; + // timer_dealloc.end(); + + // timer_op.print(task.GetFunctorName() + "_Op"); + // timer_op1.print(task.GetFunctorName() + "_Op1"); + // timer_op2.print(task.GetFunctorName() + "_Op2"); + // timer_op3.print(task.GetFunctorName() + "_Op3"); + // timer_alloc.print(task.GetFunctorName() + "_Alloc"); + // timer_init.print(task.GetFunctorName() + "_Init"); + // timer_dealloc.print(task.GetFunctorName() + "_Dealloc"); + // timer_op.clear(); + // timer_alloc.clear();timer_init.clear();timer_dealloc.clear(); +} + +#define ALLOC_BLOCK_CONDITION \ + Byte **shared_memory = new Byte *[DeviceRuntime::GetNumSMs()]; \ + TaskType ****threads = new TaskType ***[DeviceRuntime::GetNumSMs()]; \ + bool ****active = new bool ***[DeviceRuntime::GetNumSMs()]; \ + for (SIZE core = 0; core < DeviceRuntime::GetNumSMs(); core++) { \ + shared_memory[core] = new Byte[task.GetSharedMemorySize()]; \ + threads[core] = new TaskType **[task.GetBlockDimZ()]; \ + active[core] = new bool **[task.GetBlockDimZ()]; \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + threads[core][threadz] = new TaskType *[task.GetBlockDimY()]; \ + active[core][threadz] = new bool *[task.GetBlockDimY()]; \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + threads[core][threadz][thready] = new TaskType[task.GetBlockDimX()]; \ + active[core][threadz][thready] = new bool[task.GetBlockDimX()]; \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + active[core][threadz][thready][threadx] = true; \ + TaskType &thread = threads[core][threadz][thready][threadx]; \ + thread = task; \ + thread.GetFunctor().InitConfig( \ + task.GetGridDimZ(), task.GetGridDimY(), task.GetGridDimX(), \ + task.GetBlockDimZ(), task.GetBlockDimY(), task.GetBlockDimX()); \ + thread.GetFunctor().InitThreadId(threadz, thready, threadx); \ + thread.GetFunctor().InitSharedMemory(shared_memory[core]); \ + } \ + } \ + } \ + } + +#define DEALLOC_BLOCK_CONDITION \ + for (SIZE core = 0; core < DeviceRuntime::GetNumSMs(); core++) { \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + delete[] threads[core][threadz][thready]; \ + delete[] active[core][threadz][thready]; \ + } \ + delete[] threads[core][threadz]; \ + delete[] active[core][threadz]; \ + } \ + delete[] threads[core]; \ + delete[] active[core]; \ + delete[] shared_memory[core]; \ + } \ + delete[] threads; \ + delete[] active; \ + delete[] shared_memory; + +// #define INIT_BLOCK_CONDITION \ +// for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) {\ +// for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) {\ +// for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) {\ +// TaskType &thread = threads[threadz][thready][threadx];\ +// thread = task;\ +// thread.GetFunctor().InitConfig(task.GetGridDimZ(), task.GetGridDimY(), task.GetGridDimX(),\ +// task.GetBlockDimZ(), task.GetBlockDimY(), task.GetBlockDimX());\ +// thread.GetFunctor().InitSharedMemory(shared_memory);\ +// thread.GetFunctor().InitBlockId(blockz, blocky, blockx);\ +// thread.GetFunctor().InitThreadId(threadz, thready, threadx);\ +// active[threadz][thready][threadx] = true;\ +// }\ +// }\ +// } + +#define INIT_BLOCK_CONDITION \ + int core = omp_get_thread_num(); \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + TaskType &thread = threads[core][threadz][thready][threadx]; \ + thread.GetFunctor().InitBlockId(blockz, blocky, blockx); \ + active[core][threadz][thready][threadx] = true; \ + } \ + } \ + } + +#define RESET_BLOCK_CONDITION \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + active[core][threadz][thready][threadx] = true; \ + } \ + } \ + } + +#define EVALUATE_BLOCK_CONDITION(CONDITION_VAR, CONDITION_OP) \ + CONDITION_VAR = false; \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + TaskType &thread = threads[core][threadz][thready][threadx]; \ + bool thread_active = thread.GetFunctor().CONDITION_OP(); \ + active[core][threadz][thready][threadx] = thread_active; \ + CONDITION_VAR = CONDITION_VAR | thread_active; \ + } \ + } \ + } + +#define COMPUTE_BLOCK_CONDITION(OPERATION) \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + TaskType &thread = threads[core][threadz][thready][threadx]; \ + if (active[core][threadz][thready][threadx]) { \ + thread.GetFunctor().OPERATION(); \ + } \ + } \ + } \ + } + +template MGARDX_KERL void OpenmpIterKernel(TaskType task) { + ALLOC_BLOCK_CONDITION; +#pragma omp parallel for collapse(3) + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { + bool condition1 = false; + bool condition2 = false; + INIT_BLOCK_CONDITION; + COMPUTE_BLOCK_CONDITION(Operation1); + COMPUTE_BLOCK_CONDITION(Operation2); + EVALUATE_BLOCK_CONDITION(condition1, LoopCondition1); + while (condition1) { + COMPUTE_BLOCK_CONDITION(Operation3); + COMPUTE_BLOCK_CONDITION(Operation4); + COMPUTE_BLOCK_CONDITION(Operation5); + COMPUTE_BLOCK_CONDITION(Operation6); + EVALUATE_BLOCK_CONDITION(condition1, LoopCondition1); + } + RESET_BLOCK_CONDITION; + COMPUTE_BLOCK_CONDITION(Operation7); + COMPUTE_BLOCK_CONDITION(Operation8); + COMPUTE_BLOCK_CONDITION(Operation9); + COMPUTE_BLOCK_CONDITION(Operation10); + EVALUATE_BLOCK_CONDITION(condition2, LoopCondition2); + while (condition2) { + COMPUTE_BLOCK_CONDITION(Operation11); + COMPUTE_BLOCK_CONDITION(Operation12); + COMPUTE_BLOCK_CONDITION(Operation13); + COMPUTE_BLOCK_CONDITION(Operation14); + EVALUATE_BLOCK_CONDITION(condition2, LoopCondition2); + } + RESET_BLOCK_CONDITION; + COMPUTE_BLOCK_CONDITION(Operation15); + COMPUTE_BLOCK_CONDITION(Operation16); + COMPUTE_BLOCK_CONDITION(Operation17); + } + } + } + DEALLOC_BLOCK_CONDITION; +} + +#define ALLOC_GRID \ + TaskType ******threads = new TaskType *****[task.GetGridDimZ()]; \ + Byte ****shared_memory = new Byte ***[task.GetGridDimZ()]; \ + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { \ + threads[blockz] = new TaskType ****[task.GetGridDimY()]; \ + shared_memory[blockz] = new Byte **[task.GetGridDimY()]; \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + threads[blockz][blocky] = new TaskType ***[task.GetGridDimX()]; \ + shared_memory[blockz][blocky] = new Byte *[task.GetGridDimX()]; \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + threads[blockz][blocky][blockx] = \ + new TaskType **[task.GetBlockDimZ()]; \ + shared_memory[blockz][blocky][blockx] = \ + new Byte[task.GetSharedMemorySize()]; \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + threads[blockz][blocky][blockx][threadz] = \ + new TaskType *[task.GetBlockDimY()]; \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + threads[blockz][blocky][blockx][threadz][thready] = \ + new TaskType[task.GetBlockDimX()]; \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + threads[blockz][blocky][blockx][threadz][thready][threadx] = \ + task; \ + threads[blockz][blocky][blockx][threadz][thready][threadx] \ + .GetFunctor() \ + .Init(task.GetGridDimZ(), task.GetGridDimY(), \ + task.GetGridDimX(), task.GetBlockDimZ(), \ + task.GetBlockDimY(), task.GetBlockDimX(), blockz, \ + blocky, blockx, threadz, thready, threadx, \ + shared_memory[blockz][blocky][blockx]); \ + } \ + } \ + } \ + } \ + } \ + } + +#define ALLOC_ACTIVE_GRID(ACTIVE_VAR) \ + bool ******ACTIVE_VAR = new bool *****[task.GetGridDimZ()]; \ + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { \ + ACTIVE_VAR[blockz] = new bool ****[task.GetGridDimY()]; \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + ACTIVE_VAR[blockz][blocky] = new bool ***[task.GetGridDimX()]; \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + ACTIVE_VAR[blockz][blocky][blockx] = new bool **[task.GetBlockDimZ()]; \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + ACTIVE_VAR[blockz][blocky][blockx][threadz] = \ + new bool *[task.GetBlockDimY()]; \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + ACTIVE_VAR[blockz][blocky][blockx][threadz][thready] = \ + new bool[task.GetBlockDimX()]; \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + ACTIVE_VAR[blockz][blocky][blockx][threadz][thready][threadx] = \ + true; \ + } \ + } \ + } \ + } \ + } \ + } + +#define DEALLOC_GRID \ + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + } \ + delete[] threads[blockz][blocky][blockx][threadz][thready]; \ + } \ + delete[] threads[blockz][blocky][blockx][threadz]; \ + } \ + delete[] threads[blockz][blocky][blockx]; \ + delete[] shared_memory[blockz][blocky][blockx]; \ + } \ + delete[] threads[blockz][blocky]; \ + delete[] shared_memory[blockz][blocky]; \ + } \ + delete[] threads[blockz]; \ + delete[] shared_memory[blockz]; \ + } \ + delete[] threads; \ + delete[] shared_memory; + +#define DEALLOC_ACTIVE_GRID(ACTIVE_VAR) \ + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + } \ + delete[] ACTIVE_VAR[blockz][blocky][blockx][threadz][thready]; \ + } \ + delete[] ACTIVE_VAR[blockz][blocky][blockx][threadz]; \ + } \ + delete[] ACTIVE_VAR[blockz][blocky][blockx]; \ + } \ + delete[] ACTIVE_VAR[blockz][blocky]; \ + } \ + delete[] ACTIVE_VAR[blockz]; \ + } \ + delete[] ACTIVE_VAR; + +#define COMPUTE_GRID(OPERATION, ACTIVE_VAR) \ + _Pragma("omp parallel for collapse(6) ") for (SIZE blockz = 0; \ + blockz < task.GetGridDimZ(); \ + blockz++) { \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + TaskType &thread = \ + threads[blockz][blocky][blockx][threadz][thready][threadx]; \ + if (ACTIVE_VAR[blockz][blocky][blockx][threadz][thready] \ + [threadx]) { \ + thread.GetFunctor().OPERATION(); \ + } \ + } \ + } \ + } \ + } \ + } \ + } + +#define RESER_CONDITION_GRID(ACTIVE_VAR) \ + _Pragma("omp parallel for collapse(6)") for (SIZE blockz = 0; \ + blockz < task.GetGridDimZ(); \ + blockz++) { \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + ACTIVE_VAR[blockz][blocky][blockx][threadz][thready][threadx] = \ + true; \ + } \ + } \ + } \ + } \ + } \ + } + +#define INHERENT_CONDITION_GRID(ACTIVE_VAR1, ACTIVE_VAR2) \ + _Pragma("omp parallel for collapse(6)") for (SIZE blockz = 0; \ + blockz < task.GetGridDimZ(); \ + blockz++) { \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + ACTIVE_VAR2[blockz][blocky][blockx][threadz][thready][threadx] = \ + ACTIVE_VAR1[blockz][blocky][blockx][threadz][thready] \ + [threadx]; \ + } \ + } \ + } \ + } \ + } \ + } + +#define EVALUATE_CONDITION_GRID(CONDITION_VAR, ACTIVE_VAR, CONDITION_OP) \ + CONDITION_VAR = false; \ + _Pragma("omp parallel for collapse(6)") for (SIZE blockz = 0; \ + blockz < task.GetGridDimZ(); \ + blockz++) { \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + TaskType &thread = \ + threads[blockz][blocky][blockx][threadz][thready][threadx]; \ + bool thread_active = thread.GetFunctor().CONDITION_OP(); \ + ACTIVE_VAR[blockz][blocky][blockx][threadz][thready][threadx] = \ + thread_active; \ + _Pragma("omp critical") CONDITION_VAR = \ + CONDITION_VAR | thread_active; \ + } \ + } \ + } \ + } \ + } \ + } + +template +MGARDX_KERL void OpenmpHuffmanCLCustomizedKernel(TaskType task) { + + bool loop_condition1 = false; + bool loop_condition2 = false; + bool branch_condition1 = false; + + ALLOC_GRID; + ALLOC_ACTIVE_GRID(all_active); + ALLOC_ACTIVE_GRID(loop1_active); + ALLOC_ACTIVE_GRID(loop2_active); + ALLOC_ACTIVE_GRID(branch1_active); + + COMPUTE_GRID(Operation1, all_active); + INHERENT_CONDITION_GRID(all_active, loop1_active); + EVALUATE_CONDITION_GRID(loop_condition1, loop1_active, LoopCondition1); + while (loop_condition1) { + COMPUTE_GRID(Operation2, loop1_active); + COMPUTE_GRID(Operation3, loop1_active); + COMPUTE_GRID(Operation4, loop1_active); + COMPUTE_GRID(Operation5, loop1_active); + COMPUTE_GRID(Operation6, loop1_active); + INHERENT_CONDITION_GRID(loop1_active, branch1_active); + EVALUATE_CONDITION_GRID(branch_condition1, branch1_active, + BranchCondition1); + if (branch_condition1) { + INHERENT_CONDITION_GRID(branch1_active, loop2_active); + EVALUATE_CONDITION_GRID(loop_condition2, loop2_active, LoopCondition2); + while (loop_condition2) { + COMPUTE_GRID(Operation7, loop2_active); + COMPUTE_GRID(Operation8, loop2_active); + COMPUTE_GRID(Operation9, loop2_active); + EVALUATE_CONDITION_GRID(loop_condition2, loop2_active, LoopCondition2); + } + COMPUTE_GRID(Operation10, branch1_active) + COMPUTE_GRID(Operation11, branch1_active); + } + COMPUTE_GRID(Operation12, loop1_active); + COMPUTE_GRID(Operation13, loop1_active); + COMPUTE_GRID(Operation14, loop1_active); + COMPUTE_GRID(Operation15, loop1_active); + EVALUATE_CONDITION_GRID(loop_condition1, loop1_active, LoopCondition1); + } + + DEALLOC_GRID; + DEALLOC_ACTIVE_GRID(all_active); + DEALLOC_ACTIVE_GRID(loop1_active); + DEALLOC_ACTIVE_GRID(loop2_active); + DEALLOC_ACTIVE_GRID(branch1_active); +} + +template +MGARDX_KERL void OpenmpHuffmanCWCustomizedKernel(TaskType task) { + + bool loop_condition1 = false; + + ALLOC_GRID; + ALLOC_ACTIVE_GRID(all_active); + ALLOC_ACTIVE_GRID(loop1_active); + COMPUTE_GRID(Operation1, all_active); + COMPUTE_GRID(Operation2, all_active); + COMPUTE_GRID(Operation3, all_active); + INHERENT_CONDITION_GRID(all_active, loop1_active); + EVALUATE_CONDITION_GRID(loop_condition1, loop1_active, LoopCondition1); + while (loop_condition1) { + COMPUTE_GRID(Operation4, loop1_active); + COMPUTE_GRID(Operation5, loop1_active); + COMPUTE_GRID(Operation6, loop1_active); + COMPUTE_GRID(Operation7, loop1_active); + COMPUTE_GRID(Operation8, loop1_active); + EVALUATE_CONDITION_GRID(loop_condition1, loop1_active, LoopCondition1); + } + COMPUTE_GRID(Operation9, all_active); + COMPUTE_GRID(Operation10, all_active); + + DEALLOC_GRID; + DEALLOC_ACTIVE_GRID(all_active); + DEALLOC_ACTIVE_GRID(loop1_active); +} + +// template class DeviceAdapter { +// public: +// MGARDX_CONT +// DeviceAdapter(){}; + +// MGARDX_CONT +// int IsResourceEnough(TaskType &task) { +// if (task.GetBlockDimX() * task.GetBlockDimY() * task.GetBlockDimZ() > +// DeviceRuntime::GetMaxNumThreadsPerTB()) { +// return THREADBLOCK_TOO_LARGE; +// } +// if (task.GetSharedMemorySize() > +// DeviceRuntime::GetMaxSharedMemorySize()) { +// return SHARED_MEMORY_TOO_LARGE; +// } +// return RESOURCE_ENOUGH; +// } + +// MGARDX_CONT +// ExecutionReturn Execute(TaskType &task) { + +// if (DeviceRuntime::PrintKernelConfig) { +// std::cout << log::log_info << task.GetFunctorName() << ": <" +// << task.GetBlockDimX() << ", " << task.GetBlockDimY() << ", " +// << task.GetBlockDimZ() << "> <" << task.GetGridDimX() << ", " +// << task.GetGridDimY() << ", " << task.GetGridDimZ() << ">\n"; +// } + +// ExecutionReturn ret; +// if (IsResourceEnough(task) != RESOURCE_ENOUGH) { +// if (DeviceRuntime::PrintKernelConfig) { +// if (IsResourceEnough(task) == THREADBLOCK_TOO_LARGE) { +// log::info("threadblock too large."); +// } +// if (IsResourceEnough(task) == SHARED_MEMORY_TOO_LARGE) { +// log::info("shared memory too large."); +// } +// } +// ret.success = false; +// ret.execution_time = std::numeric_limits::max(); +// return ret; +// } + +// Timer timer; +// if (DeviceRuntime::TimingAllKernels || +// AutoTuner::ProfileKernels) { +// DeviceRuntime::SyncDevice(); +// timer.start(); +// } +// // if constexpr evalute at compile time otherwise this does not compile +// if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) { +// OpenmpKernel(task); +// } else if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) +// { +// OpenmpIterKernel(task); +// } else if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) +// { +// // int prev_omp_thread = omp_get_max_threads(); +// // This kernel works better with 4 threads +// // omp_set_num_threads(4); +// OpenmpHuffmanCLCustomizedKernel(task); +// // omp_set_num_threads(prev_omp_thread); +// } else if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) +// { +// // int prev_omp_thread = omp_get_max_threads(); +// // This kernel works better with 4 threads +// // omp_set_num_threads(4); +// OpenmpHuffmanCWCustomizedKernel(task); +// // omp_set_num_threads(prev_omp_thread); +// } +// // timer.end(); +// // timer.print(task.GetFunctorName()); +// // timer.clear(); +// if (DeviceRuntime::TimingAllKernels || +// AutoTuner::ProfileKernels) { +// DeviceRuntime::SyncDevice(); +// timer.end(); +// if (DeviceRuntime::TimingAllKernels) { +// timer.print(task.GetFunctorName()); +// } +// if (AutoTuner::ProfileKernels) { +// ret.success = true; +// ret.execution_time = timer.get(); +// } +// } +// return ret; +// } +// }; + +template <> class DeviceLauncher { +public: + template + MGARDX_CONT int static IsResourceEnough(TaskType &task) { + if (task.GetBlockDimX() * task.GetBlockDimY() * task.GetBlockDimZ() > + DeviceRuntime::GetMaxNumThreadsPerTB()) { + return THREADBLOCK_TOO_LARGE; + } + if (task.GetSharedMemorySize() > + DeviceRuntime::GetMaxSharedMemorySize()) { + return SHARED_MEMORY_TOO_LARGE; + } + return RESOURCE_ENOUGH; + } + + template + MGARDX_CONT ExecutionReturn static Execute(TaskType &task) { + if (DeviceRuntime::PrintKernelConfig) { + std::cout << log::log_info << task.GetFunctorName() << ": <" + << task.GetBlockDimX() << ", " << task.GetBlockDimY() << ", " + << task.GetBlockDimZ() << "> <" << task.GetGridDimX() << ", " + << task.GetGridDimY() << ", " << task.GetGridDimZ() << "> " + << " core(s): " << DeviceRuntime::GetNumSMs() << "\n"; + } + + ExecutionReturn ret; + if (IsResourceEnough(task) != RESOURCE_ENOUGH) { + if (DeviceRuntime::PrintKernelConfig) { + if (IsResourceEnough(task) == THREADBLOCK_TOO_LARGE) { + log::info("threadblock too large."); + } + if (IsResourceEnough(task) == SHARED_MEMORY_TOO_LARGE) { + log::info("shared memory too large."); + } + } + ret.success = false; + ret.execution_time = std::numeric_limits::max(); + return ret; + } + + Timer timer; + if (DeviceRuntime::TimingAllKernels || + AutoTuner::ProfileKernels) { + DeviceRuntime::SyncDevice(); + timer.start(); + } + // if constexpr evalute at compile time otherwise this does not compile + if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + OpenmpKernel(task); + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + OpenmpIterKernel(task); + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + // int prev_omp_thread = omp_get_max_threads(); + // This kernel works better with 4 threads + // omp_set_num_threads(4); + OpenmpHuffmanCLCustomizedKernel(task); + // omp_set_num_threads(prev_omp_thread); + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + // int prev_omp_thread = omp_get_max_threads(); + // This kernel works better with 4 threads + // omp_set_num_threads(4); + OpenmpHuffmanCWCustomizedKernel(task); + // omp_set_num_threads(prev_omp_thread); + } + // timer.end(); + // timer.print(task.GetFunctorName()); + // timer.clear(); + if (DeviceRuntime::TimingAllKernels || + AutoTuner::ProfileKernels) { + DeviceRuntime::SyncDevice(); + timer.end(); + if (DeviceRuntime::TimingAllKernels) { + timer.print(task.GetFunctorName()); + } + if (AutoTuner::ProfileKernels) { + ret.success = true; + ret.execution_time = timer.get(); + } + } + return ret; + } + + template + MGARDX_CONT static void ConfigTask(TaskType task) { + typename TaskType::Functor functor; + int maxbytes = DeviceRuntime::GetMaxSharedMemorySize(); + DeviceRuntime::SetMaxDynamicSharedMemorySize(functor, maxbytes); + } + + template + MGARDX_CONT static void AutoTune(KernelType kernel, int queue_idx) { +#if MGARD_ENABLE_AUTO_TUNING + double min_time = std::numeric_limits::max(); + int min_config = 0; + ExecutionReturn ret; +#define RUN_CONFIG(CONFIG_IDX) \ + { \ + constexpr ExecutionConfig config = \ + GetExecutionConfig(CONFIG_IDX); \ + auto task = \ + kernel.template GenTask(queue_idx); \ + ret = Execute(task); \ + if constexpr (KernelType::EnableConfig()) { \ + ConfigTask(task); \ + } \ + if (ret.success && min_time > ret.execution_time) { \ + min_time = ret.execution_time; \ + min_config = CONFIG_IDX; \ + } \ + } + RUN_CONFIG(0) + RUN_CONFIG(1) + RUN_CONFIG(2) + RUN_CONFIG(3) + RUN_CONFIG(4) + RUN_CONFIG(5) + RUN_CONFIG(6) +#undef RUN_CONFIG + if (AutoTuner::WriteToTable) { + FillAutoTunerTable(std::string(KernelType::Name), min_config); + } +#else + throw std::runtime_error("MGARD is not built with auto tuning enabled."); +#endif + } + + template + MGARDX_CONT static void Execute(KernelType kernel, int queue_idx) { + if constexpr (KernelType::EnableAutoTuning()) { + constexpr ExecutionConfig config = + GetExecutionConfig(KernelType::Name); + auto task = + kernel.template GenTask(queue_idx); + if constexpr (KernelType::EnableConfig()) { + ConfigTask(task); + } + Execute(task); + + if (AutoTuner::ProfileKernels) { + AutoTune(kernel, queue_idx); + } + } else { + auto task = kernel.GenTask(queue_idx); + if constexpr (KernelType::EnableConfig()) { + ConfigTask(task); + } + Execute(task); + } + } +}; + +template <> class DeviceCollective { +public: + MGARDX_CONT + DeviceCollective(){}; + + template + MGARDX_CONT static void Sum(SIZE n, SubArray<1, T, OPENMP> v, + SubArray<1, T, OPENMP> result, + Array<1, Byte, OPENMP> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + *result((IDX)0) = std::accumulate(v((IDX)0), v((IDX)n), (T)0); + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void AbsMax(SIZE n, SubArray<1, T, OPENMP> v, + SubArray<1, T, OPENMP> result, + Array<1, Byte, OPENMP> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + T max_result = 0; + for (SIZE i = 0; i < n; ++i) { + max_result = std::max((T)fabs(*v(i)), max_result); + } + *result((IDX)0) = max_result; + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void AbsMin(SIZE n, SubArray<1, T, OPENMP> v, + SubArray<1, T, OPENMP> result, + Array<1, Byte, OPENMP> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + T max_result = std::numeric_limits::max(); + for (SIZE i = 0; i < n; ++i) { + max_result = std::min((T)fabs(*v(i)), max_result); + } + *result((IDX)0) = max_result; + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void SquareSum(SIZE n, SubArray<1, T, OPENMP> v, + SubArray<1, T, OPENMP> result, + Array<1, Byte, OPENMP> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + T sum_result = 0; + for (SIZE i = 0; i < n; ++i) { + T tmp = *v(i); + sum_result += tmp * tmp; + } + *result((IDX)0) = sum_result; + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void ScanSumInclusive(SIZE n, SubArray<1, T, OPENMP> v, + SubArray<1, T, OPENMP> result, + Array<1, Byte, OPENMP> &workspace, + bool workspace_allocated, + int queue_idx) { + + // Need gcc 9 and c++17 +#if (__GNUC__ >= 9) + if (workspace_allocated) { + std::inclusive_scan(v((IDX)0), v((IDX)n), result((IDX)0)); + } else { + workspace.resize({(SIZE)1}, queue_idx); + } +#else + throw std::runtime_error( + "Please recompile with GCC 9+ to use ScanSumInclusive."); +#endif + } + + template + MGARDX_CONT static void ScanSumExclusive(SIZE n, SubArray<1, T, OPENMP> v, + SubArray<1, T, OPENMP> result, + Array<1, Byte, OPENMP> &workspace, + bool workspace_allocated, + int queue_idx) { + + // Need gcc 9 and c++17 +#if (__GNUC__ >= 9) + if (workspace_allocated) { + std::exclusive_scan(v((IDX)0), v((IDX)n), result((IDX)0)); + } else { + workspace.resize({(SIZE)1}, queue_idx); + } +#else + throw std::runtime_error( + "Please recompile with GCC 9+ to use ScanSumExclusive."); +#endif + } + + template + MGARDX_CONT static void ScanSumExtended(SIZE n, SubArray<1, T, OPENMP> v, + SubArray<1, T, OPENMP> result, + Array<1, Byte, OPENMP> &workspace, + bool workspace_allocated, + int queue_idx) { + + // Need gcc 9 and c++17 +#if (__GNUC__ >= 9) + if (workspace_allocated) { + std::inclusive_scan(v((IDX)0), v((IDX)n), result((IDX)1)); + *result((IDX)0) = 0; + } else { + workspace.resize({(SIZE)1}, queue_idx); + } +#else + throw std::runtime_error( + "Please recompile with GCC 9+ to use ScanSumExtended."); +#endif + } + + template + MGARDX_CONT static void SortByKey(SIZE n, SubArray<1, KeyT, OPENMP> in_keys, + SubArray<1, ValueT, OPENMP> in_values, + SubArray<1, KeyT, OPENMP> out_keys, + SubArray<1, ValueT, OPENMP> out_values, + Array<1, Byte, OPENMP> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + std::vector> data(n); + for (SIZE i = 0; i < n; ++i) { + data[i] = std::pair(*in_keys(i), *in_values(i)); + } + std::stable_sort(data.begin(), data.end(), + KeyValueComparator{}); + for (SIZE i = 0; i < n; ++i) { + *out_keys(i) = data[i].first; + *out_values(i) = data[i].second; + } + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h b/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h new file mode 100644 index 000000000..b3a09acb9 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h @@ -0,0 +1,1501 @@ +#ifndef MGARD_X_160_DEVICE_ADAPTER_SERIAL_H +#define MGARD_X_160_DEVICE_ADAPTER_SERIAL_H + +#include "DeviceAdapter.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mgard_x_160 { + +template <> struct SyncBlock { + MGARDX_EXEC static void Sync() { + // do nothing + } +}; + +template <> struct SyncGrid { + MGARDX_EXEC static void Sync() { + // do nothing + } +}; + +template +struct Atomic { + MGARDX_EXEC static T Min(T *result, T value) { + T old = *result; + *result = std::min(*result, value); + return old; + } + MGARDX_EXEC static T Max(T *result, T value) { + T old = *result; + *result = std::max(*result, value); + return old; + } + MGARDX_EXEC static T Add(T *result, T value) { + T old = *result; + *result += value; + return old; + } + MGARDX_EXEC static T Or(T *result, T value) { + T old = *result; + *result |= value; + return old; + } +}; + +template <> struct Math { + template MGARDX_EXEC static T Min(T a, T b) { + return std::min(a, b); + } + template MGARDX_EXEC static T Max(T a, T b) { + return std::max(a, b); + } + + MGARDX_EXEC static int ffs(unsigned int a) { return ffs(a); } + MGARDX_EXEC static int ffsll(long long unsigned int a) { + // return ffsll(a); + int pos = 0; + if (a == 0) + return pos; + pos = MultiplyDeBruijnBitPosition + [((long long unsigned int)((a & -a) * 151050438420815295)) >> 58]; + // while (!(a & 1)) + // { + // a >>= 1; + // ++pos; + // } + // if (pos2 != pos) {printf("mismatch: %d %d", pos2, pos);} + return pos + 1; + } + MGARDX_EXEC + static uint64_t binary2negabinary(const int64_t x) { + return (x + (uint64_t)0xaaaaaaaaaaaaaaaaull) ^ + (uint64_t)0xaaaaaaaaaaaaaaaaull; + } + + MGARDX_EXEC + static uint32_t binary2negabinary(const int32_t x) { + return (x + (uint32_t)0xaaaaaaaau) ^ (uint32_t)0xaaaaaaaau; + } + + MGARDX_EXEC + static int64_t negabinary2binary(const uint64_t x) { + return (x ^ 0xaaaaaaaaaaaaaaaaull) - 0xaaaaaaaaaaaaaaaaull; + } + + MGARDX_EXEC + static int32_t negabinary2binary(const uint32_t x) { + return (x ^ 0xaaaaaaaau) - 0xaaaaaaaau; + } +}; + +#define ALLOC_BLOCK \ + Byte *shared_memory = new Byte[task.GetSharedMemorySize()]; \ + TaskType ***threads = new TaskType **[task.GetBlockDimZ()]; \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + threads[threadz] = new TaskType *[task.GetBlockDimY()]; \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + threads[threadz][thready] = new TaskType[task.GetBlockDimX()]; \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + TaskType &thread = threads[threadz][thready][threadx]; \ + thread = task; \ + thread.GetFunctor().InitConfig( \ + task.GetGridDimZ(), task.GetGridDimY(), task.GetGridDimX(), \ + task.GetBlockDimZ(), task.GetBlockDimY(), task.GetBlockDimX()); \ + thread.GetFunctor().InitSharedMemory(shared_memory); \ + thread.GetFunctor().InitThreadId(threadz, thready, threadx); \ + } \ + } \ + } + +#define DEALLOC_BLOCK \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + delete[] threads[threadz][thready]; \ + } \ + delete[] threads[threadz]; \ + } \ + delete[] threads; \ + delete[] shared_memory; + +// #define INIT_BLOCK \ +// for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) {\ +// for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) {\ +// for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) {\ +// TaskType &thread = threads[threadz][thready][threadx];\ +// thread = task;\ +// threads[threadz][thready][threadx].GetFunctor().Init(\ +// task.GetGridDimZ(), task.GetGridDimY(), task.GetGridDimX(),\ +// task.GetBlockDimZ(), task.GetBlockDimY(), task.GetBlockDimX(),\ +// blockz, blocky, blockx, threadz, thready, threadx,\ +// shared_memory);\ +// }\ +// }\ +// } + +#define INIT_BLOCK \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + threads[threadz][thready][threadx].GetFunctor().InitBlockId( \ + blockz, blocky, blockx); \ + } \ + } \ + } + +#define COMPUTE_BLOCK(OPERATION) \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + threads[threadz][thready][threadx].GetFunctor().OPERATION(); \ + } \ + } \ + } + +template MGARDX_KERL void SerialKernel(TaskType task) { + // Timer timer_op, timer_op1, timer_op2, timer_op3, timer_alloc, timer_init, + // timer_dealloc; timer_op.clear(); + // timer_alloc.clear();timer_init.clear();timer_dealloc.clear(); + // timer_op1.clear(); timer_op2.clear(); timer_op3.clear(); + // timer_alloc.start(); + ALLOC_BLOCK; + // timer_alloc.end(); + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { + // timer_init.start(); + INIT_BLOCK; + COMPUTE_BLOCK(Operation1); + COMPUTE_BLOCK(Operation2); + COMPUTE_BLOCK(Operation3); + COMPUTE_BLOCK(Operation4); + COMPUTE_BLOCK(Operation5); + COMPUTE_BLOCK(Operation6); + COMPUTE_BLOCK(Operation7); + COMPUTE_BLOCK(Operation8); + COMPUTE_BLOCK(Operation9); + COMPUTE_BLOCK(Operation10); + COMPUTE_BLOCK(Operation11); + COMPUTE_BLOCK(Operation12); + COMPUTE_BLOCK(Operation13); + COMPUTE_BLOCK(Operation14); + COMPUTE_BLOCK(Operation15); + COMPUTE_BLOCK(Operation16); + COMPUTE_BLOCK(Operation17); + COMPUTE_BLOCK(Operation18); + COMPUTE_BLOCK(Operation19); + COMPUTE_BLOCK(Operation20); + } + } + } + // timer_dealloc.start(); + DEALLOC_BLOCK; + // timer_dealloc.end(); + + // timer_op.print(task.GetFunctorName() + "_Op"); + // timer_op1.print(task.GetFunctorName() + "_Op1"); + // timer_op2.print(task.GetFunctorName() + "_Op2"); + // timer_op3.print(task.GetFunctorName() + "_Op3"); + // timer_alloc.print(task.GetFunctorName() + "_Alloc"); + // timer_init.print(task.GetFunctorName() + "_Init"); + // timer_dealloc.print(task.GetFunctorName() + "_Dealloc"); + // timer_op.clear(); + // timer_alloc.clear();timer_init.clear();timer_dealloc.clear(); +} + +#define ALLOC_BLOCK_CONDITION \ + Byte *shared_memory = new Byte[task.GetSharedMemorySize()]; \ + TaskType ***threads = new TaskType **[task.GetBlockDimZ()]; \ + bool ***active = new bool **[task.GetBlockDimZ()]; \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + threads[threadz] = new TaskType *[task.GetBlockDimY()]; \ + active[threadz] = new bool *[task.GetBlockDimY()]; \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + threads[threadz][thready] = new TaskType[task.GetBlockDimX()]; \ + active[threadz][thready] = new bool[task.GetBlockDimX()]; \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + active[threadz][thready][threadx] = true; \ + TaskType &thread = threads[threadz][thready][threadx]; \ + thread = task; \ + thread.GetFunctor().InitConfig( \ + task.GetGridDimZ(), task.GetGridDimY(), task.GetGridDimX(), \ + task.GetBlockDimZ(), task.GetBlockDimY(), task.GetBlockDimX()); \ + thread.GetFunctor().InitThreadId(threadz, thready, threadx); \ + thread.GetFunctor().InitSharedMemory(shared_memory); \ + } \ + } \ + } + +#define DEALLOC_BLOCK_CONDITION \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + delete[] threads[threadz][thready]; \ + delete[] active[threadz][thready]; \ + } \ + delete[] threads[threadz]; \ + delete[] active[threadz]; \ + } \ + delete[] threads; \ + delete[] active; \ + delete[] shared_memory; + +// #define INIT_BLOCK_CONDITION \ +// for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) {\ +// for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) {\ +// for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) {\ +// TaskType &thread = threads[threadz][thready][threadx];\ +// thread = task;\ +// thread.GetFunctor().InitConfig(task.GetGridDimZ(), task.GetGridDimY(), task.GetGridDimX(),\ +// task.GetBlockDimZ(), task.GetBlockDimY(), task.GetBlockDimX());\ +// thread.GetFunctor().InitSharedMemory(shared_memory);\ +// thread.GetFunctor().InitBlockId(blockz, blocky, blockx);\ +// thread.GetFunctor().InitThreadId(threadz, thready, threadx);\ +// active[threadz][thready][threadx] = true;\ +// }\ +// }\ +// } + +#define INIT_BLOCK_CONDITION \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + TaskType &thread = threads[threadz][thready][threadx]; \ + thread.GetFunctor().InitBlockId(blockz, blocky, blockx); \ + active[threadz][thready][threadx] = true; \ + } \ + } \ + } + +#define RESET_BLOCK_CONDITION \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + active[threadz][thready][threadx] = true; \ + } \ + } \ + } + +#define EVALUATE_BLOCK_CONDITION(CONDITION_VAR, CONDITION_OP) \ + CONDITION_VAR = false; \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + TaskType &thread = threads[threadz][thready][threadx]; \ + bool thread_active = thread.GetFunctor().CONDITION_OP(); \ + active[threadz][thready][threadx] = thread_active; \ + CONDITION_VAR = CONDITION_VAR | thread_active; \ + } \ + } \ + } + +#define COMPUTE_BLOCK_CONDITION(OPERATION) \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + TaskType &thread = threads[threadz][thready][threadx]; \ + if (active[threadz][thready][threadx]) { \ + thread.GetFunctor().OPERATION(); \ + } \ + } \ + } \ + } + +template MGARDX_KERL void SerialIterKernel(TaskType task) { + ALLOC_BLOCK_CONDITION; + bool condition1 = false; + bool condition2 = false; + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { + INIT_BLOCK_CONDITION; + COMPUTE_BLOCK_CONDITION(Operation1); + COMPUTE_BLOCK_CONDITION(Operation2); + EVALUATE_BLOCK_CONDITION(condition1, LoopCondition1); + while (condition1) { + COMPUTE_BLOCK_CONDITION(Operation3); + COMPUTE_BLOCK_CONDITION(Operation4); + COMPUTE_BLOCK_CONDITION(Operation5); + COMPUTE_BLOCK_CONDITION(Operation6); + EVALUATE_BLOCK_CONDITION(condition1, LoopCondition1); + } + RESET_BLOCK_CONDITION; + COMPUTE_BLOCK_CONDITION(Operation7); + COMPUTE_BLOCK_CONDITION(Operation8); + COMPUTE_BLOCK_CONDITION(Operation9); + COMPUTE_BLOCK_CONDITION(Operation10); + EVALUATE_BLOCK_CONDITION(condition2, LoopCondition2); + while (condition2) { + COMPUTE_BLOCK_CONDITION(Operation11); + COMPUTE_BLOCK_CONDITION(Operation12); + COMPUTE_BLOCK_CONDITION(Operation13); + COMPUTE_BLOCK_CONDITION(Operation14); + EVALUATE_BLOCK_CONDITION(condition2, LoopCondition2); + } + RESET_BLOCK_CONDITION; + COMPUTE_BLOCK_CONDITION(Operation15); + COMPUTE_BLOCK_CONDITION(Operation16); + COMPUTE_BLOCK_CONDITION(Operation17); + } + } + } + DEALLOC_BLOCK_CONDITION; +} + +#define ALLOC_GRID \ + TaskType ******threads = new TaskType *****[task.GetGridDimZ()]; \ + Byte ****shared_memory = new Byte ***[task.GetGridDimZ()]; \ + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { \ + threads[blockz] = new TaskType ****[task.GetGridDimY()]; \ + shared_memory[blockz] = new Byte **[task.GetGridDimY()]; \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + threads[blockz][blocky] = new TaskType ***[task.GetGridDimX()]; \ + shared_memory[blockz][blocky] = new Byte *[task.GetGridDimX()]; \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + threads[blockz][blocky][blockx] = \ + new TaskType **[task.GetBlockDimZ()]; \ + shared_memory[blockz][blocky][blockx] = \ + new Byte[task.GetSharedMemorySize()]; \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + threads[blockz][blocky][blockx][threadz] = \ + new TaskType *[task.GetBlockDimY()]; \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + threads[blockz][blocky][blockx][threadz][thready] = \ + new TaskType[task.GetBlockDimX()]; \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + threads[blockz][blocky][blockx][threadz][thready][threadx] = \ + task; \ + threads[blockz][blocky][blockx][threadz][thready][threadx] \ + .GetFunctor() \ + .Init(task.GetGridDimZ(), task.GetGridDimY(), \ + task.GetGridDimX(), task.GetBlockDimZ(), \ + task.GetBlockDimY(), task.GetBlockDimX(), blockz, \ + blocky, blockx, threadz, thready, threadx, \ + shared_memory[blockz][blocky][blockx]); \ + } \ + } \ + } \ + } \ + } \ + } + +#define ALLOC_ACTIVE_GRID(ACTIVE_VAR) \ + bool ******ACTIVE_VAR = new bool *****[task.GetGridDimZ()]; \ + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { \ + ACTIVE_VAR[blockz] = new bool ****[task.GetGridDimY()]; \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + ACTIVE_VAR[blockz][blocky] = new bool ***[task.GetGridDimX()]; \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + ACTIVE_VAR[blockz][blocky][blockx] = new bool **[task.GetBlockDimZ()]; \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + ACTIVE_VAR[blockz][blocky][blockx][threadz] = \ + new bool *[task.GetBlockDimY()]; \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + ACTIVE_VAR[blockz][blocky][blockx][threadz][thready] = \ + new bool[task.GetBlockDimX()]; \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + ACTIVE_VAR[blockz][blocky][blockx][threadz][thready][threadx] = \ + true; \ + } \ + } \ + } \ + } \ + } \ + } + +#define DEALLOC_GRID \ + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + } \ + delete[] threads[blockz][blocky][blockx][threadz][thready]; \ + } \ + delete[] threads[blockz][blocky][blockx][threadz]; \ + } \ + delete[] threads[blockz][blocky][blockx]; \ + delete[] shared_memory[blockz][blocky][blockx]; \ + } \ + delete[] threads[blockz][blocky]; \ + delete[] shared_memory[blockz][blocky]; \ + } \ + delete[] threads[blockz]; \ + delete[] shared_memory[blockz]; \ + } \ + delete[] threads; \ + delete[] shared_memory; + +#define DEALLOC_ACTIVE_GRID(ACTIVE_VAR) \ + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + } \ + delete[] ACTIVE_VAR[blockz][blocky][blockx][threadz][thready]; \ + } \ + delete[] ACTIVE_VAR[blockz][blocky][blockx][threadz]; \ + } \ + delete[] ACTIVE_VAR[blockz][blocky][blockx]; \ + } \ + delete[] ACTIVE_VAR[blockz][blocky]; \ + } \ + delete[] ACTIVE_VAR[blockz]; \ + } \ + delete[] ACTIVE_VAR; + +#define COMPUTE_GRID(OPERATION, ACTIVE_VAR) \ + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + TaskType &thread = \ + threads[blockz][blocky][blockx][threadz][thready][threadx]; \ + if (ACTIVE_VAR[blockz][blocky][blockx][threadz][thready] \ + [threadx]) { \ + thread.GetFunctor().OPERATION(); \ + } \ + } \ + } \ + } \ + } \ + } \ + } + +#define RESER_CONDITION_GRID(ACTIVE_VAR) \ + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + ACTIVE_VAR[blockz][blocky][blockx][threadz][thready][threadx] = \ + true; \ + } \ + } \ + } \ + } \ + } \ + } + +#define INHERENT_CONDITION_GRID(ACTIVE_VAR1, ACTIVE_VAR2) \ + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + ACTIVE_VAR2[blockz][blocky][blockx][threadz][thready][threadx] = \ + ACTIVE_VAR1[blockz][blocky][blockx][threadz][thready] \ + [threadx]; \ + } \ + } \ + } \ + } \ + } \ + } + +#define EVALUATE_CONDITION_GRID(CONDITION_VAR, ACTIVE_VAR, CONDITION_OP) \ + CONDITION_VAR = false; \ + for (SIZE blockz = 0; blockz < task.GetGridDimZ(); blockz++) { \ + for (SIZE blocky = 0; blocky < task.GetGridDimY(); blocky++) { \ + for (SIZE blockx = 0; blockx < task.GetGridDimX(); blockx++) { \ + for (SIZE threadz = 0; threadz < task.GetBlockDimZ(); threadz++) { \ + for (SIZE thready = 0; thready < task.GetBlockDimY(); thready++) { \ + for (SIZE threadx = 0; threadx < task.GetBlockDimX(); threadx++) { \ + TaskType &thread = \ + threads[blockz][blocky][blockx][threadz][thready][threadx]; \ + bool thread_active = thread.GetFunctor().CONDITION_OP(); \ + ACTIVE_VAR[blockz][blocky][blockx][threadz][thready][threadx] = \ + thread_active; \ + CONDITION_VAR = CONDITION_VAR | thread_active; \ + } \ + } \ + } \ + } \ + } \ + } + +template +MGARDX_KERL void SerialHuffmanCLCustomizedKernel(TaskType task) { + + bool loop_condition1 = false; + bool loop_condition2 = false; + bool branch_condition1 = false; + + ALLOC_GRID; + ALLOC_ACTIVE_GRID(all_active); + ALLOC_ACTIVE_GRID(loop1_active); + ALLOC_ACTIVE_GRID(loop2_active); + ALLOC_ACTIVE_GRID(branch1_active); + + COMPUTE_GRID(Operation1, all_active); + INHERENT_CONDITION_GRID(all_active, loop1_active); + EVALUATE_CONDITION_GRID(loop_condition1, loop1_active, LoopCondition1); + while (loop_condition1) { + COMPUTE_GRID(Operation2, loop1_active); + COMPUTE_GRID(Operation3, loop1_active); + COMPUTE_GRID(Operation4, loop1_active); + COMPUTE_GRID(Operation5, loop1_active); + COMPUTE_GRID(Operation6, loop1_active); + INHERENT_CONDITION_GRID(loop1_active, branch1_active); + EVALUATE_CONDITION_GRID(branch_condition1, branch1_active, + BranchCondition1); + if (branch_condition1) { + INHERENT_CONDITION_GRID(branch1_active, loop2_active); + EVALUATE_CONDITION_GRID(loop_condition2, loop2_active, LoopCondition2); + while (loop_condition2) { + COMPUTE_GRID(Operation7, loop2_active); + COMPUTE_GRID(Operation8, loop2_active); + COMPUTE_GRID(Operation9, loop2_active); + EVALUATE_CONDITION_GRID(loop_condition2, loop2_active, LoopCondition2); + } + COMPUTE_GRID(Operation10, branch1_active) + COMPUTE_GRID(Operation11, branch1_active); + } + COMPUTE_GRID(Operation12, loop1_active); + COMPUTE_GRID(Operation13, loop1_active); + COMPUTE_GRID(Operation14, loop1_active); + COMPUTE_GRID(Operation15, loop1_active); + EVALUATE_CONDITION_GRID(loop_condition1, loop1_active, LoopCondition1); + } + + DEALLOC_GRID; + DEALLOC_ACTIVE_GRID(all_active); + DEALLOC_ACTIVE_GRID(loop1_active); + DEALLOC_ACTIVE_GRID(loop2_active); + DEALLOC_ACTIVE_GRID(branch1_active); +} + +template +MGARDX_KERL void SerialHuffmanCWCustomizedKernel(TaskType task) { + + bool loop_condition1 = false; + + ALLOC_GRID; + ALLOC_ACTIVE_GRID(all_active); + ALLOC_ACTIVE_GRID(loop1_active); + COMPUTE_GRID(Operation1, all_active); + COMPUTE_GRID(Operation2, all_active); + COMPUTE_GRID(Operation3, all_active); + INHERENT_CONDITION_GRID(all_active, loop1_active); + EVALUATE_CONDITION_GRID(loop_condition1, loop1_active, LoopCondition1); + while (loop_condition1) { + COMPUTE_GRID(Operation4, loop1_active); + COMPUTE_GRID(Operation5, loop1_active); + COMPUTE_GRID(Operation6, loop1_active); + COMPUTE_GRID(Operation7, loop1_active); + COMPUTE_GRID(Operation8, loop1_active); + EVALUATE_CONDITION_GRID(loop_condition1, loop1_active, LoopCondition1); + } + COMPUTE_GRID(Operation9, all_active); + COMPUTE_GRID(Operation10, all_active); + + DEALLOC_GRID; + DEALLOC_ACTIVE_GRID(all_active); + DEALLOC_ACTIVE_GRID(loop1_active); +} + +template <> class DeviceSpecification { +public: + MGARDX_CONT + DeviceSpecification() { + // Get the number of cores + NumDevices = std::thread::hardware_concurrency(); + MaxSharedMemorySize = new int[NumDevices]; + WarpSize = new int[NumDevices]; + NumSMs = new int[NumDevices]; + ArchitectureGeneration = new int[NumDevices]; + MaxNumThreadsPerSM = new int[NumDevices]; + MaxNumThreadsPerTB = new int[NumDevices]; + AvailableMemory = new size_t[NumDevices]; + SupportCooperativeGroups = new bool[NumDevices]; + DeviceNames = new std::string[NumDevices]; + + for (int d = 0; d < NumDevices; d++) { + MaxSharedMemorySize[d] = 1e6; // 4096 * 4; + WarpSize[d] = MGARDX_WARP_SIZE; + NumSMs[d] = 80; + MaxNumThreadsPerSM[d] = 1024; + MaxNumThreadsPerTB[d] = 1024; + ArchitectureGeneration[d] = 1; + SupportCooperativeGroups[d] = true; + AvailableMemory[d] = std::numeric_limits::max(); // unlimited + DeviceNames[d] = "CPU (Serial)"; + } + } + + MGARDX_CONT int GetNumDevices() { return NumDevices; } + + MGARDX_CONT int GetMaxSharedMemorySize(int dev_id) { + return MaxSharedMemorySize[dev_id]; + } + + MGARDX_CONT int GetWarpSize(int dev_id) { return WarpSize[dev_id]; } + + MGARDX_CONT int GetNumSMs(int dev_id) { return NumSMs[dev_id]; } + + MGARDX_CONT int GetArchitectureGeneration(int dev_id) { + return ArchitectureGeneration[dev_id]; + } + + MGARDX_CONT int GetMaxNumThreadsPerSM(int dev_id) { + return MaxNumThreadsPerSM[dev_id]; + } + + MGARDX_CONT int GetMaxNumThreadsPerTB(int dev_id) { + return MaxNumThreadsPerTB[dev_id]; + } + + MGARDX_CONT int GetAvailableMemory(int dev_id) { + AvailableMemory[dev_id] = 8e9; + return AvailableMemory[dev_id]; + } + + MGARDX_CONT bool SupportCG(int dev_id) { + return SupportCooperativeGroups[dev_id]; + } + + MGARDX_CONT std::string GetDeviceName(int dev_id) { + return DeviceNames[dev_id]; + } + + MGARDX_CONT + ~DeviceSpecification() { + delete[] MaxSharedMemorySize; + delete[] WarpSize; + delete[] NumSMs; + delete[] ArchitectureGeneration; + delete[] MaxNumThreadsPerSM; + delete[] MaxNumThreadsPerTB; + delete[] AvailableMemory; + delete[] SupportCooperativeGroups; + delete[] DeviceNames; + } + + int NumDevices; + int *MaxSharedMemorySize; + int *WarpSize; + int *NumSMs; + int *ArchitectureGeneration; + int *MaxNumThreadsPerSM; + int *MaxNumThreadsPerTB; + size_t *AvailableMemory; + bool *SupportCooperativeGroups; + std::string *DeviceNames; +}; + +template <> class DeviceQueues { +public: + MGARDX_CONT + void Initialize() { + log::dbg("Calling DeviceQueues::Initialize"); + initialized = true; + } + + MGARDX_CONT + void Destroy() { + log::dbg("Calling DeviceQueues::Destroy"); + initialized = false; + } + + MGARDX_CONT + DeviceQueues() { + // do nothing + } + + MGARDX_CONT int GetQueue(int dev_id, SIZE queue_id) { return 0; } + + MGARDX_CONT void SyncQueue(int dev_id, SIZE queue_id) { + // do nothing + } + + MGARDX_CONT void SyncAllQueues(int dev_id) { + // do nothing + } + + MGARDX_CONT + ~DeviceQueues() { + // do nothing + } + int initialized = false; +}; + +extern int serial_dev_id; +#pragma omp threadprivate(serial_dev_id) + +// SERIAL has no SIMD lanes: its sub-group is the size-1 scalar group, so any +// cooperative kernel degenerates to the plain serial algorithm. +template <> struct SubGroup : SubGroupScalar {}; + +template <> class DeviceRuntime { +public: + MGARDX_CONT + DeviceRuntime() {} + + MGARDX_CONT static void Initialize() { queues.Initialize(); } + + MGARDX_CONT static void Finalize() { queues.Destroy(); } + + MGARDX_CONT static int GetDeviceCount() { return DeviceSpecs.NumDevices; } + + MGARDX_CONT static void SelectDevice(SIZE dev_id) { + // do nothing + } + + MGARDX_CONT static int GetDevice() { return serial_dev_id; } + + MGARDX_CONT static int GetQueue(SIZE queue_id) { return 0; } + + MGARDX_CONT static void SyncQueue(SIZE queue_id) { + // do nothing + } + + MGARDX_CONT static void SyncAllQueues() { + // do nothing + } + + MGARDX_CONT static void SyncDevice() { + // do nothing + } + + MGARDX_CONT static std::string GetDeviceName() { + return DeviceSpecs.GetDeviceName(serial_dev_id); + } + + MGARDX_CONT static int GetMaxSharedMemorySize() { + return DeviceSpecs.GetMaxSharedMemorySize(serial_dev_id); + } + + MGARDX_CONT static int GetWarpSize() { + return DeviceSpecs.GetWarpSize(serial_dev_id); + } + + MGARDX_CONT static int GetNumSMs() { + return DeviceSpecs.GetNumSMs(serial_dev_id); + } + + MGARDX_CONT static int GetArchitectureGeneration() { + return DeviceSpecs.GetArchitectureGeneration(serial_dev_id); + } + + MGARDX_CONT static int GetMaxNumThreadsPerSM() { + return DeviceSpecs.GetMaxNumThreadsPerSM(serial_dev_id); + } + + MGARDX_CONT static int GetMaxNumThreadsPerTB() { + return DeviceSpecs.GetMaxNumThreadsPerTB(serial_dev_id); + } + + MGARDX_CONT static size_t GetAvailableMemory() { + return DeviceSpecs.GetAvailableMemory(serial_dev_id); + } + + MGARDX_CONT static bool SupportCG() { + return DeviceSpecs.SupportCG(serial_dev_id); + } + + template + MGARDX_CONT static int + GetOccupancyMaxActiveBlocksPerSM(FunctorType functor, int blockSize, + size_t dynamicSMemSize) { + return 32; + } + + template + MGARDX_CONT static void SetMaxDynamicSharedMemorySize(FunctorType functor, + int maxbytes) { + // do nothing + } + + MGARDX_CONT + ~DeviceRuntime() {} + + static DeviceQueues queues; + static bool SyncAllKernelsAndCheckErrors; + static DeviceSpecification DeviceSpecs; + static bool TimingAllKernels; + static bool PrintKernelConfig; +}; + +template <> class MemoryManager { +public: + MGARDX_CONT + MemoryManager(){}; + + template + MGARDX_CONT static void Malloc1D(T *&ptr, SIZE n, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Malloc1D"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = (T *)std::malloc(n * sizeof(converted_T)); + if (ptr == nullptr) { + throw std::runtime_error("MemoryManager::Malloc1D error."); + } + } + + template + MGARDX_CONT static void MallocND(T *&ptr, SIZE n1, SIZE n2, SIZE &ld, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MallocND"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = (T *)std::malloc(n1 * n2 * sizeof(converted_T)); + ld = n1; + if (ptr == nullptr) { + throw std::runtime_error("MemoryManager::MallocND error."); + } + } + + template + MGARDX_CONT static void + MallocManaged1D(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MallocManaged1D"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = (T *)std::malloc(n * sizeof(converted_T)); + if (ptr == nullptr) { + throw std::runtime_error("MemoryManager::MallocManaged1D error."); + } + } + + template + MGARDX_CONT static void Free(T *ptr, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Free"); + if (ptr == nullptr) + return; + std::free(ptr); + } + + template + MGARDX_CONT static void Copy1D(T *dst_ptr, const T *src_ptr, SIZE n, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Copy1D"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + std::memcpy(dst_ptr, src_ptr, sizeof(converted_T) * n); + } + + template + MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, + SIZE src_ld, SIZE n1, SIZE n2, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::CopyND"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + if ((dst_ld == n1) && (src_ld == n1)) { + // Can copy all rows at once + std::memcpy(dst_ptr, src_ptr, sizeof(converted_T) * n1 * n2); + } else { + // There is padding in between rows in memory. Must copy each row. + for (SIZE row = 0; row < n2; ++row) { + std::memcpy((converted_T *)dst_ptr + row * dst_ld, + (converted_T *)src_ptr + row * src_ld, + sizeof(converted_T) * n1); + } + } + } + + template + MGARDX_CONT static void + MallocHost(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MallocHost"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = (T *)std::malloc(n * sizeof(converted_T)); + if (ptr == nullptr) { + throw std::runtime_error("MemoryManager::MallocHost error."); + } + } + + template + MGARDX_CONT static void FreeHost(T *ptr, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::FreeHost"); + if (ptr == nullptr) + return; + std::free(ptr); + } + + template + MGARDX_CONT static void Memset1D(T *ptr, SIZE n, int value, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Memset1D"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + memset(ptr, value, n * sizeof(converted_T)); + } + + template + MGARDX_CONT static void MemsetND(T *ptr, SIZE ld, SIZE n1, SIZE n2, int value, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MemsetND"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + memset(ptr, value, n1 * n2 * sizeof(converted_T)); + } + + template MGARDX_CONT static bool IsDevicePointer(T *ptr) { + log::dbg("Calling MemoryManager::IsDevicePointer"); + return true; + } + + template MGARDX_CONT static int GetPointerDevice(T *ptr) { + log::dbg("Calling MemoryManager::GetPointerDevice"); + return 0; + } + + template MGARDX_CONT static bool CheckHostRegister(T *ptr) { + log::dbg("Calling MemoryManager::CheckHostRegister"); + return false; + } + + template MGARDX_CONT static void HostRegister(T *ptr, SIZE n) { + log::dbg("Calling MemoryManager::HostRegister"); + } + + template MGARDX_CONT static void HostUnregister(T *ptr) { + log::dbg("Calling MemoryManager::HostUnregister"); + } +}; + +#define ALIGN_LEFT 0 // for encoding +#define ALIGN_RIGHT 1 // for decoding + +#define Sign_Encoding_Atomic 0 +#define Sign_Encoding_Reduce 1 +#define Sign_Encoding_Ballot 2 + +#define Sign_Decoding_Parallel 0 + +#define BINARY 0 +#define NEGABINARY 1 + +#define Bit_Transpose_Serial_All 0 +#define Bit_Transpose_Parallel_B_Serial_b 1 +#define Bit_Transpose_Parallel_B_Atomic_b 2 +#define Bit_Transpose_Parallel_B_Reduce_b 3 +#define Bit_Transpose_Parallel_B_Ballot_b 4 +#define Bit_Transpose_TCU 5 + +#define Warp_Bit_Transpose_Serial_All 0 +#define Warp_Bit_Transpose_Parallel_B_Serial_b 1 +#define Warp_Bit_Transpose_Serial_B_Atomic_b 2 +#define Warp_Bit_Transpose_Serial_B_Reduce_b 3 +#define Warp_Bit_Transpose_Serial_B_Ballot_b 4 +#define Warp_Bit_Transpose_TCU 5 + +#define Error_Collecting_Disable -1 +#define Error_Collecting_Serial_All 0 +#define Error_Collecting_Parallel_Bitplanes_Serial_Error 1 +#define Error_Collecting_Parallel_Bitplanes_Atomic_Error 2 +#define Error_Collecting_Parallel_Bitplanes_Reduce_Error 3 + +#define Warp_Error_Collecting_Disable -1 +#define Warp_Error_Collecting_Serial_All 0 +#define Warp_Error_Collecting_Parallel_Bitplanes_Serial_Error 1 +#define Warp_Error_Collecting_Serial_Bitplanes_Atomic_Error 2 +#define Warp_Error_Collecting_Serial_Bitplanes_Reduce_Error 3 + +typedef unsigned long long int uint64_cu; + +template +struct BlockBitTranspose { + + MGARDX_EXEC + static void Serial_All(T_org *v, T_trans *tv, SIZE b, SIZE B, SIZE IdX, + SIZE IdY) { + if (IdX == 0 && IdY == 0) { + // printf("add-in: %llu %u\n", v, v[0]); + // for (int i = 0; i < b; i++) { + // printf("v: %u\n", v[i]); + // } + for (SIZE B_idx = 0; B_idx < B; B_idx++) { + T_trans buffer = 0; + for (SIZE b_idx = 0; b_idx < b; b_idx++) { + T_trans bit = (v[b_idx] >> (sizeof(T_org) * 8 - 1 - B_idx)) & 1u; + if (ALIGN == ALIGN_LEFT) { + buffer += bit << (sizeof(T_trans) * 8 - 1 - b_idx); + // if (B_idx == 0) { + // printf("%u %u %u\n", B_idx, b_idx, bit); + // print_bits(buffer, sizeof(T_trans)*8, false); + // printf("\n"); + // } + } else if (ALIGN == ALIGN_RIGHT) { + buffer += bit << (b - 1 - b_idx); + // if (b_idx == 0) printf("%u %u %u\n", B_idx, b_idx, bit); + } else { + } + // if (j == 0 ) {printf("i %u j %u shift %u bit %u\n", i,j,b-1-j, + // bit); } + } + + // printf("buffer: %u\n", buffer); + + tv[B_idx] = buffer; + } + } + } + + MGARDX_EXEC + static void Transpose(T_org *v, T_trans *tv, SIZE b, SIZE B, SIZE IdX, + SIZE IdY) { + Serial_All(v, tv, b, B, IdX, IdY); + } +}; + +template +struct BlockErrorCollect { + + MGARDX_EXEC + static void Serial_All(T *v, T_error *temp, T_error *errors, SIZE num_elems, + SIZE num_bitplanes, SIZE IdX, SIZE IdY) { + if (IdX == 0 && IdY == 0) { + for (SIZE elem_idx = 0; elem_idx < num_elems; elem_idx++) { + T data = v[elem_idx]; + T_fp fp_data = (T_fp)fabs(data); + T_sfp fps_data = (T_sfp)data; + T_fp ngb_data = Math::binary2negabinary(fps_data); + T_error mantissa; + if (BinaryType == BINARY) { + mantissa = fabs(data) - fp_data; + } else if (BinaryType == NEGABINARY) { + mantissa = data - fps_data; + } + for (SIZE bitplane_idx = 0; bitplane_idx < num_bitplanes; + bitplane_idx++) { + uint64_t mask = (1 << bitplane_idx) - 1; + T_error diff = 0; + if (BinaryType == BINARY) { + diff = (T_error)(fp_data & mask) + mantissa; + } else if (BinaryType == NEGABINARY) { + diff = (T_error)Math::negabinary2binary(ngb_data & mask) + + mantissa; + } + errors[num_bitplanes - bitplane_idx] += diff * diff; + // if (blockIdx.x == 0 && num_bitplanes-bitplane_idx == 2) { + // printf("elem error[%u]: %f\n", elem_idx, diff * diff); + // } + } + errors[0] += data * data; + } + } + } + + MGARDX_EXEC + static void Collect(T *v, T_error *temp, T_error *errors, SIZE num_elems, + SIZE num_bitplanes, SIZE IdX, SIZE IdY) { + Serial_All(v, temp, errors, num_elems, num_bitplanes, IdX, IdY); + } +}; + +template class DeviceAdapter { +public: + MGARDX_CONT + DeviceAdapter(){}; + + MGARDX_CONT + int IsResourceEnough(TaskType &task) { + if (task.GetBlockDimX() * task.GetBlockDimY() * task.GetBlockDimZ() > + DeviceRuntime::GetMaxNumThreadsPerTB()) { + return THREADBLOCK_TOO_LARGE; + } + if (task.GetSharedMemorySize() > + DeviceRuntime::GetMaxSharedMemorySize()) { + return SHARED_MEMORY_TOO_LARGE; + } + return RESOURCE_ENOUGH; + } + + MGARDX_CONT + ExecutionReturn Execute(TaskType &task) { + + if (DeviceRuntime::PrintKernelConfig) { + std::cout << log::log_info << task.GetFunctorName() << ": <" + << task.GetBlockDimX() << ", " << task.GetBlockDimY() << ", " + << task.GetBlockDimZ() << "> <" << task.GetGridDimX() << ", " + << task.GetGridDimY() << ", " << task.GetGridDimZ() << ">\n"; + } + + ExecutionReturn ret; + if (IsResourceEnough(task) != RESOURCE_ENOUGH) { + if (DeviceRuntime::PrintKernelConfig) { + if (IsResourceEnough(task) == THREADBLOCK_TOO_LARGE) { + log::info("threadblock too large."); + } + if (IsResourceEnough(task) == SHARED_MEMORY_TOO_LARGE) { + log::info("shared memory too large."); + } + } + ret.success = false; + ret.execution_time = std::numeric_limits::max(); + return ret; + } + + Timer timer; + if (DeviceRuntime::TimingAllKernels || + AutoTuner::ProfileKernels) { + DeviceRuntime::SyncDevice(); + timer.start(); + } + // if constexpr evalute at compile time otherwise this does not compile + if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + SerialKernel(task); + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + SerialIterKernel(task); + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + SerialHuffmanCLCustomizedKernel(task); + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + SerialHuffmanCWCustomizedKernel(task); + } + // timer.end(); + // timer.print(task.GetFunctorName()); + // timer.clear(); + if (DeviceRuntime::TimingAllKernels || + AutoTuner::ProfileKernels) { + DeviceRuntime::SyncDevice(); + timer.end(); + if (DeviceRuntime::TimingAllKernels) { + timer.print(task.GetFunctorName()); + } + if (AutoTuner::ProfileKernels) { + ret.success = true; + ret.execution_time = timer.get(); + } + } + return ret; + } +}; + +template <> class DeviceLauncher { +public: + template + MGARDX_CONT int static IsResourceEnough(TaskType &task) { + if (task.GetBlockDimX() * task.GetBlockDimY() * task.GetBlockDimZ() > + DeviceRuntime::GetMaxNumThreadsPerTB()) { + return THREADBLOCK_TOO_LARGE; + } + if (task.GetSharedMemorySize() > + DeviceRuntime::GetMaxSharedMemorySize()) { + return SHARED_MEMORY_TOO_LARGE; + } + return RESOURCE_ENOUGH; + } + + template + MGARDX_CONT ExecutionReturn static Execute(TaskType &task) { + + if (DeviceRuntime::PrintKernelConfig) { + std::cout << log::log_info << task.GetFunctorName() << ": <" + << task.GetBlockDimX() << ", " << task.GetBlockDimY() << ", " + << task.GetBlockDimZ() << "> <" << task.GetGridDimX() << ", " + << task.GetGridDimY() << ", " << task.GetGridDimZ() << ">\n"; + } + + ExecutionReturn ret; + if (IsResourceEnough(task) != RESOURCE_ENOUGH) { + if (DeviceRuntime::PrintKernelConfig) { + if (IsResourceEnough(task) == THREADBLOCK_TOO_LARGE) { + log::info("threadblock too large."); + } + if (IsResourceEnough(task) == SHARED_MEMORY_TOO_LARGE) { + log::info("shared memory too large."); + } + } + ret.success = false; + ret.execution_time = std::numeric_limits::max(); + return ret; + } + + Timer timer; + if (DeviceRuntime::TimingAllKernels || + AutoTuner::ProfileKernels) { + DeviceRuntime::SyncDevice(); + timer.start(); + } + // if constexpr evalute at compile time otherwise this does not compile + if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + SerialKernel(task); + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + SerialIterKernel(task); + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + SerialHuffmanCLCustomizedKernel(task); + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + SerialHuffmanCWCustomizedKernel(task); + } + // timer.end(); + // timer.print(task.GetFunctorName()); + // timer.clear(); + if (DeviceRuntime::TimingAllKernels || + AutoTuner::ProfileKernels) { + DeviceRuntime::SyncDevice(); + timer.end(); + if (DeviceRuntime::TimingAllKernels) { + timer.print(task.GetFunctorName()); + } + if (AutoTuner::ProfileKernels) { + ret.success = true; + ret.execution_time = timer.get(); + } + } + return ret; + } + + template + MGARDX_CONT static void ConfigTask(TaskType task) { + typename TaskType::Functor functor; + int maxbytes = DeviceRuntime::GetMaxSharedMemorySize(); + DeviceRuntime::SetMaxDynamicSharedMemorySize(functor, maxbytes); + } + + template + MGARDX_CONT static void AutoTune(KernelType kernel, int queue_idx) { +#if MGARD_ENABLE_AUTO_TUNING + double min_time = std::numeric_limits::max(); + int min_config = 0; + ExecutionReturn ret; +#define RUN_CONFIG(CONFIG_IDX) \ + { \ + constexpr ExecutionConfig config = \ + GetExecutionConfig(CONFIG_IDX); \ + auto task = \ + kernel.template GenTask(queue_idx); \ + if constexpr (KernelType::EnableConfig()) { \ + ConfigTask(task); \ + } \ + ret = Execute(task); \ + if (ret.success && min_time > ret.execution_time) { \ + min_time = ret.execution_time; \ + min_config = CONFIG_IDX; \ + } \ + } + RUN_CONFIG(0) + RUN_CONFIG(1) + RUN_CONFIG(2) + RUN_CONFIG(3) + RUN_CONFIG(4) + RUN_CONFIG(5) + RUN_CONFIG(6) +#undef RUN_CONFIG + if (AutoTuner::WriteToTable) { + FillAutoTunerTable(std::string(KernelType::Name), min_config); + } +#else + throw std::runtime_error("MGARD is not built with auto tuning enabled."); +#endif + } + + template + MGARDX_CONT static void Execute(KernelType kernel, int queue_idx) { + if constexpr (KernelType::EnableAutoTuning()) { + constexpr ExecutionConfig config = + GetExecutionConfig(KernelType::Name); + auto task = + kernel.template GenTask(queue_idx); + if constexpr (KernelType::EnableConfig()) { + ConfigTask(task); + } + Execute(task); + + if (AutoTuner::ProfileKernels) { + AutoTune(kernel, queue_idx); + } + } else { + auto task = kernel.GenTask(queue_idx); + if constexpr (KernelType::EnableConfig()) { + ConfigTask(task); + } + Execute(task); + } + } +}; + +template <> class DeviceCollective { +public: + MGARDX_CONT + DeviceCollective(){}; + + template + MGARDX_CONT static void Sum(SIZE n, SubArray<1, T, SERIAL> v, + SubArray<1, T, SERIAL> result, + Array<1, Byte, SERIAL> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + *result((IDX)0) = std::accumulate(v((IDX)0), v((IDX)n), (T)0); + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void AbsMax(SIZE n, SubArray<1, T, SERIAL> v, + SubArray<1, T, SERIAL> result, + Array<1, Byte, SERIAL> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + T max_result = 0; + for (SIZE i = 0; i < n; ++i) { + max_result = std::max((T)fabs(*v(i)), max_result); + } + *result((IDX)0) = max_result; + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void AbsMin(SIZE n, SubArray<1, T, SERIAL> v, + SubArray<1, T, SERIAL> result, + Array<1, Byte, SERIAL> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + T max_result = std::numeric_limits::max(); + for (SIZE i = 0; i < n; ++i) { + max_result = std::min((T)fabs(*v(i)), max_result); + } + *result((IDX)0) = max_result; + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void SquareSum(SIZE n, SubArray<1, T, SERIAL> v, + SubArray<1, T, SERIAL> result, + Array<1, Byte, SERIAL> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + T sum_result = 0; + for (SIZE i = 0; i < n; ++i) { + T tmp = *v(i); + sum_result += tmp * tmp; + } + *result((IDX)0) = sum_result; + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void ScanSumInclusive(SIZE n, SubArray<1, T, SERIAL> v, + SubArray<1, T, SERIAL> result, + Array<1, Byte, SERIAL> &workspace, + bool workspace_allocated, + int queue_idx) { + + if (workspace_allocated) { + // Serial inclusive scan: result[i] = sum(v[0..i]). Hand-rolled instead of + // std::inclusive_scan so the header parses under nvcc's default host + // compiler (older libstdc++ hides the C++17 algorithms). + T acc = (T)0; + for (SIZE i = 0; i < n; i++) { + acc += *v((IDX)i); + *result((IDX)i) = acc; + } + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void ScanSumExclusive(SIZE n, SubArray<1, T, SERIAL> v, + SubArray<1, T, SERIAL> result, + Array<1, Byte, SERIAL> &workspace, + bool workspace_allocated, + int queue_idx) { + + if (workspace_allocated) { + // Serial exclusive scan: result[0] = 0, result[i] = sum(v[0..i-1]). + // Hand-rolled in place of std::exclusive_scan (see ScanSumInclusive). + T acc = (T)0; + for (SIZE i = 0; i < n; i++) { + *result((IDX)i) = acc; + acc += *v((IDX)i); + } + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void ScanSumExtended(SIZE n, SubArray<1, T, SERIAL> v, + SubArray<1, T, SERIAL> result, + Array<1, Byte, SERIAL> &workspace, + bool workspace_allocated, + int queue_idx) { + + if (workspace_allocated) { + // Serial extended scan: result has n+1 entries with result[0] = 0 and + // result[i+1] = sum(v[0..i]) (exclusive prefix plus the grand total in + // the last slot). Hand-rolled in place of std::inclusive_scan into + // result+1. + T acc = (T)0; + *result((IDX)0) = (T)0; + for (SIZE i = 0; i < n; i++) { + acc += *v((IDX)i); + *result((IDX)(i + 1)) = acc; + } + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void SortByKey(SIZE n, SubArray<1, KeyT, SERIAL> in_keys, + SubArray<1, ValueT, SERIAL> in_values, + SubArray<1, KeyT, SERIAL> out_keys, + SubArray<1, ValueT, SERIAL> out_values, + Array<1, Byte, SERIAL> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + std::vector> data(n); + for (SIZE i = 0; i < n; ++i) { + data[i] = std::pair(*in_keys(i), *in_values(i)); + } + std::stable_sort(data.begin(), data.end(), + KeyValueComparator{}); + for (SIZE i = 0; i < n; ++i) { + *out_keys(i) = data[i].first; + *out_values(i) = data[i].second; + } + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } +}; + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h b/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h new file mode 100644 index 000000000..efe2de9a6 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h @@ -0,0 +1,2000 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "DeviceAdapter.h" +#include + +#ifndef MGARD_X_160_DEVICE_ADAPTER_SYCL_H +#define MGARD_X_160_DEVICE_ADAPTER_SYCL_H + +#define MGARD_X_160_SYCL_ND_RANGE_3D 0 +#define MGARD_X_160_SYCL_ND_RANGE_1D 1 + +namespace mgard_x_160 { + +using LocalMemory = sycl::accessor; + +// Create an exception sycl::handler for asynchronous SYCL exceptions +static auto exception_handler = [](sycl::exception_list e_list) { + for (std::exception_ptr const &e : e_list) { + try { + std::rethrow_exception(e); + } catch (std::exception const &e) { + std::cout << "Failure" << std::endl; + std::terminate(); + } + } +}; + +template +struct Atomic { + MGARDX_EXEC static T Min(T *result, T value) { + if constexpr (MemoryType == AtomicGlobalMemory) { + if constexpr (Scope == AtomicSystemScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::system, + sycl::access::address_space::global_space>; + return AtomicRef(result[0]).fetch_min(value); + } else if constexpr (Scope == AtomicDeviceScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::device, + sycl::access::address_space::global_space>; + return AtomicRef(result[0]).fetch_min(value); + } else { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::work_group, + sycl::access::address_space::global_space>; + return AtomicRef(result[0]).fetch_min(value); + } + } else { + if constexpr (Scope == AtomicSystemScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::system, + sycl::access::address_space::local_space>; + return AtomicRef(result[0]).fetch_min(value); + } else if constexpr (Scope == AtomicDeviceScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::device, + sycl::access::address_space::local_space>; + return AtomicRef(result[0]).fetch_min(value); + } else { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::work_group, + sycl::access::address_space::local_space>; + return AtomicRef(result[0]).fetch_min(value); + } + } + } + MGARDX_EXEC static T Max(T *result, T value) { + if constexpr (MemoryType == AtomicGlobalMemory) { + if constexpr (Scope == AtomicSystemScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::system, + sycl::access::address_space::global_space>; + return AtomicRef(result[0]).fetch_max(value); + } else if constexpr (Scope == AtomicDeviceScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::device, + sycl::access::address_space::global_space>; + return AtomicRef(result[0]).fetch_max(value); + } else { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::work_group, + sycl::access::address_space::global_space>; + return AtomicRef(result[0]).fetch_max(value); + } + } else { + if constexpr (Scope == AtomicSystemScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::system, + sycl::access::address_space::local_space>; + return AtomicRef(result[0]).fetch_max(value); + } else if constexpr (Scope == AtomicDeviceScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::device, + sycl::access::address_space::local_space>; + return AtomicRef(result[0]).fetch_max(value); + } else { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::work_group, + sycl::access::address_space::local_space>; + return AtomicRef(result[0]).fetch_max(value); + } + } + } + MGARDX_EXEC static T Add(T *result, T value) { + if constexpr (MemoryType == AtomicGlobalMemory) { + if constexpr (Scope == AtomicSystemScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::system, + sycl::access::address_space::global_space>; + T new_value = AtomicRef(result[0]) += value; + return new_value - value; + } else if constexpr (Scope == AtomicDeviceScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::device, + sycl::access::address_space::global_space>; + T new_value = AtomicRef(result[0]) += value; + return new_value - value; + } else { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::work_group, + sycl::access::address_space::global_space>; + T new_value = AtomicRef(result[0]) += value; + return new_value - value; + } + } else { + if constexpr (Scope == AtomicSystemScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::system, + sycl::access::address_space::local_space>; + T new_value = AtomicRef(result[0]) += value; + return new_value - value; + } else if constexpr (Scope == AtomicDeviceScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::device, + sycl::access::address_space::local_space>; + T new_value = AtomicRef(result[0]) += value; + return new_value - value; + } else { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::work_group, + sycl::access::address_space::local_space>; + T new_value = AtomicRef(result[0]) += value; + return new_value - value; + } + } + } + MGARDX_EXEC static T Or(T *result, T value) { + if constexpr (MemoryType == AtomicGlobalMemory) { + if constexpr (Scope == AtomicSystemScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::system, + sycl::access::address_space::global_space>; + return AtomicRef(result[0]).fetch_or(value); + } else if constexpr (Scope == AtomicDeviceScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::device, + sycl::access::address_space::global_space>; + return AtomicRef(result[0]).fetch_or(value); + } else { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::work_group, + sycl::access::address_space::global_space>; + return AtomicRef(result[0]).fetch_or(value); + } + } else { + if constexpr (Scope == AtomicSystemScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::system, + sycl::access::address_space::local_space>; + return AtomicRef(result[0]).fetch_or(value); + } else if constexpr (Scope == AtomicDeviceScope) { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::device, + sycl::access::address_space::local_space>; + return AtomicRef(result[0]).fetch_or(value); + } else { + using AtomicRef = sycl::ext::oneapi::atomic_ref< + T, sycl::memory_order::relaxed, sycl::memory_scope::work_group, + sycl::access::address_space::local_space>; + return AtomicRef(result[0]).fetch_or(value); + } + } + } +}; + +template <> struct Math { + template MGARDX_EXEC static T Min(T a, T b) { + if constexpr (std::is_integral::value) + return sycl::min(a, b); + else { + return sycl::fmin(a, b); + } + } + template MGARDX_EXEC static T Max(T a, T b) { + if constexpr (std::is_integral::value) + return sycl::max(a, b); + else { + return sycl::fmax(a, b); + } + } + MGARDX_EXEC static int ffs(unsigned int a) { + int pos = 0; + if (a == 0) + return pos; + while (!(a & 1)) { + a >>= 1; + ++pos; + } + return pos + 1; + } + MGARDX_EXEC static int ffsll(long long unsigned int a) { + int pos = 0; + if (a == 0) + return pos; + while (!(a & 1)) { + a >>= 1; + ++pos; + } + return pos + 1; + } + MGARDX_EXEC + static uint64_t binary2negabinary(const int64_t x) { + return (x + (uint64_t)0xaaaaaaaaaaaaaaaaull) ^ + (uint64_t)0xaaaaaaaaaaaaaaaaull; + } + + MGARDX_EXEC + static uint32_t binary2negabinary(const int32_t x) { + return (x + (uint32_t)0xaaaaaaaau) ^ (uint32_t)0xaaaaaaaau; + } + + MGARDX_EXEC + static int64_t negabinary2binary(const uint64_t x) { + return (x ^ 0xaaaaaaaaaaaaaaaaull) - 0xaaaaaaaaaaaaaaaaull; + } + + MGARDX_EXEC + static int32_t negabinary2binary(const uint32_t x) { + return (x ^ 0xaaaaaaaau) - 0xaaaaaaaau; + } +}; + +template <> class DeviceSpecification { +public: + MGARDX_CONT + DeviceSpecification() { + sycl::default_selector d_selector; + sycl::platform d_platform(d_selector); + std::vector d_devices = d_platform.get_devices(); + NumDevices = d_devices.size(); + MaxSharedMemorySize = new int[NumDevices]; + WarpSize = new int[NumDevices]; + NumSMs = new int[NumDevices]; + ArchitectureGeneration = new int[NumDevices]; + MaxNumThreadsPerSM = new int[NumDevices]; + MaxNumThreadsPerTB = new int[NumDevices]; + AvailableMemory = new size_t[NumDevices]; + SupportCooperativeGroups = new bool[NumDevices]; + DeviceNames = new std::string[NumDevices]; + + int d = 0; + for (auto &device : d_devices) { + MaxSharedMemorySize[d] = + device.get_info(); + WarpSize[d] = MGARDX_WARP_SIZE; + NumSMs[d] = device.get_info(); + MaxNumThreadsPerSM[d] = + device.get_info(); + // Larger limit can cause resource insufficient error + MaxNumThreadsPerTB[d] = std::min( + 1024ul, device.get_info()); + AvailableMemory[d] = + device.get_info(); + SupportCooperativeGroups[d] = false; + DeviceNames[d] = std::string(device.get_info()); + d++; + } + } + + MGARDX_CONT int GetNumDevices() { return NumDevices; } + + MGARDX_CONT int GetMaxSharedMemorySize(int dev_id) { + return MaxSharedMemorySize[dev_id]; + } + + MGARDX_CONT int GetWarpSize(int dev_id) { return WarpSize[dev_id]; } + + MGARDX_CONT int GetNumSMs(int dev_id) { return NumSMs[dev_id]; } + + MGARDX_CONT int GetArchitectureGeneration(int dev_id) { + return ArchitectureGeneration[dev_id]; + } + + MGARDX_CONT int GetMaxNumThreadsPerSM(int dev_id) { + return MaxNumThreadsPerSM[dev_id]; + } + + MGARDX_CONT int GetMaxNumThreadsPerTB(int dev_id) { + return MaxNumThreadsPerTB[dev_id]; + } + + MGARDX_CONT size_t GetAvailableMemory(int dev_id) { + return AvailableMemory[dev_id]; + } + + MGARDX_CONT bool SupportCG(int dev_id) { + return SupportCooperativeGroups[dev_id]; + } + + MGARDX_CONT std::string GetDeviceName(int dev_id) { + return DeviceNames[dev_id]; + } + + MGARDX_CONT + ~DeviceSpecification() { + delete[] MaxSharedMemorySize; + delete[] WarpSize; + delete[] NumSMs; + delete[] ArchitectureGeneration; + delete[] MaxNumThreadsPerSM; + delete[] MaxNumThreadsPerTB; + delete[] AvailableMemory; + delete[] SupportCooperativeGroups; + delete[] DeviceNames; + } + + int NumDevices; + int *MaxSharedMemorySize; + int *WarpSize; + int *NumSMs; + int *ArchitectureGeneration; + int *MaxNumThreadsPerSM; + int *MaxNumThreadsPerTB; + size_t *AvailableMemory; + bool *SupportCooperativeGroups; + std::string *DeviceNames; +}; + +template <> class DeviceQueues { +public: + MGARDX_CONT + void Initialize() { + if (!initialized) { + log::dbg("Calling DeviceQueues::Initialize"); + sycl::default_selector d_selector; + sycl::platform d_platform(d_selector); + std::vector d_devices = d_platform.get_devices(); + NumDevices = d_devices.size(); + queues = new sycl::queue *[NumDevices]; + for (SIZE d = 0; d < NumDevices; d++) { + queues[d] = new sycl::queue[MGARDX_NUM_QUEUES]; + for (SIZE i = 0; i < MGARDX_NUM_QUEUES; i++) { + queues[d][i] = sycl::queue(d_devices[d], exception_handler); + } + } + initialized = true; + } + } + + MGARDX_CONT + void Destroy() { + if (initialized) { + log::dbg("Calling DeviceQueues::Destroy"); + for (SIZE d = 0; d < NumDevices; d++) { + delete[] queues[d]; + } + delete[] queues; + queues = nullptr; + initialized = false; + } + } + + MGARDX_CONT + DeviceQueues() { Initialize(); } + + MGARDX_CONT sycl::queue GetQueue(int dev_id, SIZE queue_id) { + Initialize(); + return queues[dev_id][queue_id]; + } + + MGARDX_CONT void SyncQueue(int dev_id, SIZE queue_id) { + Initialize(); + queues[dev_id][queue_id].wait(); + } + + MGARDX_CONT void SyncAllQueues(int dev_id) { + for (SIZE i = 0; i < MGARDX_NUM_QUEUES; i++) { + Initialize(); + queues[dev_id][i].wait(); + } + } + + MGARDX_CONT + ~DeviceQueues() {} + + int initialized = false; + int NumDevices; + sycl::queue **queues = nullptr; +}; + +extern int sycl_dev_id; +#pragma omp threadprivate(sycl_dev_id) + +// Intel sub-group (oneAPI/DPC++). UNTESTED -- no Intel GPU / SYCL toolchain +// here. Holds the native sycl::sub_group (constructed from nd_item in the +// kernel); width pinned to 32 by [[sycl::reqd_sub_group_size(32)]] at the +// launch site so the ballot mask fits a uint32. See verification notes at the +// original definition (group_ballot/extract_bits, select_from_group, +// group_barrier). +template <> struct SubGroup { + sycl::sub_group sg_; + using mask_t = uint32_t; + MGARDX_EXEC SubGroup(sycl::sub_group s) : sg_(s) {} + static constexpr int size() { return 32; } + MGARDX_EXEC int lane() const { return (int)sg_.get_local_linear_id(); } + MGARDX_EXEC mask_t full_mask() const { return 0xffffffffu; } + template MGARDX_EXEC T shfl(T v, int src) const { + return sycl::select_from_group(sg_, v, (size_t)src); + } + MGARDX_EXEC mask_t ballot(int pred) const { + auto m = sycl::ext::oneapi::group_ballot(sg_, pred != 0); + uint32_t bits = 0; + m.extract_bits(bits); + return bits; + } + MGARDX_EXEC int ffs(mask_t m) const { return __builtin_ffs((int)m); } + MGARDX_EXEC void sync() const { sycl::group_barrier(sg_); } +}; + +template <> class DeviceRuntime { +public: + MGARDX_CONT + DeviceRuntime() {} + + MGARDX_CONT static void Initialize() { queues.Initialize(); } + + MGARDX_CONT static void Finalize() { queues.Destroy(); } + + MGARDX_CONT static int GetDeviceCount() { return DeviceSpecs.NumDevices; } + + MGARDX_CONT static void SelectDevice(SIZE dev_id) { sycl_dev_id = dev_id; } + + MGARDX_CONT static int GetDevice() { return sycl_dev_id; } + + MGARDX_CONT static sycl::queue GetQueue(SIZE queue_id) { + return queues.GetQueue(sycl_dev_id, queue_id); + } + + MGARDX_CONT static void SyncQueue(SIZE queue_id) { + log::dbg("Calling DeviceRuntime::SyncQueue: " + + std::to_string(queue_id)); + queues.SyncQueue(sycl_dev_id, queue_id); + } + + MGARDX_CONT static void SyncAllQueues() { queues.SyncAllQueues(sycl_dev_id); } + + MGARDX_CONT static void SyncDevice() { + log::dbg("Calling DeviceRuntime::SyncDevice: "); + queues.SyncAllQueues(sycl_dev_id); + } + + MGARDX_CONT static std::string GetDeviceName() { + return DeviceSpecs.GetDeviceName(sycl_dev_id); + } + + MGARDX_CONT static int GetMaxSharedMemorySize() { + return DeviceSpecs.GetMaxSharedMemorySize(sycl_dev_id); + } + + MGARDX_CONT static int GetWarpSize() { + return DeviceSpecs.GetWarpSize(sycl_dev_id); + } + + MGARDX_CONT static int GetNumSMs() { + return DeviceSpecs.GetNumSMs(sycl_dev_id); + } + + MGARDX_CONT static int GetArchitectureGeneration() { + return DeviceSpecs.GetArchitectureGeneration(sycl_dev_id); + } + + MGARDX_CONT static int GetMaxNumThreadsPerSM() { + return DeviceSpecs.GetMaxNumThreadsPerSM(sycl_dev_id); + } + + MGARDX_CONT static int GetMaxNumThreadsPerTB() { + return DeviceSpecs.GetMaxNumThreadsPerTB(sycl_dev_id); + } + + MGARDX_CONT static size_t GetAvailableMemory() { + return DeviceSpecs.GetAvailableMemory(sycl_dev_id); + } + + MGARDX_CONT static bool SupportCG() { + return DeviceSpecs.SupportCG(sycl_dev_id); + } + + template + MGARDX_CONT static int + GetOccupancyMaxActiveBlocksPerSM(FunctorType functor, int blockSize, + size_t dynamicSMemSize) { + return 32; + } + + template + MGARDX_CONT static void SetMaxDynamicSharedMemorySize(FunctorType functor, + int maxbytes) { + // do nothing + } + + MGARDX_CONT + ~DeviceRuntime() {} + + static DeviceQueues queues; + static bool SyncAllKernelsAndCheckErrors; + static DeviceSpecification DeviceSpecs; + static bool TimingAllKernels; + static bool PrintKernelConfig; +}; + +template <> class MemoryManager { +public: + MGARDX_CONT + MemoryManager(){}; + + template + MGARDX_CONT static void Malloc1D(T *&ptr, SIZE n, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Malloc1D"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + sycl::queue q = DeviceRuntime::GetQueue(queue_idx); + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = malloc_device(n, q); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void MallocND(T *&ptr, SIZE n1, SIZE n2, SIZE &ld, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MallocND"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + sycl::queue q = DeviceRuntime::GetQueue(queue_idx); + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = malloc_device(n1 * n2, q); + ld = n1; + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void + MallocManaged1D(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MallocManaged1D"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + sycl::queue q = DeviceRuntime::GetQueue(queue_idx); + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = malloc_shared(n, q); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void Free(T *ptr, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Free"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + sycl::queue q = DeviceRuntime::GetQueue(queue_idx); + if (ptr == nullptr) + return; + sycl::free(ptr, q); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void Copy1D(T *dst_ptr, const T *src_ptr, SIZE n, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Copy1D"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + sycl::queue q = DeviceRuntime::GetQueue(queue_idx); + using converted_T = + typename std::conditional::value, Byte, T>::type; + q.memcpy((converted_T *)dst_ptr, (converted_T *)src_ptr, + n * sizeof(converted_T)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, + SIZE src_ld, SIZE n1, SIZE n2, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::CopyND"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + sycl::queue q = DeviceRuntime::GetQueue(queue_idx); + using converted_T = + typename std::conditional::value, Byte, T>::type; + q.memcpy((converted_T *)dst_ptr, (converted_T *)src_ptr, + n1 * n2 * sizeof(converted_T)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void + MallocHost(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MallocHost"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + sycl::queue q = DeviceRuntime::GetQueue(queue_idx); + using converted_T = + typename std::conditional::value, Byte, T>::type; + ptr = malloc_host(n, q); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void FreeHost(T *ptr, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::FreeHost"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + sycl::queue q = DeviceRuntime::GetQueue(queue_idx); + if (ptr == nullptr) + return; + sycl::free(ptr, q); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void Memset1D(T *ptr, SIZE n, int value, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Memset1D"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + sycl::queue q = DeviceRuntime::GetQueue(queue_idx); + using converted_T = + typename std::conditional::value, Byte, T>::type; + q.memset((converted_T *)ptr, value, n * sizeof(converted_T)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template + MGARDX_CONT static void MemsetND(T *ptr, SIZE ld, SIZE n1, SIZE n2, int value, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::MemsetND"); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + sycl::queue q = DeviceRuntime::GetQueue(queue_idx); + using converted_T = + typename std::conditional::value, Byte, T>::type; + q.memset((converted_T *)ptr, value, n1 * n2 * sizeof(converted_T)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + + template MGARDX_CONT static bool IsDevicePointer(T *ptr) { + log::dbg("Calling MemoryManager::IsDevicePointer"); + for (int i = 0; i < DeviceRuntime::GetDeviceCount(); i++) { + DeviceRuntime::SelectDevice(i); + for (int j = 0; j < MGARDX_NUM_QUEUES; j++) { + sycl::queue q = DeviceRuntime::GetQueue(j); + if (sycl::get_pointer_type(ptr, q.get_context()) == + sycl::usm::alloc::device) { + return true; + } + } + } + return false; + } + + template MGARDX_CONT static int GetPointerDevice(T *ptr) { + log::dbg("Calling MemoryManager::GetPointerDevice"); + sycl::default_selector d_selector; + sycl::platform d_platform(d_selector); + std::vector d_devices = d_platform.get_devices(); + for (int i = 0; i < DeviceRuntime::GetDeviceCount(); i++) { + DeviceRuntime::SelectDevice(i); + for (int j = 0; j < MGARDX_NUM_QUEUES; j++) { + sycl::queue q = DeviceRuntime::GetQueue(j); + sycl::device ptr_device = + sycl::get_pointer_device(ptr, q.get_context()); + if (d_devices[i] == ptr_device) { + return i; + } + } + } + return 0; + } + + template MGARDX_CONT static bool CheckHostRegister(T *ptr) { + log::dbg("Calling MemoryManager::CheckHostRegister"); + return false; + } + + template MGARDX_CONT static void HostRegister(T *ptr, SIZE n) { + log::dbg("Calling MemoryManager::HostRegister"); + } + + template MGARDX_CONT static void HostUnregister(T *ptr) { + log::dbg("Calling MemoryManager::HostUnregister"); + } +}; + +template class SyclKernel { +public: + SyclKernel(FunctorType functor, LocalMemory localAccess, THREAD_IDX ngridz, + THREAD_IDX ngridy, THREAD_IDX ngridx, THREAD_IDX nblockz, + THREAD_IDX nblocky, THREAD_IDX nblockx) + : functor(functor), localAccess(localAccess), ngridz(ngridz), + ngridy(ngridy), ngridx(ngridx), nblockz(nblockz), nblocky(nblocky), + nblockx(nblockx) {} + void operator()(sycl::nd_item<3> i) const { + FunctorType my_functor = functor; + sycl::local_ptr l_ptr = localAccess.get_pointer(); + Byte *shared_memory = l_ptr.get(); +#if MGARD_X_160_SYCL_ND_RANGE_3D + my_functor.Init( + i.get_group_range(2), i.get_group_range(1), i.get_group_range(0), + i.get_global_range(2) / i.get_group_range(2), + i.get_global_range(1) / i.get_group_range(1), + i.get_global_range(0) / i.get_group_range(0), i.get_group().get_id(2), + i.get_group().get_id(1), i.get_group().get_id(0), i.get_local_id(2), + i.get_local_id(1), i.get_local_id(0), shared_memory); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + my_functor.Init(ngridz, ngridy, ngridx, nblockz, nblocky, nblockx, + (i.get_group().get_group_id(0) / ngridx) / ngridy, + (i.get_group().get_group_id(0) / ngridx) % ngridy, + i.get_group().get_group_id(0) % ngridx, + (i.get_local_id(0) / nblockx) / nblocky, + (i.get_local_id(0) / nblockx) % nblocky, + i.get_local_id(0) % nblockx, shared_memory); +#endif + my_functor.Operation1(); + i.barrier(); + my_functor.Operation2(); + i.barrier(); + my_functor.Operation3(); + i.barrier(); + my_functor.Operation4(); + i.barrier(); + my_functor.Operation5(); + i.barrier(); + my_functor.Operation6(); + i.barrier(); + my_functor.Operation7(); + i.barrier(); + my_functor.Operation8(); + i.barrier(); + my_functor.Operation9(); + i.barrier(); + my_functor.Operation10(); + i.barrier(); + my_functor.Operation11(); + i.barrier(); + my_functor.Operation12(); + i.barrier(); + my_functor.Operation13(); + i.barrier(); + my_functor.Operation14(); + i.barrier(); + my_functor.Operation15(); + i.barrier(); + my_functor.Operation16(); + i.barrier(); + my_functor.Operation17(); + i.barrier(); + my_functor.Operation18(); + i.barrier(); + my_functor.Operation19(); + i.barrier(); + my_functor.Operation20(); + } + +private: + FunctorType functor; + LocalMemory localAccess; + THREAD_IDX ngridz; + THREAD_IDX ngridy; + THREAD_IDX ngridx; + THREAD_IDX nblockz; + THREAD_IDX nblocky; + THREAD_IDX nblockx; +}; + +template class SyclIterKernel { +public: + SyclIterKernel(FunctorType functor, LocalMemory localAccess, + THREAD_IDX ngridz, THREAD_IDX ngridy, THREAD_IDX ngridx, + THREAD_IDX nblockz, THREAD_IDX nblocky, THREAD_IDX nblockx) + : functor(functor), localAccess(localAccess), ngridz(ngridz), + ngridy(ngridy), ngridx(ngridx), nblockz(nblockz), nblocky(nblocky), + nblockx(nblockx) {} + void operator()(sycl::nd_item<3> i) const { + FunctorType my_functor = functor; + Byte *shared_memory = localAccess.get_pointer().get(); +#if MGARD_X_160_SYCL_ND_RANGE_3D + my_functor.Init( + i.get_group_range(2), i.get_group_range(1), i.get_group_range(0), + i.get_global_range(2) / i.get_group_range(2), + i.get_global_range(1) / i.get_group_range(1), + i.get_global_range(0) / i.get_group_range(0), i.get_group().get_id(2), + i.get_group().get_id(1), i.get_group().get_id(0), i.get_local_id(2), + i.get_local_id(1), i.get_local_id(0), shared_memory); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + my_functor.Init(ngridz, ngridy, ngridx, nblockz, nblocky, nblockx, + (i.get_group().get_group_id(0) / ngridx) / ngridy, + (i.get_group().get_group_id(0) / ngridx) % ngridy, + i.get_group().get_group_id(0) % ngridx, + (i.get_local_id(0) / nblockx) / nblocky, + (i.get_local_id(0) / nblockx) % nblocky, + i.get_local_id(0) % nblockx, shared_memory); +#endif + + my_functor.Operation1(); + i.barrier(); + + my_functor.Operation2(); + i.barrier(); + + while (my_functor.LoopCondition1()) { + my_functor.Operation3(); + i.barrier(); + my_functor.Operation4(); + i.barrier(); + my_functor.Operation5(); + i.barrier(); + my_functor.Operation6(); + i.barrier(); + } + + my_functor.Operation7(); + i.barrier(); + my_functor.Operation8(); + i.barrier(); + my_functor.Operation9(); + i.barrier(); + my_functor.Operation10(); + i.barrier(); + + while (my_functor.LoopCondition2()) { + my_functor.Operation11(); + i.barrier(); + my_functor.Operation12(); + i.barrier(); + my_functor.Operation13(); + i.barrier(); + my_functor.Operation14(); + i.barrier(); + } + + my_functor.Operation15(); + i.barrier(); + my_functor.Operation16(); + i.barrier(); + my_functor.Operation17(); + i.barrier(); + } + +private: + FunctorType functor; + LocalMemory localAccess; + THREAD_IDX ngridz; + THREAD_IDX ngridy; + THREAD_IDX ngridx; + THREAD_IDX nblockz; + THREAD_IDX nblocky; + THREAD_IDX nblockx; +}; + +#define SINGLE_KERNEL_3D(OPERATION) \ + template class Single_##OPERATION##_Kernel_3D { \ + public: \ + Single_##OPERATION##_Kernel_3D(FunctorType functor, \ + LocalMemory localAccess) \ + : functor(functor), localAccess(localAccess) {} \ + void operator()(sycl::nd_item<3> i) const { \ + FunctorType my_functor = functor; \ + Byte *shared_memory = localAccess.get_pointer().get(); \ + my_functor.Init(i.get_group_range(2), i.get_group_range(1), \ + i.get_group_range(0), \ + i.get_global_range(2) / i.get_group_range(2), \ + i.get_global_range(1) / i.get_group_range(1), \ + i.get_global_range(0) / i.get_group_range(0), \ + i.get_group().get_id(2), i.get_group().get_id(1), \ + i.get_group().get_id(0), i.get_local_id(2), \ + i.get_local_id(1), i.get_local_id(0), shared_memory); \ + my_functor.OPERATION(); \ + i.barrier(); \ + } \ + \ + private: \ + FunctorType functor; \ + LocalMemory localAccess; \ + }; + +SINGLE_KERNEL_3D(Operation1); +SINGLE_KERNEL_3D(Operation2); +SINGLE_KERNEL_3D(Operation3); +SINGLE_KERNEL_3D(Operation4); +SINGLE_KERNEL_3D(Operation5); +SINGLE_KERNEL_3D(Operation6); +SINGLE_KERNEL_3D(Operation7); +SINGLE_KERNEL_3D(Operation8); +SINGLE_KERNEL_3D(Operation9); +SINGLE_KERNEL_3D(Operation10); +SINGLE_KERNEL_3D(Operation11); +SINGLE_KERNEL_3D(Operation12); +SINGLE_KERNEL_3D(Operation13); +SINGLE_KERNEL_3D(Operation14); +SINGLE_KERNEL_3D(Operation15); + +#undef SINGLE_KERNEL_3D + +#define SINGLE_KERNEL_1D(OPERATION) \ + template class Single_##OPERATION##_Kernel_1D { \ + public: \ + Single_##OPERATION##_Kernel_1D(FunctorType functor, \ + LocalMemory localAccess, THREAD_IDX ngridz, \ + THREAD_IDX ngridy, THREAD_IDX ngridx, \ + THREAD_IDX nblockz, THREAD_IDX nblocky, \ + THREAD_IDX nblockx) \ + : functor(functor), localAccess(localAccess), ngridz(ngridz), \ + ngridy(ngridy), ngridx(ngridx), nblockz(nblockz), nblocky(nblocky), \ + nblockx(nblockx) {} \ + void operator()(sycl::nd_item<3> i) const { \ + FunctorType my_functor = functor; \ + Byte *shared_memory = localAccess.get_pointer().get(); \ + my_functor.Init(ngridz, ngridy, ngridx, nblockz, nblocky, nblockx, \ + (i.get_group().get_group_id(0) / ngridx) / ngridy, \ + (i.get_group().get_group_id(0) / ngridx) % ngridy, \ + i.get_group().get_group_id(0) % ngridx, \ + (i.get_local_id(0) / nblockx) / nblocky, \ + (i.get_local_id(0) / nblockx) % nblocky, \ + i.get_local_id(0) % nblockx, shared_memory); \ + my_functor.OPERATION(); \ + i.barrier(); \ + } \ + \ + private: \ + FunctorType functor; \ + LocalMemory localAccess; \ + THREAD_IDX ngridz; \ + THREAD_IDX ngridy; \ + THREAD_IDX ngridx; \ + THREAD_IDX nblockz; \ + THREAD_IDX nblocky; \ + THREAD_IDX nblockx; \ + }; + +SINGLE_KERNEL_1D(Operation1); +SINGLE_KERNEL_1D(Operation2); +SINGLE_KERNEL_1D(Operation3); +SINGLE_KERNEL_1D(Operation4); +SINGLE_KERNEL_1D(Operation5); +SINGLE_KERNEL_1D(Operation6); +SINGLE_KERNEL_1D(Operation7); +SINGLE_KERNEL_1D(Operation8); +SINGLE_KERNEL_1D(Operation9); +SINGLE_KERNEL_1D(Operation10); +SINGLE_KERNEL_1D(Operation11); +SINGLE_KERNEL_1D(Operation12); +SINGLE_KERNEL_1D(Operation13); +SINGLE_KERNEL_1D(Operation14); +SINGLE_KERNEL_1D(Operation15); + +#undef SINGLE_KERNEL_1D + +template class SyclParallelMergeKernel { +public: + SyclParallelMergeKernel(FunctorType functor, LocalMemory localAccess, + THREAD_IDX ngridz, THREAD_IDX ngridy, + THREAD_IDX ngridx, THREAD_IDX nblockz, + THREAD_IDX nblocky, THREAD_IDX nblockx) + : functor(functor), localAccess(localAccess), ngridz(ngridz), + ngridy(ngridy), ngridx(ngridx), nblockz(nblockz), nblocky(nblocky), + nblockx(nblockx) {} + void operator()(sycl::nd_item<3> i) const { + FunctorType my_functor = functor; + Byte *shared_memory = localAccess.get_pointer().get(); +#if MGARD_X_160_SYCL_ND_RANGE_3D + my_functor.Init( + i.get_group_range(2), i.get_group_range(1), i.get_group_range(0), + i.get_global_range(2) / i.get_group_range(2), + i.get_global_range(1) / i.get_group_range(1), + i.get_global_range(0) / i.get_group_range(0), i.get_group().get_id(2), + i.get_group().get_id(1), i.get_group().get_id(0), i.get_local_id(2), + i.get_local_id(1), i.get_local_id(0), shared_memory); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + my_functor.Init(ngridz, ngridy, ngridx, nblockz, nblocky, nblockx, + (i.get_group().get_group_id(0) / ngridx) / ngridy, + (i.get_group().get_group_id(0) / ngridx) % ngridy, + i.get_group().get_group_id(0) % ngridx, + (i.get_local_id(0) / nblockx) / nblocky, + (i.get_local_id(0) / nblockx) % nblocky, + i.get_local_id(0) % nblockx, shared_memory); +#endif + my_functor.Operation6(); + i.barrier(); + while (my_functor.LoopCondition2()) { + my_functor.Operation7(); + i.barrier(); + my_functor.Operation8(); + i.barrier(); + my_functor.Operation9(); + i.barrier(); + } + my_functor.Operation10(); + } + +private: + FunctorType functor; + LocalMemory localAccess; + THREAD_IDX ngridz; + THREAD_IDX ngridy; + THREAD_IDX ngridx; + THREAD_IDX nblockz; + THREAD_IDX nblocky; + THREAD_IDX nblockx; +}; + +template void SyclHuffmanCLCustomizedNoCGKernel(Task task) { + // std::cout << "calling SyclHuffmanCLCustomizedNoCGKernel\n"; +#if MGARD_X_160_SYCL_ND_RANGE_3D + sycl::range global_threads(task.GetBlockDimX() * task.GetGridDimX(), + task.GetBlockDimY() * task.GetGridDimY(), + task.GetBlockDimZ() * task.GetGridDimZ()); + + sycl::range local_threads(task.GetBlockDimX(), task.GetBlockDimY(), + task.GetBlockDimZ()); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + sycl::range global_threads(task.GetBlockDimX() * task.GetGridDimX() * + task.GetBlockDimY() * task.GetGridDimY() * + task.GetBlockDimZ() * task.GetGridDimZ(), + 1, 1); + + sycl::range local_threads( + task.GetBlockDimX() * task.GetBlockDimY() * task.GetBlockDimZ(), 1, 1); +#endif + size_t sm_size = task.GetSharedMemorySize(); + if (sm_size == 0) + sm_size = 1; // avoid -51 (CL_INVALID_ARG_SIZE) error + + sycl::queue q = DeviceRuntime::GetQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation1_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation1_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation1_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), task.GetGridDimY(), + task.GetGridDimX(), task.GetBlockDimZ(), task.GetBlockDimY(), + task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling LoopCondition1\n"; + while (task.GetFunctor().LoopCondition1()) { + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation2_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation2_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation2_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation3_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation3_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation3_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation4_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation4_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation4_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation4_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation5_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation5_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling BranchCondition1\n"; + if (task.GetFunctor().BranchCondition1()) { + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling SyclParallelMergeKernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; + SyclParallelMergeKernel kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation10_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation11_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation11_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + } + + // std::cout << "calling Single_Operation11_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation12_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation12_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation12_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation13_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation13_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation13_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation14_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation14_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + // std::cout << "calling Single_Operation14_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation15_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation15_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + } +} + +template void SyclHuffmanCWCustomizedNoCGKernel(Task task) { + // std::cout << "calling SyclHuffmanCWCustomizedNoCGKernel\n"; +#if MGARD_X_160_SYCL_ND_RANGE_3D + sycl::range global_threads(task.GetBlockDimX() * task.GetGridDimX(), + task.GetBlockDimY() * task.GetGridDimY(), + task.GetBlockDimZ() * task.GetGridDimZ()); + + sycl::range local_threads(task.GetBlockDimX(), task.GetBlockDimY(), + task.GetBlockDimZ()); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + sycl::range global_threads(task.GetBlockDimX() * task.GetGridDimX() * + task.GetBlockDimY() * task.GetGridDimY() * + task.GetBlockDimZ() * task.GetGridDimZ(), + 1, 1); + + sycl::range local_threads( + task.GetBlockDimX() * task.GetBlockDimY() * task.GetBlockDimZ(), 1, 1); +#endif + size_t sm_size = task.GetSharedMemorySize(); + if (sm_size == 0) + sm_size = 1; // avoid -51 (CL_INVALID_ARG_SIZE) error + + sycl::queue q = DeviceRuntime::GetQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation1_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation1_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation1_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), task.GetGridDimY(), + task.GetGridDimX(), task.GetBlockDimZ(), task.GetBlockDimY(), + task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + // std::cout << "calling Single_Operation2_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation2_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation2_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), task.GetGridDimY(), + task.GetGridDimX(), task.GetBlockDimZ(), task.GetBlockDimY(), + task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + // std::cout << "calling Single_Operation3_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation3_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation3_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), task.GetGridDimY(), + task.GetGridDimX(), task.GetBlockDimZ(), task.GetBlockDimY(), + task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling LoopCondition1\n"; + while (task.GetFunctor().LoopCondition1()) { + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation4_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation4_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation4_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation5_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation5_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation5_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation6_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation6_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation6_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation7_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation7_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation7_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation8_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation8_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation8_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + } + + // std::cout << "calling Single_Operation9_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation9_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation9_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), task.GetGridDimY(), + task.GetGridDimX(), task.GetBlockDimZ(), task.GetBlockDimY(), + task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + + // std::cout << "calling Single_Operation10_Kernel\n"; + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; +#if MGARD_X_160_SYCL_ND_RANGE_3D + Single_Operation10_Kernel_3D kernel(task.GetFunctor(), localAccess); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + Single_Operation10_Kernel_1D kernel( + task.GetFunctor(), localAccess, task.GetGridDimZ(), task.GetGridDimY(), + task.GetGridDimX(), task.GetBlockDimZ(), task.GetBlockDimY(), + task.GetBlockDimX()); +#endif + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + DeviceRuntime::SyncQueue(task.GetQueueIdx()); +} + +// template class DeviceAdapter { +// public: +// // inline constexpr bool sycl::is_device_copyable_v = true; + +// MGARDX_CONT +// DeviceAdapter(){}; + +// MGARDX_CONT +// int IsResourceEnough(TaskType &task) { +// if (task.GetBlockDimX() * task.GetBlockDimY() * task.GetBlockDimZ() > +// DeviceRuntime::GetMaxNumThreadsPerTB()) { +// return THREADBLOCK_TOO_LARGE; +// } +// if (task.GetSharedMemorySize() > +// DeviceRuntime::GetMaxSharedMemorySize()) { +// return SHARED_MEMORY_TOO_LARGE; +// } +// return RESOURCE_ENOUGH; +// } + +// MGARDX_CONT +// ExecutionReturn Execute(TaskType &task) { +// #if MGARD_X_160_SYCL_ND_RANGE_3D +// sycl::range global_threads(task.GetBlockDimX() * task.GetGridDimX(), +// task.GetBlockDimY() * task.GetGridDimY(), +// task.GetBlockDimZ() * task.GetGridDimZ()); + +// sycl::range local_threads(task.GetBlockDimX(), task.GetBlockDimY(), +// task.GetBlockDimZ()); +// #endif +// #if MGARD_X_160_SYCL_ND_RANGE_1D +// sycl::range global_threads(task.GetBlockDimX() * task.GetGridDimX() * +// task.GetBlockDimY() * task.GetGridDimY() * +// task.GetBlockDimZ() * task.GetGridDimZ(), +// 1, 1); + +// sycl::range local_threads( +// task.GetBlockDimX() * task.GetBlockDimY() * task.GetBlockDimZ(), 1, +// 1); +// #endif +// size_t sm_size = task.GetSharedMemorySize(); +// if (sm_size == 0) +// sm_size = 1; // avoid -51 (CL_INVALID_ARG_SIZE) error + +// sycl::queue q = DeviceRuntime::GetQueue(task.GetQueueIdx()); + +// if (DeviceRuntime::PrintKernelConfig) { +// #if MGARD_X_160_SYCL_ND_RANGE_3D +// std::cout << log::log_info << task.GetFunctorName() << ": <" +// << task.GetBlockDimX() << ", " << task.GetBlockDimY() << ", " +// << task.GetBlockDimZ() << "> <" << task.GetGridDimX() << ", " +// << task.GetGridDimY() << ", " << task.GetGridDimZ() << ">\n"; +// #endif +// #if MGARD_X_160_SYCL_ND_RANGE_1D +// std::cout << log::log_info << task.GetFunctorName() << ": <" +// << task.GetBlockDimX() * task.GetBlockDimY() * +// task.GetBlockDimZ() +// << ", " << 1 << ", " << 1 << "> <" +// << task.GetGridDimX() * task.GetGridDimY() * +// task.GetGridDimZ() +// << ", " << 1 << ", " << 1 << ">\n"; +// #endif +// } + +// ExecutionReturn ret; +// if (IsResourceEnough(task) != RESOURCE_ENOUGH) { +// if (DeviceRuntime::PrintKernelConfig) { +// if (IsResourceEnough(task) == THREADBLOCK_TOO_LARGE) { +// log::info("threadblock too large."); +// } +// if (IsResourceEnough(task) == SHARED_MEMORY_TOO_LARGE) { +// log::info("shared memory too large."); +// } +// } +// ret.success = false; +// ret.execution_time = std::numeric_limits::max(); +// return ret; +// } + +// Timer timer; +// if (task.GetQueueIdx() == MGARDX_SYNCHRONIZED_QUEUE || +// DeviceRuntime::TimingAllKernels || +// AutoTuner::ProfileKernels) { +// DeviceRuntime::SyncDevice(); +// timer.start(); +// } + +// // if constexpr evaluate at compile time otherwise this does not compile +// if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) { +// q.submit([&](sycl::handler &h) { +// LocalMemory localAccess{sm_size, h}; +// SyclKernel kernel(task.GetFunctor(), localAccess, task.GetGridDimZ(), +// task.GetGridDimY(), task.GetGridDimX(), +// task.GetBlockDimZ(), task.GetBlockDimY(), +// task.GetBlockDimX()); +// h.parallel_for(sycl::nd_range{global_threads, local_threads}, +// kernel); +// }); +// } else if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) +// { +// q.submit([&](sycl::handler &h) { +// LocalMemory localAccess{sm_size, h}; +// SyclIterKernel kernel(task.GetFunctor(), localAccess, +// task.GetGridDimZ(), task.GetGridDimY(), +// task.GetGridDimX(), task.GetBlockDimZ(), +// task.GetBlockDimY(), task.GetBlockDimX()); +// h.parallel_for(sycl::nd_range{global_threads, local_threads}, +// kernel); +// }); +// } else if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) +// { +// SyclHuffmanCLCustomizedNoCGKernel(task); +// } else if constexpr (std::is_base_of, +// typename TaskType::Functor>::value) +// { +// SyclHuffmanCWCustomizedNoCGKernel(task); +// } +// if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { +// DeviceRuntime::SyncQueue(task.GetQueueIdx()); +// } + +// if (task.GetQueueIdx() == MGARDX_SYNCHRONIZED_QUEUE || +// DeviceRuntime::TimingAllKernels || +// AutoTuner::ProfileKernels) { +// DeviceRuntime::SyncDevice(); +// timer.end(); +// if (DeviceRuntime::TimingAllKernels) { +// timer.print(task.GetFunctorName()); +// } +// if (AutoTuner::ProfileKernels) { +// ret.success = true; +// ret.execution_time = timer.get(); +// } +// } +// return ret; +// } +// }; + +template <> class DeviceLauncher { +public: + template + MGARDX_CONT int static IsResourceEnough(TaskType &task) { + if (task.GetBlockDimX() * task.GetBlockDimY() * task.GetBlockDimZ() > + DeviceRuntime::GetMaxNumThreadsPerTB()) { + return THREADBLOCK_TOO_LARGE; + } + if (task.GetSharedMemorySize() > + DeviceRuntime::GetMaxSharedMemorySize()) { + return SHARED_MEMORY_TOO_LARGE; + } + return RESOURCE_ENOUGH; + } + + template + MGARDX_CONT ExecutionReturn static Execute(TaskType &task) { +#if MGARD_X_160_SYCL_ND_RANGE_3D + sycl::range global_threads(task.GetBlockDimX() * task.GetGridDimX(), + task.GetBlockDimY() * task.GetGridDimY(), + task.GetBlockDimZ() * task.GetGridDimZ()); + + sycl::range local_threads(task.GetBlockDimX(), task.GetBlockDimY(), + task.GetBlockDimZ()); +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + sycl::range global_threads(task.GetBlockDimX() * task.GetGridDimX() * + task.GetBlockDimY() * task.GetGridDimY() * + task.GetBlockDimZ() * task.GetGridDimZ(), + 1, 1); + + sycl::range local_threads( + task.GetBlockDimX() * task.GetBlockDimY() * task.GetBlockDimZ(), 1, 1); +#endif + size_t sm_size = task.GetSharedMemorySize(); + if (sm_size == 0) + sm_size = 1; // avoid -51 (CL_INVALID_ARG_SIZE) error + + sycl::queue q = DeviceRuntime::GetQueue(task.GetQueueIdx()); + + if (DeviceRuntime::PrintKernelConfig) { +#if MGARD_X_160_SYCL_ND_RANGE_3D + std::cout << log::log_info << task.GetFunctorName() << ": <" + << task.GetBlockDimX() << ", " << task.GetBlockDimY() << ", " + << task.GetBlockDimZ() << "> <" << task.GetGridDimX() << ", " + << task.GetGridDimY() << ", " << task.GetGridDimZ() << ">\n"; +#endif +#if MGARD_X_160_SYCL_ND_RANGE_1D + std::cout << log::log_info << task.GetFunctorName() << ": <" + << task.GetBlockDimX() * task.GetBlockDimY() * + task.GetBlockDimZ() + << ", " << 1 << ", " << 1 << "> <" + << task.GetGridDimX() * task.GetGridDimY() * task.GetGridDimZ() + << ", " << 1 << ", " << 1 << ">\n"; +#endif + } + + ExecutionReturn ret; + if (IsResourceEnough(task) != RESOURCE_ENOUGH) { + if (DeviceRuntime::PrintKernelConfig) { + if (IsResourceEnough(task) == THREADBLOCK_TOO_LARGE) { + log::info("threadblock too large."); + } + if (IsResourceEnough(task) == SHARED_MEMORY_TOO_LARGE) { + log::info("shared memory too large."); + } + } + ret.success = false; + ret.execution_time = std::numeric_limits::max(); + return ret; + } + + Timer timer; + if (task.GetQueueIdx() == MGARDX_SYNCHRONIZED_QUEUE || + DeviceRuntime::TimingAllKernels || + AutoTuner::ProfileKernels) { + DeviceRuntime::SyncDevice(); + timer.start(); + } + + // if constexpr evaluate at compile time otherwise this does not compile + if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; + SyclKernel kernel(task.GetFunctor(), localAccess, task.GetGridDimZ(), + task.GetGridDimY(), task.GetGridDimX(), + task.GetBlockDimZ(), task.GetBlockDimY(), + task.GetBlockDimX()); + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + q.submit([&](sycl::handler &h) { + LocalMemory localAccess{sm_size, h}; + SyclIterKernel kernel(task.GetFunctor(), localAccess, + task.GetGridDimZ(), task.GetGridDimY(), + task.GetGridDimX(), task.GetBlockDimZ(), + task.GetBlockDimY(), task.GetBlockDimX()); + h.parallel_for(sycl::nd_range{global_threads, local_threads}, kernel); + }); + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + SyclHuffmanCLCustomizedNoCGKernel(task); + } else if constexpr (std::is_base_of, + typename TaskType::Functor>::value) { + SyclHuffmanCWCustomizedNoCGKernel(task); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncQueue(task.GetQueueIdx()); + } + + if (task.GetQueueIdx() == MGARDX_SYNCHRONIZED_QUEUE || + DeviceRuntime::TimingAllKernels || + AutoTuner::ProfileKernels) { + DeviceRuntime::SyncDevice(); + timer.end(); + if (DeviceRuntime::TimingAllKernels) { + timer.print(task.GetFunctorName()); + } + if (AutoTuner::ProfileKernels) { + ret.success = true; + ret.execution_time = timer.get(); + } + } + return ret; + } + + template + MGARDX_CONT static void ConfigTask(TaskType task) { + typename TaskType::Functor functor; + int maxbytes = DeviceRuntime::GetMaxSharedMemorySize(); + DeviceRuntime::SetMaxDynamicSharedMemorySize(functor, maxbytes); + } + + template + MGARDX_CONT static void AutoTune(KernelType kernel, int queue_idx) { +#if MGARD_ENABLE_AUTO_TUNING + double min_time = std::numeric_limits::max(); + int min_config = 0; + ExecutionReturn ret; +#define RUN_CONFIG(CONFIG_IDX) \ + { \ + constexpr ExecutionConfig config = \ + GetExecutionConfig(CONFIG_IDX); \ + auto task = \ + kernel.template GenTask(queue_idx); \ + ret = Execute(task); \ + if constexpr (KernelType::EnableConfig()) { \ + ConfigTask(task); \ + } \ + if (ret.success && min_time > ret.execution_time) { \ + min_time = ret.execution_time; \ + min_config = CONFIG_IDX; \ + } \ + } + RUN_CONFIG(0) + RUN_CONFIG(1) + RUN_CONFIG(2) + RUN_CONFIG(3) + RUN_CONFIG(4) + RUN_CONFIG(5) + RUN_CONFIG(6) +#undef RUN_CONFIG + if (AutoTuner::WriteToTable) { + FillAutoTunerTable(std::string(KernelType::Name), min_config); + } +#else + throw std::runtime_error("MGARD is not built with auto tuning enabled."); +#endif + } + + template + MGARDX_CONT static void Execute(KernelType kernel, int queue_idx) { + if constexpr (KernelType::EnableAutoTuning()) { + constexpr ExecutionConfig config = + GetExecutionConfig(KernelType::Name); + auto task = + kernel.template GenTask(queue_idx); + if constexpr (KernelType::EnableConfig()) { + ConfigTask(task); + } + Execute(task); + + if (AutoTuner::ProfileKernels) { + AutoTune(kernel, queue_idx); + } + } else { + auto task = kernel.GenTask(queue_idx); + if constexpr (KernelType::EnableConfig()) { + ConfigTask(task); + } + Execute(task); + } + } +}; + +template struct AbsMaxOp { + T operator()(const T &a, const T &b) const { + return (fabs(b) > fabs(a)) ? fabs(b) : fabs(a); + } +}; + +template struct AbsMinOp { + T operator()(const T &a, const T &b) const { + return (fabs(b) > fabs(a)) ? fabs(a) : fabs(b); + } +}; + +template struct SquareOp { + T operator()(const T &a) const { return a * a; } +}; + +template <> class DeviceCollective { +public: + MGARDX_CONT + DeviceCollective(){}; + + template + MGARDX_CONT static void Sum(SIZE n, SubArray<1, T, SYCL> v, + SubArray<1, T, SYCL> result, + Array<1, Byte, SYCL> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + sycl::queue q = DeviceRuntime::GetQueue(queue_idx); + q.submit([&](sycl::handler &h) { + T *res = result.data(); + T *input = v.data(); + h.parallel_for( + sycl::range{n}, sycl::reduction(res, (T)0, sycl::plus()), + [=](sycl::id<1> i, auto &res) { res.combine(input[i]); }); + }); + DeviceRuntime::SyncDevice(); + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void AbsMax(SIZE n, SubArray<1, T, SYCL> v, + SubArray<1, T, SYCL> result, + Array<1, Byte, SYCL> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + sycl::queue q = DeviceRuntime::GetQueue(queue_idx); + q.submit([&](sycl::handler &h) { + T *res = result.data(); + T *input = v.data(); + h.parallel_for( + sycl::range{n}, sycl::reduction(res, (T)0, AbsMaxOp()), + [=](sycl::id<1> i, auto &res) { res.combine(input[i]); }); + }); + DeviceRuntime::SyncDevice(); + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void AbsMin(SIZE n, SubArray<1, T, SYCL> v, + SubArray<1, T, SYCL> result, + Array<1, Byte, SYCL> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + sycl::queue q = DeviceRuntime::GetQueue(queue_idx); + q.submit([&](sycl::handler &h) { + T *res = result.data(); + T *input = v.data(); + h.parallel_for( + sycl::range{n}, + sycl::reduction(res, std::numeric_limits::max(), AbsMinOp()), + [=](sycl::id<1> i, auto &res) { res.combine(input[i]); }); + }); + DeviceRuntime::SyncDevice(); + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void SquareSum(SIZE n, SubArray<1, T, SYCL> v, + SubArray<1, T, SYCL> result, + Array<1, Byte, SYCL> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + sycl::queue q = DeviceRuntime::GetQueue(queue_idx); + q.submit([&](sycl::handler &h) { + T *res = result.data(); + T *input = v.data(); + h.parallel_for(sycl::range{n}, + sycl::reduction(res, (T)0, sycl::plus()), + [=](sycl::id<1> i, auto &res) { + res.combine(input[i] * input[i]); + }); + }); + DeviceRuntime::SyncDevice(); + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void + ScanSumInclusive(SIZE n, SubArray<1, T, SYCL> v, SubArray<1, T, SYCL> result, + Array<1, Byte, SYCL> &workspace, bool workspace_allocated, + int queue_idx) {} + + template + MGARDX_CONT static void + ScanSumExclusive(SIZE n, SubArray<1, T, SYCL> v, SubArray<1, T, SYCL> result, + Array<1, Byte, SYCL> &workspace, bool workspace_allocated, + int queue_idx) {} + + template + MGARDX_CONT static void + ScanSumExtended(SIZE n, SubArray<1, T, SYCL> v, SubArray<1, T, SYCL> result, + Array<1, Byte, SYCL> &workspace, bool workspace_allocated, + int queue_idx) {} + + template + MGARDX_CONT static void SortByKey(SIZE n, SubArray<1, KeyT, SYCL> in_keys, + SubArray<1, ValueT, SYCL> in_values, + SubArray<1, KeyT, SYCL> out_keys, + SubArray<1, ValueT, SYCL> out_values, + Array<1, Byte, SYCL> &workspace, + bool workspace_allocated, int queue_idx) { + + if (workspace_allocated) { + KeyT *keys_array = new KeyT[n]; + ValueT *values_array = new ValueT[n]; + MemoryManager::Copy1D(keys_array, in_keys.data(), n, 0); + MemoryManager::Copy1D(values_array, in_values.data(), n, 0); + DeviceRuntime::SyncQueue(0); + + std::vector> data(n); + for (SIZE i = 0; i < n; ++i) { + data[i] = std::pair(keys_array[i], values_array[i]); + } + std::stable_sort(data.begin(), data.end(), + KeyValueComparator{}); + for (SIZE i = 0; i < n; ++i) { + keys_array[i] = data[i].first; + values_array[i] = data[i].second; + } + MemoryManager::Copy1D(out_keys.data(), keys_array, n, 0); + MemoryManager::Copy1D(out_values.data(), values_array, n, 0); + DeviceRuntime::SyncDevice(); + delete[] keys_array; + delete[] values_array; + } else { + workspace.resize({(SIZE)1}, queue_idx); + } + } + + template + MGARDX_CONT static void + ScanOpInclusiveByKey(SubArray<1, SIZE, SYCL> &key, + SubArray<1, ValueT, SYCL> &v, + SubArray<1, ValueT, SYCL> &result, int queue_idx) {} +}; + +} // namespace mgard_x_160 +#endif diff --git a/include/mgard-x-160/RuntimeX/Functors/Functor.h b/include/mgard-x-160/RuntimeX/Functors/Functor.h new file mode 100644 index 000000000..36982f933 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/Functors/Functor.h @@ -0,0 +1,300 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_FUNCTOR +#define MGARD_X_160_FUNCTOR + +namespace mgard_x_160 { + +template class FunctorBase { +public: + MGARDX_CONT + FunctorBase() {} + MGARDX_EXEC void Init(THREAD_IDX ngridz, THREAD_IDX ngridy, THREAD_IDX ngridx, + THREAD_IDX nblockz, THREAD_IDX nblocky, + THREAD_IDX nblockx, THREAD_IDX blockz, + THREAD_IDX blocky, THREAD_IDX blockx, + THREAD_IDX threadz, THREAD_IDX thready, + THREAD_IDX threadx, Byte *shared_memory) { + this->ngridz = ngridz; + this->ngridy = ngridy; + this->ngridx = ngridx; + this->nblockz = nblockz; + this->nblocky = nblocky; + this->nblockx = nblockx; + this->blockz = blockz; + this->blocky = blocky; + this->blockx = blockx; + this->threadz = threadz; + this->thready = thready; + this->threadx = threadx; + this->shared_memory = shared_memory; + } + + MGARDX_EXEC void InitConfig(THREAD_IDX ngridz, THREAD_IDX ngridy, + THREAD_IDX ngridx, THREAD_IDX nblockz, + THREAD_IDX nblocky, THREAD_IDX nblockx) { + this->ngridz = ngridz; + this->ngridy = ngridy; + this->ngridx = ngridx; + this->nblockz = nblockz; + this->nblocky = nblocky; + this->nblockx = nblockx; + } + + MGARDX_EXEC void InitBlockId(THREAD_IDX blockz, THREAD_IDX blocky, + THREAD_IDX blockx) { + this->blockz = blockz; + this->blocky = blocky; + this->blockx = blockx; + } + + MGARDX_EXEC void InitThreadId(THREAD_IDX threadz, THREAD_IDX thready, + THREAD_IDX threadx) { + this->threadz = threadz; + this->thready = thready; + this->threadx = threadx; + } + + MGARDX_EXEC void InitSharedMemory(Byte *shared_memory) { + this->shared_memory = shared_memory; + } + + MGARDX_EXEC + const THREAD_IDX &GetThreadIdX() const { return threadx; } + + MGARDX_EXEC + const THREAD_IDX &GetThreadIdY() const { return thready; } + + MGARDX_EXEC + const THREAD_IDX &GetThreadIdZ() const { return threadz; } + + MGARDX_EXEC + const THREAD_IDX &GetBlockDimX() const { return nblockx; } + + MGARDX_EXEC + const THREAD_IDX &GetBlockDimY() const { return nblocky; } + + MGARDX_EXEC + const THREAD_IDX &GetBlockDimZ() const { return nblockz; } + + MGARDX_EXEC + const THREAD_IDX &GetBlockIdX() const { return blockx; } + + MGARDX_EXEC + const THREAD_IDX &GetBlockIdY() const { return blocky; } + + MGARDX_EXEC + const THREAD_IDX &GetBlockIdZ() const { return blockz; } + + MGARDX_EXEC + const THREAD_IDX &GetGridDimX() const { return ngridx; } + + MGARDX_EXEC + const THREAD_IDX &GetGridDimY() const { return ngridy; } + + MGARDX_EXEC + const THREAD_IDX &GetGridDimZ() const { return ngridz; } + + MGARDX_EXEC + Byte *GetSharedMemory() { return shared_memory; } + +private: + Byte *shared_memory; + THREAD_IDX threadz, thready, threadx; + THREAD_IDX blockz, blocky, blockx; + THREAD_IDX ngridz, ngridy, ngridx; + THREAD_IDX nblockz, nblocky, nblockx; +}; + +template class Functor : public FunctorBase { +public: + MGARDX_EXEC void Operation1() {} + + MGARDX_EXEC void Operation2() {} + + MGARDX_EXEC void Operation3() {} + + MGARDX_EXEC void Operation4() {} + + MGARDX_EXEC void Operation5() {} + + MGARDX_EXEC void Operation6() {} + + MGARDX_EXEC void Operation7() {} + + MGARDX_EXEC void Operation8() {} + + MGARDX_EXEC void Operation9() {} + + MGARDX_EXEC void Operation10() {} + + MGARDX_EXEC void Operation11() {} + + MGARDX_EXEC void Operation12() {} + + MGARDX_EXEC void Operation13() {} + + MGARDX_EXEC void Operation14() {} + + MGARDX_EXEC void Operation15() {} + + MGARDX_EXEC void Operation16() {} + + MGARDX_EXEC void Operation17() {} + + MGARDX_EXEC void Operation18() {} + + MGARDX_EXEC void Operation19() {} + + MGARDX_EXEC void Operation20() {} + + MGARDX_EXEC void Operation21() {} + + MGARDX_EXEC void Operation22() {} + + MGARDX_EXEC void Operation23() {} + + MGARDX_EXEC void Operation24() {} + + MGARDX_EXEC void Operation25() {} + + MGARDX_EXEC void Operation26() {} + + MGARDX_EXEC void Operation27() {} + + MGARDX_EXEC void Operation28() {} + + MGARDX_EXEC void Operation29() {} + + MGARDX_EXEC void Operation30() {} +}; + +template +class IterFunctor : public FunctorBase { +public: + MGARDX_EXEC bool LoopCondition1() { return false; } + + MGARDX_EXEC bool LoopCondition2() { return false; } + + MGARDX_EXEC void Operation1() {} + + MGARDX_EXEC void Operation2() {} + + MGARDX_EXEC void Operation3() {} + + MGARDX_EXEC void Operation4() {} + + MGARDX_EXEC void Operation5() {} + + MGARDX_EXEC void Operation6() {} + + MGARDX_EXEC void Operation7() {} + + MGARDX_EXEC void Operation8() {} + + MGARDX_EXEC void Operation9() {} + + MGARDX_EXEC void Operation10() {} + + MGARDX_EXEC void Operation11() {} + + MGARDX_EXEC void Operation12() {} + + MGARDX_EXEC void Operation13() {} + + MGARDX_EXEC void Operation14() {} + + MGARDX_EXEC void Operation15() {} + + MGARDX_EXEC void Operation16() {} + + MGARDX_EXEC void Operation17() {} +}; + +template +class HuffmanCLCustomizedFunctor : public FunctorBase { +public: + MGARDX_EXEC void Operation1() {} // init + + MGARDX_EXEC bool LoopCondition1() { return false; } // global loop + + MGARDX_EXEC void Operation2() {} // combine two nodes + + MGARDX_EXEC void Operation3() {} // copy to temp + + MGARDX_EXEC void Operation4() {} // updatre iterator + + MGARDX_EXEC void Operation5() {} // parallel merge: diagonal devide: init + + MGARDX_EXEC bool LoopCondition2() { + return false; + } // parallel merge: diagonal devide: found + + MGARDX_EXEC void Operation6() { + } // parallel merge: diagonal devide: generate 0/1 + + MGARDX_EXEC void Operation7() {} // parallel merge: diagonal devide: check 0/1 + + MGARDX_EXEC void Operation8() { + } // parallel merge: diagonal devide: adjust window + + // end of loop 2 + + MGARDX_EXEC void Operation9() { + } // parallel merge: diagonal devide: boundary cases + + MGARDX_EXEC void Operation10() {} // parallel merge: merge path + + // end of parallel merge + + MGARDX_EXEC void Operation11() {} // meld + + MGARDX_EXEC void Operation12() {} // update iNodeRear + + MGARDX_EXEC void Operation13() {} // update leaders + + MGARDX_EXEC void Operation14() {} // update iNodesSize + + bool use_CG; +}; + +template +class HuffmanCWCustomizedFunctor : public FunctorBase { +public: + MGARDX_EXEC void Operation1() {} + + MGARDX_EXEC void Operation2() {} + + MGARDX_EXEC void Operation3() {} + + MGARDX_EXEC bool LoopCondition1() { return false; } // global loop + + MGARDX_EXEC void Operation4() {} + + MGARDX_EXEC void Operation5() {} + + MGARDX_EXEC bool BranchCondition1() { return false; } + + MGARDX_EXEC void Operation6() {} + + MGARDX_EXEC void Operation7() {} + + MGARDX_EXEC void Operation8() {} + + // end of loop + + MGARDX_EXEC void Operation9() {} + + MGARDX_EXEC void Operation10() {} + + bool use_CG; +}; + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/Kernel/Kernel.h b/include/mgard-x-160/RuntimeX/Kernel/Kernel.h new file mode 100644 index 000000000..7f0f99dc8 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/Kernel/Kernel.h @@ -0,0 +1,17 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ +#ifndef MGARD_X_160_KERNEL +#define MGARD_X_160_KERNEL + +namespace mgard_x_160 { +class Kernel { +public: + constexpr static bool EnableConfig() { return true; } + constexpr static bool EnableAutoTuning() { return true; } +}; +} // namespace mgard_x_160 +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/RuntimeX.h b/include/mgard-x-160/RuntimeX/RuntimeX.h new file mode 100644 index 000000000..9c8ab25d0 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/RuntimeX.h @@ -0,0 +1,70 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ +#include "DataTypes.h" + +#include "DataStructures/Array.h" + +#include "RuntimeXPublic.h" + +#include "DataTypes.h" + +#include "AutoTuners/AutoTuner.h" +#include "DeviceAdapters/DeviceAdapter.h" +#include "Kernel/Kernel.h" +#include "Tasks/Task.h" + +// Serial backend should be always available +#if MGARD_ENABLE_SERIAL +#include "DeviceAdapters/DeviceAdapterSerial.h" +#else +#error "The serial MGARD device should always be available." +#endif + +#if MGARD_ENABLE_OPENMP +#include "DeviceAdapters/DeviceAdapterOpenmp.h" +#endif + +#if MGARD_ENABLE_CUDA +#ifdef MGARDX_COMPILE_CUDA +#include "DeviceAdapters/DeviceAdapterCuda.h" +#endif +#endif + +#if MGARD_ENABLE_HIP +#ifdef MGARDX_COMPILE_HIP +#include "DeviceAdapters/DeviceAdapterHip.h" +#endif +#endif + +#if MGARD_ENABLE_SYCL +#ifdef MGARDX_COMPILE_SYCL +#include "DeviceAdapters/DeviceAdapterSycl.h" +#endif +#endif + +#if RUNTIME_X_ENABLE_KOKKOS +#include "DeviceAdapters/DeviceAdapterKokkos.h" +#endif + +#if MGARD_ENABLE_MULTI_DEVICE +#include +#endif + +#include "Utilities/CheckShape.hpp" +#include "Utilities/Exceptions.h" +#include "Utilities/OffsetCalculators.hpp" + +#include "DataStructures/Array.hpp" +#include "DataStructures/SubArray.hpp" +#include "DataStructures/SubArrayCopy.hpp" +#include "Utilities/SubArrayPrinter.hpp" + +#include "Utilities/Serializer.hpp" + +#include "DataStructures/MDRMetadata.hpp" + +#include "DataStructures/MDRData.hpp" diff --git a/include/mgard-x-160/RuntimeX/RuntimeXPublic.h b/include/mgard-x-160/RuntimeX/RuntimeXPublic.h new file mode 100644 index 000000000..b18a495bf --- /dev/null +++ b/include/mgard-x-160/RuntimeX/RuntimeXPublic.h @@ -0,0 +1,22 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ +#include "MGARDXConfig.h" + +#include "DataTypes.h" + +#include "Utilities/Log.h" +#include "Utilities/Timer.hpp" + +#ifndef MGARD_X_160_RUNTIME_X_PUBLIC_H +#define MGARD_X_160_RUNTIME_X_PUBLIC_H + +namespace mgard_x_160 { + +template bool deviceAvailable(); +} + +#endif diff --git a/include/mgard-x-160/RuntimeX/Tasks/Task.h b/include/mgard-x-160/RuntimeX/Tasks/Task.h new file mode 100644 index 000000000..25126c447 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/Tasks/Task.h @@ -0,0 +1,82 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ +#ifndef MGARD_X_160_TASK +#define MGARD_X_160_TASK + +#include "../Functors/Functor.h" + +namespace mgard_x_160 { + +template class Task { +public: + MGARDX_CONT + Task() {} + + MGARDX_CONT + Task(FunctorType functor, IDX ngridz, IDX ngridy, IDX ngridx, IDX nblockz, + IDX nblocky, IDX nblockx, SIZE shared_memory_size, int queue_idx) + : functor(functor), ngridz(ngridz), ngridy(ngridy), ngridx(ngridx), + nblockz(nblockz), nblocky(nblocky), nblockx(nblockx), + shared_memory_size(shared_memory_size), queue_idx(queue_idx) {} + + MGARDX_CONT + Task(FunctorType functor, IDX ngridz, IDX ngridy, IDX ngridx, IDX nblockz, + IDX nblocky, IDX nblockx, SIZE shared_memory_size, int queue_idx, + std::string functor_name) + : functor(functor), ngridz(ngridz), ngridy(ngridy), ngridx(ngridx), + nblockz(nblockz), nblocky(nblocky), nblockx(nblockx), + shared_memory_size(shared_memory_size), queue_idx(queue_idx), + functor_name(functor_name) {} + + // copy contructure does not work in device so return by reference + MGARDX_CONT_EXEC FunctorType &GetFunctor() { return functor; } + MGARDX_CONT int GetQueueIdx() { return queue_idx; } + MGARDX_CONT_EXEC IDX GetGridDimZ() { return ngridz; } + MGARDX_CONT_EXEC IDX GetGridDimY() { return ngridy; } + MGARDX_CONT_EXEC IDX GetGridDimX() { return ngridx; } + MGARDX_CONT_EXEC IDX GetBlockDimZ() { return nblockz; } + MGARDX_CONT_EXEC IDX GetBlockDimY() { return nblocky; } + MGARDX_CONT_EXEC IDX GetBlockDimX() { return nblockx; } + MGARDX_CONT_EXEC SIZE GetSharedMemorySize() { return shared_memory_size; } + MGARDX_CONT void SetFunctorName(std::string functor_name) { + this->functor_name = functor_name; + } + MGARDX_CONT std::string GetFunctorName() { return this->functor_name; } + using Functor = FunctorType; + +private: + FunctorType functor; + IDX ngridz, ngridy, ngridx; + IDX nblockz, nblocky, nblockx; + SIZE shared_memory_size; + int queue_idx; + std::string functor_name; +}; + +// template +// class ReduceTask { +// public: +// MGARDX_CONT +// ReduceTask(int num_items, SubArray<1, T_reduce> d_in, SubArray<1, T_reduce> +// d_out, T_reduce init, int queue_idx): +// num_items(num_items), d_in(d_in), d_out(d_out), init(init), +// queue_idx(queue_idx) {} +// MGARDX_CONT int get_queue_idx() { return queue_idx; } +// MGARDX_CONT int get_num_items() { return num_items; } +// MGARDX_CONT SubArray<1, T_reduce> get_d_in() { return d_in; } +// MGARDX_CONT SubArray<1, T_reduce> get_d_out() { return d_out; } +// MGARDX_CONT T_reduce get_init() { return init; } +// private: +// int num_items; +// SubArray<1, T_reduce> d_in; +// SubArray<1, T_reduce> d_out; +// T_reduce init; +// int queue_idx; +// }; + +} // namespace mgard_x_160 +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/Utilities/CheckShape.hpp b/include/mgard-x-160/RuntimeX/Utilities/CheckShape.hpp new file mode 100644 index 000000000..6167d0498 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/Utilities/CheckShape.hpp @@ -0,0 +1,30 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_CHECK_SHAPE_HPP +#define MGARD_X_160_CHECK_SHAPE_HPP + +namespace mgard_x_160 { + +template int check_shape(std::vector shape) { + if (D != shape.size()) { + return -1; + } + for (DIM i = 0; i < shape.size(); i++) { + if (shape[i] < 3) + return -2; + } + return 0; +} + +template T roundup(T a, T b) { + return ((double)(a - 1) / b + 1) * b; +} + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/Utilities/Exceptions.h b/include/mgard-x-160/RuntimeX/Utilities/Exceptions.h new file mode 100644 index 000000000..d8578e6c8 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/Utilities/Exceptions.h @@ -0,0 +1,53 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + */ + +#ifndef MGARD_X_160_EXCEPTIONS_H +#define MGARD_X_160_EXCEPTIONS_H + +#include +#include + +#include "../../Utilities/Types.h" + +namespace mgard_x_160 { + +//! Base class for all exceptions thrown by the MGARD-X library. +//! +//! Errors that are internal to the (de)compression process are reported by +//! throwing an exception rather than calling `exit()`, so that the calling +//! application can handle the failure gracefully (clean up, report to a job +//! scheduler, keep other threads/GPUs alive, etc.). Every exception carries a +//! `compress_status_type` so the high-level API can translate a thrown +//! exception back into the status code returned by `compress()`/`decompress()`. +class Exception : public std::runtime_error { +public: + explicit Exception(const std::string &msg, compress_status_type status = + compress_status_type::Failure) + : std::runtime_error(msg), status_(status) {} + + //! Status code that the high-level API should return for this exception. + compress_status_type status() const noexcept { return status_; } + +private: + compress_status_type status_; +}; + +//! Thrown when compressed data or its metadata header is malformed, +//! corrupted, or was produced by an incompatible version of MGARD. +class InvalidDataException : public Exception { +public: + explicit InvalidDataException(const std::string &msg) : Exception(msg) {} +}; + +//! Thrown when an internal processing step fails (e.g. a lossless backend +//! error or a Huffman codebook that cannot be represented). +class ProcessingException : public Exception { +public: + explicit ProcessingException(const std::string &msg) : Exception(msg) {} +}; + +} // namespace mgard_x_160 + +#endif // MGARD_X_160_EXCEPTIONS_H diff --git a/include/mgard-x-160/RuntimeX/Utilities/Log.h b/include/mgard-x-160/RuntimeX/Utilities/Log.h new file mode 100644 index 000000000..756c021e3 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/Utilities/Log.h @@ -0,0 +1,54 @@ +#ifndef MGARD_X_160_MESSGAE_HH +#define MGARD_X_160_MESSGAE_HH + +#include +#include +#include +#include + +using std::string; + +namespace mgard_x_160 { +namespace log { + +extern int INFO; +extern int TIME; +extern int DBG; +extern int WARN; +extern int ERR; + +extern int level; + +extern const string log_null; +extern const string log_info; +extern const string log_time; +extern const string log_dbg; +extern const string log_warn; +extern const string log_err; + +void info(std::string msg, bool override = false); +void time(std::string msg, bool override = false); +void dbg(std::string msg, bool override = false); +void warn(std::string msg, bool override = false); +void err(std::string msg, bool override = false); + +// https://stackoverflow.com/a/26080768/8740097 +template void build(std::ostream &o, T t); + +template +void build(std::ostream &o, T t, Args... args); + +template void print(string log_head, Args... args); + +template void csv(std::string file, T v) { + std::ofstream myfile; + myfile.open(file, std::ios_base::app); + myfile << v << ", "; + myfile.close(); +} + +} // namespace log + +} // namespace mgard_x_160 + +#endif // FORMAT_HH diff --git a/include/mgard-x-160/RuntimeX/Utilities/OffsetCalculators.hpp b/include/mgard-x-160/RuntimeX/Utilities/OffsetCalculators.hpp new file mode 100644 index 000000000..db4ba29dc --- /dev/null +++ b/include/mgard-x-160/RuntimeX/Utilities/OffsetCalculators.hpp @@ -0,0 +1,61 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_OFFSET_CALCULATORS_H +#define MGARD_X_160_OFFSET_CALCULATORS_H + +namespace mgard_x_160 { + +MGARDX_CONT_EXEC SIZE get_idx(const SIZE ld1, const SIZE ld2, const SIZE z, + const SIZE y, const SIZE x) { + return ld2 * ld1 * z + ld1 * y + x; +} + +// leading dimension first +MGARDX_CONT SIZE get_idx(std::vector lds, std::vector idx) { + SIZE curr_stride = 1; + SIZE ret_idx = 0; + for (DIM i = 0; i < idx.size(); i++) { + ret_idx += idx[i] * curr_stride; + curr_stride *= lds[i]; + } + return ret_idx; +} + +template MGARDX_CONT_EXEC SIZE get_idx(SIZE *lds, SIZE *idx) { + SIZE curr_stride = 1; + SIZE ret_idx = 0; + for (DIM i = 0; i < D; i++) { + ret_idx += idx[i] * curr_stride; + curr_stride *= lds[i]; + } + return ret_idx; +} + +MGARDX_CONT std::vector gen_idx(DIM D, DIM curr_dim_r, DIM curr_dim_c, + DIM curr_dim_f, SIZE idx_r, SIZE idx_c, + SIZE idx_f) { + std::vector idx(D, 0); + idx[curr_dim_r] = idx_r; + idx[curr_dim_c] = idx_c; + idx[curr_dim_f] = idx_f; + return idx; +} + +MGARDX_CONT_EXEC int div_roundup(SIZE a, SIZE b) { return (a - 1) / b + 1; } + +template MGARDX_CONT_EXEC SIZE roundup(T2 a) { + return ((a - 1) / sizeof(T1) + 1) * sizeof(T1); +} + +template MGARDX_CONT_EXEC T round_up(T a, T b) { + return ((a - 1) / b + 1) * b; +} + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/RuntimeX/Utilities/Serializer.hpp b/include/mgard-x-160/RuntimeX/Utilities/Serializer.hpp new file mode 100644 index 000000000..1fd81b539 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/Utilities/Serializer.hpp @@ -0,0 +1,79 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_SERIALIZER_HH +#define MGARD_X_160_SERIALIZER_HH + +namespace mgard_x_160 { +template +MGARDX_CONT_EXEC void align_byte_offset(SIZE &byte_offset) { + if (byte_offset % sizeof(T) != 0) { + byte_offset = ((byte_offset - 1) / sizeof(T) + 1) * sizeof(T); + } +} + +template +MGARDX_CONT_EXEC void advance_with_align(SIZE &byte_offset, SIZE count) { + align_byte_offset(byte_offset); + byte_offset += count * sizeof(T); +} + +template +void SerializeArray(SubArray<1, Byte, DeviceType> &array, T *data_ptr, + SIZE count, SIZE &byte_offset, int queue_idx) { + using Mem = MemoryManager; + align_byte_offset(byte_offset); + Mem::Copy1D(array(byte_offset), (Byte *)data_ptr, count * sizeof(T), + queue_idx); + byte_offset += count * sizeof(T); +} + +template +void DeserializeArray(SubArray<1, Byte, DeviceType> &array, T *&data_ptr, + SIZE count, SIZE &byte_offset, bool zero_copy, + int queue_idx) { + using Mem = MemoryManager; + align_byte_offset(byte_offset); + if (zero_copy) { + data_ptr = (T *)array(byte_offset); + } else { + Mem::Copy1D((Byte *)data_ptr, array(byte_offset), count * sizeof(T), + queue_idx); + } + byte_offset += count * sizeof(T); +} + +template +void Serialize(Byte *serialize_ptr, T *data_ptr, SIZE count, SIZE &byte_offset, + int queue_idx) { + using Mem = MemoryManager; + align_byte_offset(byte_offset); + Mem::Copy1D(serialize_ptr + byte_offset, (Byte *)data_ptr, count * sizeof(T), + queue_idx); + byte_offset += count * sizeof(T); +} + +template +void Deserialize(Byte *serialize_ptr, T *&data_ptr, SIZE count, + SIZE &byte_offset, bool zero_copy, int queue_idx) { + using Mem = MemoryManager; + // Serialize aligns each value to sizeof(T); skip the same padding here, or + // a record whose byte count is not a multiple of 8 (e.g. a subdomain stored + // uncompressed) shifts every size field after it. + align_byte_offset(byte_offset); + if (zero_copy) { + data_ptr = (T *)(serialize_ptr + byte_offset); + } else { + Mem::Copy1D((Byte *)data_ptr, serialize_ptr + byte_offset, + count * sizeof(T), queue_idx); + } + byte_offset += count * sizeof(T); +} + +} // namespace mgard_x_160 + +#endif \ No newline at end of file diff --git a/include/mgard-x-160/RuntimeX/Utilities/SubArrayPrinter.hpp b/include/mgard-x-160/RuntimeX/Utilities/SubArrayPrinter.hpp new file mode 100644 index 000000000..ff5f9818a --- /dev/null +++ b/include/mgard-x-160/RuntimeX/Utilities/SubArrayPrinter.hpp @@ -0,0 +1,535 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_MEMORY_MANAGEMENT_HPP +#define MGARD_X_160_MEMORY_MANAGEMENT_HPP + +#include +#include +#include +#include +#include +#include // std::stringstream +#include // std::runtime_error +#include +#include +#include // std::pair +#include + +// #include "MemoryManagement.h" + +#define ANSI_RED "\x1b[31m" +#define ANSI_GREEN "\x1b[32m" +#define ANSI_RESET "\x1b[0m" + +namespace mgard_x_160 { + +template +void PrintSubarray(std::string name, SubArrayType subArray) { + // Handle<1, float> tmp_handle; + + SIZE nrow = 1; + SIZE ncol = 1; + SIZE nfib = 1; + + DIM D = SubArrayType::NumDims; + + nfib = subArray.shape(D - 1); + if (SubArrayType::NumDims >= 2) + ncol = subArray.shape(D - 2); + if (SubArrayType::NumDims >= 3) + nrow = subArray.shape(D - 3); + + using T = typename SubArrayType::DataType; + using DeviceType = typename SubArrayType::DevType; + + std::cout << "SubArray: " << name << "(" << nrow << " * " << ncol << " * " + << nfib << ") sizeof(T) = " << sizeof(T) << std::endl; + // std::cout << name << "\n"; + + T *v = new T[nrow * ncol * nfib]; + // cudaMemcpy3DAsyncHelper(tmp_handle, v, nfib * sizeof(T), nfib * sizeof(T), + // ncol, subArray.data(), subArray.lddv1 * sizeof(T), + // nfib * sizeof(T), subArray.lddv2, nfib * sizeof(T), + // ncol, nrow, D2H, 0); + DeviceRuntime::SyncQueue(0); + for (SIZE i = 0; i < nrow; i++) { + MemoryManager::CopyND( + v + ncol * nfib * i, nfib, + subArray.data() + subArray.lddv1() * subArray.lddv2() * i, + subArray.lddv1(), nfib, ncol, 0); + } + DeviceRuntime::SyncQueue(0); + // tmp_handle.sync(0); + + for (int i = 0; i < nrow; i++) { + printf("[i = %d]\n", i); + for (int j = 0; j < ncol; j++) { + for (int k = 0; k < nfib; k++) { + // std::cout << "[ " << j << ", " << k <<" ]: "; + if (std::is_same::value) { + std::cout << std::setw(8) + << (unsigned int)v[nfib * ncol * i + nfib * j + k] << " "; + } else { + std::cout << std::setw(8) << std::setprecision(6) << std::fixed + << v[nfib * ncol * i + nfib * j + k] << " "; + } + // std::cout << "\n"; + } + std::cout << std::endl; + } + std::cout << std::endl; + } + std::cout << std::endl; + delete[] v; +} + +template +void CompareSubarray(std::string name, SubArrayType subArray1, + SubArrayType subArray2) { + // Handle<1, float> tmp_handle; + + DIM D = SubArrayType::NumDims; + + SIZE nrow = 1; + SIZE ncol = 1; + SIZE nfib = 1; + + nfib = subArray1.shape(D - 1); + if (SubArrayType::NumDims >= 2) + ncol = subArray1.shape(D - 2); + if (SubArrayType::NumDims >= 3) + nrow = subArray1.shape(D - 3); + + if (subArray1.shape(D - 1) != subArray2.shape(D - 1) || + subArray1.shape(D - 2) != subArray2.shape(D - 2) || + subArray1.shape(D - 3) != subArray2.shape(D - 3)) { + throw std::runtime_error("CompareSubarray: shape mismatch!"); + } + + using T = typename SubArrayType::DataType; + using DeviceType = typename SubArrayType::DevType; + std::cout << "SubArray: " << name << "(" << nrow << " * " << ncol << " * " + << nfib << ") sizeof(T) = " << sizeof(T) << std::endl; + + T *v1 = new T[nrow * ncol * nfib]; + T *v2 = new T[nrow * ncol * nfib]; + + for (SIZE i = 0; i < nrow; i++) { + MemoryManager::CopyND( + v1 + ncol * nfib * i, nfib, + subArray1.data() + subArray1.lddv1() * subArray1.lddv2() * i, + subArray1.lddv1(), nfib, ncol, 0); + MemoryManager::CopyND( + v2 + ncol * nfib * i, nfib, + subArray2.data() + subArray2.lddv1() * subArray2.lddv2() * i, + subArray2.lddv1(), nfib, ncol, 0); + } + DeviceRuntime::SyncQueue(0); + + T max_error = 0; + bool pass = true; + for (int i = 0; i < nrow; i++) { + printf("[i = %d]\n", i); + for (int j = 0; j < ncol; j++) { + for (int k = 0; k < nfib; k++) { + T a = v1[nfib * ncol * i + nfib * j + k]; + T b = v2[nfib * ncol * i + nfib * j + k]; + max_error = std::max(max_error, fabs(a - b)); + if (fabs(a - b) > 1e-5 * fabs(a)) { + std::cout << ANSI_RED; + pass = false; + } else { + std::cout << ANSI_GREEN; + } + if (std::is_same::value) { + std::cout << std::setw(8) + << (unsigned int)v2[nfib * ncol * i + nfib * j + k] << ", "; + } else { + std::cout << std::setw(8) << std::setprecision(6) << std::fixed + << v2[nfib * ncol * i + nfib * j + k] << ", "; + } + std::cout << ANSI_RESET; + } + std::cout << std::endl; + } + std::cout << std::endl; + } + std::cout << std::endl; + + printf("Check: %d, max error: %f\n", pass, max_error); + delete[] v1; + delete[] v2; +} + +template +void CompareSubarray(std::string name, SubArrayType1 subArray1, + SubArrayType2 subArray2, bool print, + double error_thresold) { + // Handle<1, float> tmp_handle; + + DIM D = SubArrayType1::NumDims; + + SIZE nrow = 1; + SIZE ncol = 1; + SIZE nfib = 1; + + nfib = subArray1.shape(D - 1); + if (SubArrayType1::NumDims >= 2) + ncol = subArray1.shape(D - 2); + if (SubArrayType1::NumDims >= 3) + nrow = subArray1.shape(D - 3); + + if (subArray1.shape(D - 1) != subArray2.shape[0] || + (SubArrayType1::NumDims >= 2 && + subArray1.shape(D - 2) != subArray2.shape[1]) || + (SubArrayType1::NumDims >= 3 && + subArray1.shape(D - 3) != subArray2.shape[2])) { + throw std::runtime_error("CompareSubarray: shape mismatch!"); + } + + using T = typename SubArrayType1::DataType; + using DeviceType = typename SubArrayType1::DevType; + // std::cout << "SubArray: " << name << "(" << nrow << " * " << ncol << " * " + // << nfib << ") sizeof(T) = " <::CopyND( + v1 + ncol * nfib * i, nfib, + subArray1.data() + subArray1.lddv1() * subArray1.lddv2() * i, + subArray1.lddv1(), nfib, ncol, 0); + MemoryManager::CopyND(v2 + ncol * nfib * i, nfib, + subArray2.data() + + subArray2.lddv1 * subArray2.lddv2 * i, + subArray2.lddv1, nfib, ncol, 0); + } + DeviceRuntime::SyncQueue(0); + + T max_error = 0; + bool pass = true; + for (int i = 0; i < nrow; i++) { + if (print) + printf("[i = %d]\n", i); + for (int j = 0; j < ncol; j++) { + for (int k = 0; k < nfib; k++) { + T a = v1[nfib * ncol * i + nfib * j + k]; + T b = v2[nfib * ncol * i + nfib * j + k]; + max_error = std::max(max_error, fabs(a - b)); + if (fabs(a - b) > error_thresold * fabs(a)) { + if (print) + std::cout << ANSI_RED; + pass = false; + } else { + if (print) + std::cout << ANSI_GREEN; + } + if (std::is_same::value) { + if (print) + std::cout << std::setw(8) + << (unsigned int)v2[nfib * ncol * i + nfib * j + k] + << ", "; + } else { + if (print) + std::cout << std::setw(8) << std::setprecision(6) << std::fixed + << v2[nfib * ncol * i + nfib * j + k] << ", "; + } + if (print) + std::cout << ANSI_RESET; + } + if (print) + std::cout << std::endl; + } + if (print) + std::cout << std::endl; + } + if (print) + std::cout << std::endl; + + printf("Check: %d, max error: %f\n", pass, max_error); + delete[] v1; + delete[] v2; +} + +template +void CompareSubarray4D(SubArrayType subArray1, SubArrayType subArray2) { + if (SubArrayType::NumDims != 4) { + throw std::runtime_error("CompareSubarray4D expects 4D subarray type."); + } + + DIM D = SubArrayType::NumDims; + + if (subArray1.shape(D - 4) != subArray2.shape(D - 4)) { + throw std::runtime_error("CompareSubarray4D mismatch 4D size."); + } + + using T = typename SubArrayType::DataType; + SIZE idx[4] = {0, 0, 0, 0}; + for (SIZE i = 0; i < subArray1.shape(0); i++) { + idx[3] = i; + SubArrayType temp1 = subArray1; + SubArrayType temp2 = subArray2; + // Adding offset to the 4th dim. (slowest) + temp1.offset_dim(0, i); + temp2.offset_dim(0, i); + // Make 3D slice on the other three dims + CompareSubarray("4D = " + std::to_string(i), temp1.Slice3D(1, 2, 3), + temp2.Slice3D(1, 2, 3)); + } +} + +template +void PrintSubarray4D(std::string name, SubArrayType subArray1) { + if (SubArrayType::NumDims != 4) { + throw std::runtime_error("PrintSubarray4D expects 4D subarray type."); + } + + DIM D = SubArrayType::NumDims; + + std::cout << name << "\n"; + using T = typename SubArrayType::DataType; + SIZE idx[4] = {0, 0, 0, 0}; + for (SIZE i = 0; i < subArray1.shape(D - 4); i++) { + idx[3] = i; + SubArrayType temp1 = subArray1; + temp1.offset_dim(3, i); + PrintSubarray("i = " + std::to_string(i), temp1.Slice3D(0, 1, 2)); + } +} + +// print 3D CPU +template +void verify_matrix(SIZE nrow, SIZE ncol, SIZE nfib, T *v, SIZE ldv1, SIZE ldv2, + std::string file_prefix, bool store, bool verify) { + std::string filename = file_prefix + ".dat"; + if (store) { + std::ofstream myfile; + myfile.open(filename, std::ios::out | std::ios::binary); + if (!myfile) { + printf("Error: cannot write file\n"); + return; + } + myfile.write((char *)v, nrow * ncol * nfib * sizeof(T)); + myfile.close(); + if (!myfile.good()) { + printf("Error occurred at write time!\n"); + return; + } + } + if (verify) { + std::fstream fin; + fin.open(filename, std::ios::in | std::ios::binary); + if (!fin) { + printf("Error: cannot read file\n"); + return; + } + T *v2 = new T[nrow * ncol * nfib]; + fin.read((char *)v2, nrow * ncol * nfib * sizeof(T)); + fin.close(); + if (!fin.good()) { + printf("Error occurred at reading time!\n"); + return; + } + + bool mismatch = false; + for (int i = 0; i < nrow; i++) { + for (int j = 0; j < ncol; j++) { + for (int k = 0; k < nfib; k++) { + if (v[get_idx(ldv1, ldv2, i, j, k)] != + v2[get_idx(nfib, ncol, i, j, k)]) { + std::cout << filename << ": "; + printf("Mismatch[%d %d %d] %f - %f\n", i, j, k, + v[get_idx(ldv1, ldv2, i, j, k)], + v2[get_idx(nfib, ncol, i, j, k)]); + mismatch = true; + } + } + } + } + printf("Mismatch result: %d\n", mismatch); + + delete[] v2; + if (mismatch) + throw std::runtime_error("verify_matrix: mismatch"); + } +} + +// print 3D GPU +template +void verify_matrix_cuda(SIZE nrow, SIZE ncol, SIZE nfib, T *dv, SIZE lddv1, + SIZE lddv2, SIZE sizex, std::string file_prefix, + bool store, bool verify) { + // std::cout << std::setw(10); + // std::cout << std::setprecision(2) << std::fixed; + if (store || verify) { + // Handle<3, float> *tmp_handle = new Handle<3, float>(); + int queue_idx = 0; + + T *v = new T[nrow * ncol * nfib]; + // cudaMemcpy3DAsyncHelper(*tmp_handle, v, nfib * sizeof(T), nfib * + // sizeof(T), + // ncol, dv, lddv1 * sizeof(T), sizex * sizeof(T), + // lddv2, nfib * sizeof(T), ncol, nrow, D2H, + // queue_idx); + // MemoryManager::CopyND(v, nfib, dv, lddv1, nfib, ncol * nrow, + // 0); DeviceRuntime::SyncQueue(0); tmp_handle->sync(queue_idx); + verify_matrix(nrow, ncol, nfib, v, nfib, ncol, file_prefix, store, verify); + delete[] v; + // delete tmp_handle; + } +} + +template +void CompareSubArrays(SubArray<1, T, DeviceType> array1, + SubArray<1, T, DeviceType> array2) { + SIZE n = array1.shape[0]; + using Mem = MemoryManager; + T *q1 = new T[n]; + T *q2 = new T[n]; + Mem::Copy1D(q1, array1.data(), n, 0); + Mem::Copy1D(q2, array2.data(), n, 0); + DeviceRuntime::SyncQueue(0); + bool pass = true; + for (int i = 0; i < n; i++) { + if (q1[i] != q2[i]) { + pass = false; + std::cout << "diff at " << i << "(" << q1[i] << ", " << q2[i] << ")\n"; + } + } + printf("pass: %d\n", pass); + delete[] q1; + delete[] q2; +} + +template +void DumpSubArray(std::string name, SubArray subArray) { + SIZE nrow = 1; + SIZE ncol = 1; + SIZE nfib = 1; + + nfib = subArray.shape(D - 1); + if (D >= 2) + ncol = subArray.shape(D - 2); + if (D >= 3) + nrow = subArray.shape(D - 3); + + T *v = new T[nrow * ncol * nfib]; + DeviceRuntime::SyncQueue(0); + MemoryManager::CopyND(v, nfib, subArray.data(), + subArray.ld(D - 1), nfib, ncol * nrow, 0); + DeviceRuntime::SyncQueue(0); + std::fstream myfile; + myfile.open(name, std::ios::out | std::ios::binary); + if (!myfile) { + printf("Error: cannot open file\n"); + return; + } + myfile.write((char *)v, nrow * ncol * nfib * sizeof(T)); + myfile.close(); + if (!myfile.good()) { + printf("Error occurred at write time!\n"); + return; + } + delete[] v; +} + +template +void LoadSubArray(std::string name, SubArray subArray) { + + SIZE nrow = 1; + SIZE ncol = 1; + SIZE nfib = 1; + + nfib = subArray.shape(D - 1); + if (D >= 2) + ncol = subArray.shape(D - 2); + if (D >= 3) + nrow = subArray.shape(D - 3); + + T *v = new T[nrow * ncol * nfib]; + + std::fstream myfile; + myfile.open(name, std::ios::in | std::ios::binary); + if (!myfile) { + printf("Error: cannot open file\n"); + return; + } + myfile.read((char *)v, nrow * ncol * nfib * sizeof(T)); + myfile.close(); + if (!myfile.good()) { + printf("Error occurred at read time!\n"); + return; + } + MemoryManager::CopyND(subArray.data(), subArray.ld(D - 1), v, + nfib, nfib, ncol * nrow, 0); + delete[] v; +} + +template +void VerifySubArray(std::string name, SubArray subArray, + bool dump, bool verify) { + if (dump) { + printf("dump %s\n", name.c_str()); + DumpSubArray(name, subArray); + } + if (verify) { + SIZE nrow = 1; + SIZE ncol = 1; + SIZE nfib = 1; + + nfib = subArray.shape(D - 1); + if (D >= 2) + ncol = subArray.shape(D - 2); + if (D >= 3) + nrow = subArray.shape(D - 3); + + T *v1 = new T[nrow * ncol * nfib]; + T *v2 = new T[nrow * ncol * nfib]; + + std::fstream myfile; + myfile.open(name, std::ios::in | std::ios::binary); + if (!myfile) { + printf("Error: cannot open file\n"); + return; + } + myfile.read((char *)v1, nrow * ncol * nfib * sizeof(T)); + myfile.close(); + if (!myfile.good()) { + printf("Error occurred at read time!\n"); + return; + } + + MemoryManager::CopyND(v2, nfib, subArray.data(), + subArray.ld(D - 1), nfib, ncol * nrow, 0); + DeviceRuntime::SyncQueue(0); + + printf("verify %s\n", name.c_str()); + bool pass = true; + for (int i = 0; i < nrow; i++) { + for (int j = 0; j < ncol; j++) { + for (int k = 0; k < nfib; k++) { + if (v1[i * ncol * nfib + j * nfib + k] != + v2[i * ncol * nfib + j * nfib + k]) { + pass = false; + printf("diff at [%d][%d][%d]: ", i, j, k); + std::cout << v1[i * ncol * nfib + j * nfib + k] << ", " + << v2[i * ncol * nfib + j * nfib + k] << "\n"; + } + } + } + } + printf("pass: %d\n", pass); + delete[] v1; + delete[] v2; + } +} + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/RuntimeX/Utilities/Timer.hpp b/include/mgard-x-160/RuntimeX/Utilities/Timer.hpp new file mode 100644 index 000000000..d0afbf5e5 --- /dev/null +++ b/include/mgard-x-160/RuntimeX/Utilities/Timer.hpp @@ -0,0 +1,54 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_TIMER_HPP +#define MGARD_X_160_TIMER_HPP + +namespace mgard_x_160 { +class Timer { +public: + void start() { err = clock_gettime(CLOCK_REALTIME, &start_time); } + void end() { + err = clock_gettime(CLOCK_REALTIME, &end_time); + total_time += + (double)(end_time.tv_sec - start_time.tv_sec) + + (double)(end_time.tv_nsec - start_time.tv_nsec) / (double)1000000000; + } + double get() { + double time = + (double)(end_time.tv_sec - start_time.tv_sec) + + (double)(end_time.tv_nsec - start_time.tv_nsec) / (double)1000000000; + return total_time; + } + + double get_throughput(SIZE num_bytes) { + return (double)num_bytes / get() / 1e9; + } + + void clear() { total_time = 0; } + void print(std::string s, SIZE num_bytes = 0, bool override = false) { + if (num_bytes == 0) { + log::time(s + ": " + std::to_string(total_time) + " s", override); + } else { + log::time(s + ": " + std::to_string(total_time) + " s (" + + std::to_string(get_throughput(num_bytes)) + " GB/s)", + override); + } + } + + void print_throughput(std::string s, SIZE n, bool override = false) { + log::time(s + " throughput: " + std::to_string(get_throughput(n)) + " GB/s", + override); + } + +private: + int err = 0; + double total_time = 0; + struct timespec start_time, end_time; +}; +} // namespace mgard_x_160 +#endif \ No newline at end of file diff --git a/include/mgard-x-160/Utilities/ErrorCalculator.h b/include/mgard-x-160/Utilities/ErrorCalculator.h new file mode 100644 index 000000000..6db7e691f --- /dev/null +++ b/include/mgard-x-160/Utilities/ErrorCalculator.h @@ -0,0 +1,125 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_ERROR_CALCULATOR +#define MGARD_X_160_ERROR_CALCULATOR + +// #include "../../TensorMeshHierarchy.hpp" +// #include "../../TensorNorms.hpp" +// #include "../../shuffle.hpp" +#include "Types.h" + +#include +#include +#include + +namespace mgard_x_160 { + +template double L_inf_norm(size_t n, const T *data) { + double L_inf = 0; + for (size_t i = 0; i < n; ++i) { + double temp = fabs(data[i]); + if (temp > L_inf) + L_inf = temp; + } + // Avoiding issue with norm == 0 + if (L_inf == 0) + L_inf = std::numeric_limits::epsilon(); + return L_inf; +} + +template +double L_2_norm(std::vector shape, const T *data, + bool normalize_coordinates) { + SIZE n = 1; + for (DIM d = 0; d < shape.size(); d++) + n *= shape[d]; + double L_2 = 0; + for (size_t i = 0; i < n; ++i) { + double temp = fabs(data[i]); + L_2 += temp * temp; + } + // Avoiding issue with norm == 0 + if (L_2 == 0) + L_2 = std::numeric_limits::epsilon(); + if (!normalize_coordinates) { + return std::sqrt(L_2); + } else { + return std::sqrt(L_2 / n); + } +} + +template +double L_inf_error(size_t n, const T *original_data, const T *decompressed_data, + enum error_bound_type mode) { + double error_L_inf_norm = 0; + for (size_t i = 0; i < n; ++i) { + double temp = fabs(original_data[i] - decompressed_data[i]); + if (temp > error_L_inf_norm) + error_L_inf_norm = temp; + } + if (mode == error_bound_type::ABS) { + return error_L_inf_norm; + } else if (mode == error_bound_type::REL) { + return error_L_inf_norm / L_inf_norm(n, original_data); + } else { + return 0; + } +} + +template +double L_2_error(std::vector shape, const T *original_data, + const T *decompressed_data, enum error_bound_type mode, + bool normalize_coordinates) { + SIZE n = 1; + for (DIM d = 0; d < shape.size(); d++) + n *= shape[d]; + T *const error = new T[n]; + for (size_t i = 0; i < n; ++i) { + T temp = fabs(original_data[i] - decompressed_data[i]); + error[i] = temp; + } + double org_norm = L_2_norm(shape, original_data, normalize_coordinates); + double err_norm = L_2_norm(shape, error, normalize_coordinates); + delete[] error; + + if (mode == error_bound_type::ABS) { + return err_norm; + } else if (mode == error_bound_type::REL) { + return err_norm / org_norm; + } else { + return 0; + } +} + +template +double MSE(size_t n, const T *original_data, const T *decompressed_data) { + double mse = 0; + for (size_t i = 0; i < n; ++i) { + double temp = fabs(original_data[i] - decompressed_data[i]); + mse += temp * temp; + } + return mse / n; +} + +template +double PSNR(size_t n, const T *original_data, const T *decompressed_data) { + double mse = MSE(n, original_data, decompressed_data); + double max = 0, min = std::numeric_limits::max(); + for (size_t i = 0; i < n; ++i) { + if (max < original_data[i]) + max = original_data[i]; + if (min > original_data[i]) + min = original_data[i]; + } + double range = max - min; + return 20 * std::log10(range / std::sqrt(mse)); +} + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Utilities/ProjectionMode.h b/include/mgard-x-160/Utilities/ProjectionMode.h new file mode 100644 index 000000000..f5b19a4c2 --- /dev/null +++ b/include/mgard-x-160/Utilities/ProjectionMode.h @@ -0,0 +1,52 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + */ + +#ifndef MGARD_X_160_UTILITIES_PROJECTION_MODE_H +#define MGARD_X_160_UTILITIES_PROJECTION_MODE_H + +#include + +#include "../RuntimeX/Utilities/Exceptions.h" +#include "Types.h" + +namespace mgard_x_160 { + +// Resolves Auto against the error-bound norm s, then validates the result: +// Hierarchical only supports L-infinity error control (its reconstruction is +// a partition-of-unity prolongation, so it only bounds the max error, not the +// L_2 norm). This is the single choke point both Compressor and +// HybridHierarchyCompressor call, so one Config field controls both. +template +inline compression_projection_mode_type +resolve_projection_mode(compression_projection_mode_type mode, T s) { + if (mode == compression_projection_mode_type::Auto) { + mode = s == std::numeric_limits::infinity() + ? compression_projection_mode_type::Hierarchical + : compression_projection_mode_type::Orthogonal; + } + if (mode == compression_projection_mode_type::Hierarchical && + s != std::numeric_limits::infinity()) { + throw ProcessingException( + "the hierarchical basis only supports L-infinity error control"); + } + if (mode != compression_projection_mode_type::Orthogonal && + mode != compression_projection_mode_type::Hierarchical) { + throw ProcessingException("unknown compression projection mode"); + } + return mode; +} + +// The boolean the (de)compose/quantize kernels actually consume. MGARD 1.6.0 +// had only the orthogonal (L2-projection) basis, for every norm, and did not +// record the basis in its header, so its data is always reconstructed with it. +template +inline bool infer_orthogonal_projection(compression_projection_mode_type mode, + T s) { + return true; +} + +} // namespace mgard_x_160 + +#endif diff --git a/include/mgard-x-160/Utilities/Types.h b/include/mgard-x-160/Utilities/Types.h new file mode 100644 index 000000000..c081de271 --- /dev/null +++ b/include/mgard-x-160/Utilities/Types.h @@ -0,0 +1,133 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#ifndef MGARD_X_160_UTILITIES_TYPES +#define MGARD_X_160_UTILITIES_TYPES + +// Types defined here are for compression + +#include +#include +#include + +namespace mgard_x_160 { + +enum class decomposition_type : uint8_t { MultiDim, SingleDim, Hybrid }; + +// Edge length of the block-local transform block used by the hybrid +// (BlockMGARD) hierarchy. The 8 -> 5 coarsening in the block-local refactor +// and the ROI block grid are both built on this. Compile-time today; recorded +// in the file header so a future change stays readable. +constexpr uint64_t MGARDX_HYBRID_LOCAL_BLOCK_SIZE = 8; + +// Coarse edge length one block-local level produces: 8 fine nodes coarsen to +// the 5 nodes {0, 2, 4, 6, 7} per dimension. +constexpr uint64_t MGARDX_HYBRID_LOCAL_COARSE_SIZE = 5; + +// Number of coefficients one block-local block contributes per level: +// 8^D - 5^D (387 in 3D, 39 in 2D, 3 in 1D). The coefficient array is a +// row-major sequence of blocks, so `index / this` is the block index -- which +// is how the ROI per-block quantizers are addressed. +constexpr uint64_t hybrid_local_coeff_per_block(uint64_t num_dims) { + uint64_t fine = 1, coarse = 1; + for (uint64_t d = 0; d < num_dims; d++) { + fine *= MGARDX_HYBRID_LOCAL_BLOCK_SIZE; + coarse *= MGARDX_HYBRID_LOCAL_COARSE_SIZE; + } + return fine - coarse; +} + +// Transform basis used by the (de)compose step, shared by the plain +// Compressor and the hybrid (BlockMGARD) HybridHierarchyCompressor via +// Config::projection_mode -- one setting controls both. +// +// - Auto (default): resolved from the error-bound norm at (de)compress time +// -- Hierarchical under an L-infinity bound, Orthogonal otherwise. See +// resolve_projection_mode in Utilities/ProjectionMode.h. +// - Orthogonal / Hierarchical: forced explicitly. Hierarchical still only +// supports L-infinity error control; requesting it under any other bound +// throws. Unlike kernel fusion, the resolved basis changes the coefficient +// stream, so it is recorded in the compressed-file metadata for the hybrid +// path. +enum class compression_projection_mode_type : uint8_t { + Auto, + Orthogonal, + Hierarchical +}; + +enum class processor_type : uint8_t { + CPU, + GPU_CUDA, + X_SERIAL, + X_OPENMP, + X_CUDA, + X_HIP, + X_SYCL +}; + +enum class error_bound_type : uint8_t { REL, ABS }; +enum class norm_type : uint8_t { L_Inf, L_2 }; +enum class lossless_type : uint8_t { + Huffman, + Huffman_LZ4, // Huffman, then the portable LZ4 backend (Lossless/LZ4/) + Huffman_Zstd, + CPU_Lossless, + BlockDelta, + // Portable LZ4 applied directly to the (raw signed) quantized integer stream, + // with no Huffman entropy stage. Self-contained like BlockDelta. + LZ4, + // Zero-run-length encoding (RLE0) of the quantized stream, then a + // byte-alphabet rANS entropy stage on the resulting (counts, symbols) blob. + // Breaks Huffman's ~1-bit/symbol ratio floor. Self-contained like LZ4. + ZeroRLE_Rans, + // Symbol-alphabet rANS: same model as Huffman (outlier separation + dict_size + // histogram) but rANS entropy coding instead of Huffman codes -> always <= + // Huffman size (fractional bits), uncapped. Self-contained. + SymbolRans +}; + +// Encoding variant for the BlockDelta lossless backend (mirrors cuSZp): +// Fixed - fixed-length encoding only (no delta) +// Delta - per-block delta + zigzag + fixed-length encoding +// Outlier - Delta plus per-block outlier peeling (large values stored aside) +enum class block_delta_mode_type : uint8_t { Fixed, Delta, Outlier }; + +enum class data_type : uint8_t { Float, Double }; +enum class data_structure_type : uint8_t { + Cartesian_Grid_Uniform, + Cartesian_Grid_Non_Uniform +}; + +enum class endiness_type : uint8_t { Little_Endian, Big_Endian }; + +enum class coordinate_location : uint8_t { Embedded, External }; + +enum class domain_decomposition_type : uint8_t { MaxDim, Block, Variable }; + +enum class operation_type : uint8_t { Compression, MDR }; + +enum class bitplane_encoding_type : uint8_t { GroupedBitplaneEncoding }; + +enum class compress_status_type : uint8_t { + Success, + Failure, + OutputTooLargeFailure, + NotSupportHigherNumberOfDimensionsFailure, + NotSupportDataTypeFailure, + BackendNotAvailableFailure +}; + +enum class compressor_type : uint8_t { MGARD, ZFP }; + +} // namespace mgard_x_160 + +#include +#include +#include +#include + +#endif diff --git a/include/mgard-x/CompressionHighLevel/CMakeLists.txt b/include/mgard-x/CompressionHighLevel/CMakeLists.txt index 9bc8a00b9..31358a1d8 100644 --- a/include/mgard-x/CompressionHighLevel/CMakeLists.txt +++ b/include/mgard-x/CompressionHighLevel/CMakeLists.txt @@ -1,5 +1,6 @@ list(APPEND MGARD_X_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/CompressionHighLevel.h + ${CMAKE_CURRENT_SOURCE_DIR}/DataCompatibility.h ${CMAKE_CURRENT_SOURCE_DIR}/CompressionHighLevel.hpp ${CMAKE_CURRENT_SOURCE_DIR}/DomainDecomposer.hpp ${CMAKE_CURRENT_SOURCE_DIR}/ErrorToleranceCalculator.hpp diff --git a/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp b/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp index 801d2f4a9..ee55c76c0 100644 --- a/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp +++ b/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp @@ -28,6 +28,7 @@ #include "../DomainDecomposer/DomainDecomposer.hpp" #include "../Metadata/Metadata.hpp" +#include "DataCompatibility.h" #include "compress_x.hpp" #ifndef MGARD_X_COMPRESSION_HIGH_LEVEL_API_HPP @@ -1053,6 +1054,13 @@ template void release_cache() { release_cache<5, T, DeviceType>(); } +template +bool compressed_by_v160(const void *compressed_data) { + Metadata meta; + meta.DeserializeVersion((SERIALIZED_TYPE *)compressed_data); + return meta.WrittenByV160OrEarlier(); +} + template enum compress_status_type release_cache() { release_cache(); release_cache(); diff --git a/include/mgard-x/CompressionHighLevel/DataCompatibility.h b/include/mgard-x/CompressionHighLevel/DataCompatibility.h new file mode 100644 index 000000000..d08d3fac7 --- /dev/null +++ b/include/mgard-x/CompressionHighLevel/DataCompatibility.h @@ -0,0 +1,47 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: October 5, 2026 + */ + +#ifndef MGARD_X_DATA_COMPATIBILITY_H +#define MGARD_X_DATA_COMPATIBILITY_H + +#include + +#include "../Config/Config.h" + +namespace mgard_x { + +// Whether compressed_data was written by MGARD 1.6.x or earlier, read from the +// software version in its header. Such streams are decompressed by the +// separate MGARD 1.6.0-compatible pipeline (include/mgard-x-160, +// src/mgard-x-160) when the library is built with +// MGARD_ENABLE_DATA_COMPATIBILITY. +template +bool compressed_by_v160(const void *compressed_data); + +namespace v160 { + +// Entry points of the MGARD 1.6.0-compatible decompression pipeline. Same +// contracts as the corresponding mgard_x::decompress overloads; only defined +// when the library is built with MGARD_ENABLE_DATA_COMPATIBILITY. +template +enum compress_status_type +decompress(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, Config config, bool output_pre_allocated); + +template +enum compress_status_type +decompress(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, data_type &dtype, + std::vector &shape, Config config, bool output_pre_allocated); + +template enum compress_status_type release_cache(); + +} // namespace v160 + +} // namespace mgard_x + +#endif diff --git a/include/mgard-x/Metadata/Metadata.hpp b/include/mgard-x/Metadata/Metadata.hpp index 1e34e3b46..f45c75ade 100644 --- a/include/mgard-x/Metadata/Metadata.hpp +++ b/include/mgard-x/Metadata/Metadata.hpp @@ -89,6 +89,12 @@ struct MetadataBase { std::vector Serialize(); void Deserialize(const std::vector &serialized_data); + // Reads only the software/file versions. Unlike Deserialize it does not + // validate the rest of the header, so it works on streams this version can + // no longer fully parse (e.g. to route them to a compatibility decoder). + void DeserializeVersion(const std::vector &serialized_data); + // Whether the stream was written by MGARD 1.6.x or earlier. + bool WrittenByV160OrEarlier() const; protected: static uint64_t SerializePreambleSize(); @@ -259,6 +265,18 @@ template struct Metadata : MetadataBase { DeviceRuntime::SyncQueue(0); MetadataBase::Deserialize(data_h); } + void DeserializeVersion(const SERIALIZED_TYPE *serialized_data) { + std::vector data_h(MetadataBase::SerializePreambleSize()); + Mem::Copy1D(data_h.data(), serialized_data, data_h.size(), 0); + DeviceRuntime::SyncQueue(0); + auto data_h_iter = data_h.cbegin(); + std::uint64_t total_size = MetadataBase::DeserializeSize(data_h_iter); + + data_h.resize(total_size); + Mem::Copy1D(data_h.data(), serialized_data, total_size, 0); + DeviceRuntime::SyncQueue(0); + MetadataBase::DeserializeVersion(data_h); + } }; bool verify(const void *compressed_data, size_t compressed_size); diff --git a/include/mgard-x/Utilities/Types.h b/include/mgard-x/Utilities/Types.h index e4caf284e..d19f34ef4 100644 --- a/include/mgard-x/Utilities/Types.h +++ b/include/mgard-x/Utilities/Types.h @@ -118,7 +118,10 @@ enum class compress_status_type : uint8_t { OutputTooLargeFailure, NotSupportHigherNumberOfDimensionsFailure, NotSupportDataTypeFailure, - BackendNotAvailableFailure + BackendNotAvailableFailure, + // The data was written by MGARD 1.6.x or earlier and this build lacks + // MGARD_ENABLE_DATA_COMPATIBILITY. + DataCompatibilityNotEnabledFailure }; enum class compressor_type : uint8_t { MGARD, ZFP }; diff --git a/src/mgard-x-160/CMakeLists.txt b/src/mgard-x-160/CMakeLists.txt new file mode 100644 index 000000000..4edfdcd7b --- /dev/null +++ b/src/mgard-x-160/CMakeLists.txt @@ -0,0 +1,13 @@ +add_subdirectory (Config) +add_subdirectory (Hierarchy) +add_subdirectory (DataRefactoring) +add_subdirectory (CompressionLowLevel) +add_subdirectory (CompressionHighLevel) +add_subdirectory (Metadata) +add_subdirectory (RuntimeX) +set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) +set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) +set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) +set(MGARD_X_HIP_SRC ${MGARD_X_HIP_SRC} PARENT_SCOPE) +set(MGARD_X_SYCL_SRC ${MGARD_X_SYCL_SRC} PARENT_SCOPE) +set(MGARD_X_SRC ${MGARD_X_SRC} PARENT_SCOPE) diff --git a/src/mgard-x-160/CompressionHighLevel/Bridge.cpp.in b/src/mgard-x-160/CompressionHighLevel/Bridge.cpp.in new file mode 100644 index 000000000..33e1c9b0e --- /dev/null +++ b/src/mgard-x-160/CompressionHighLevel/Bridge.cpp.in @@ -0,0 +1,103 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: October 5, 2026 + */ + +// Connects the public mgard_x API to the MGARD 1.6.0-compatible decompression +// pipeline in namespace mgard_x_160 (see mgard-x/CompressionHighLevel/ +// DataCompatibility.h). The two pipelines have identical but distinct types, +// so the Config and the results are converted at this boundary. + +#include "mgard-x/CompressionHighLevel/DataCompatibility.h" +#include "mgard-x-160/CompressionHighLevel/CompressionHighLevel.hpp" + +namespace mgard_x { +namespace v160 { + +namespace { + +template struct DeviceTypeV160; +template <> struct DeviceTypeV160 { + using type = mgard_x_160::@DEVICE_TYPE@; +}; + +template To convert_enum(From value) { + return static_cast(static_cast(value)); +} + +// MGARD 1.6.0 decompression took the data layout from the header plus a few +// settings from the caller's Config. Start from the 1.6.0 defaults (see +// src/mgard-x-160/Config/Config.cpp) and take from the caller only the +// settings 1.6.0 also took from it and whose meaning is unchanged. Everything +// else keeps its 1.6.0 default rather than this version's (e.g. the level cap +// is unlimited, not num_global_refactoring_level's default of 3). +mgard_x_160::Config convert_config(const Config &c) { + mgard_x_160::Config v; + v.dev_type = convert_enum(c.dev_type); + v.dev_id = c.dev_id; + v.log_level = c.log_level; + v.auto_pin_host_buffers = c.auto_pin_host_buffers; + v.auto_cache_release = c.auto_cache_release; + v.cpu_mode = convert_enum(c.cpu_mode); + v.compressor = convert_enum(c.compressor); + v.normalize_coordinates = c.normalize_coordinates; + v.adjust_shape = c.adjust_shape; + v.max_memory_footprint = c.max_memory_footprint; + v.estimate_outlier_ratio = c.estimate_outlier_ratio; + return v; +} + +} // namespace + +template +enum compress_status_type +decompress(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, Config config, bool output_pre_allocated) { + using V160 = typename DeviceTypeV160::type; + return convert_enum(mgard_x_160::decompress( + compressed_data, compressed_size, decompressed_data, + convert_config(config), output_pre_allocated)); +} + +template +enum compress_status_type +decompress(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, data_type &dtype, + std::vector &shape, Config config, bool output_pre_allocated) { + using V160 = typename DeviceTypeV160::type; + mgard_x_160::data_type dtype_v160; + std::vector shape_v160; + enum compress_status_type status = + convert_enum(mgard_x_160::decompress( + compressed_data, compressed_size, decompressed_data, dtype_v160, + shape_v160, convert_config(config), output_pre_allocated)); + dtype = convert_enum(dtype_v160); + shape.assign(shape_v160.begin(), shape_v160.end()); + return status; +} + +template enum compress_status_type release_cache() { + using V160 = typename DeviceTypeV160::type; + return convert_enum( + mgard_x_160::release_cache()); +} + +// clang-format off +template enum compress_status_type +decompress(const void *compressed_data, + size_t compressed_size, + void *&decompressed_data, Config config, + bool output_pre_allocated); +template enum compress_status_type +decompress(const void *compressed_data, + size_t compressed_size, + void *&decompressed_data, data_type &dtype, + std::vector &shape, Config config, + bool output_pre_allocated); +template enum compress_status_type release_cache(); +// clang-format on + +} // namespace v160 +} // namespace mgard_x diff --git a/src/mgard-x-160/CompressionHighLevel/CMakeLists.txt b/src/mgard-x-160/CompressionHighLevel/CMakeLists.txt new file mode 100644 index 000000000..5cd1b8f2a --- /dev/null +++ b/src/mgard-x-160/CompressionHighLevel/CMakeLists.txt @@ -0,0 +1,10 @@ +# MGARD 1.6.0 data compatibility: decompression only, reached through Bridge. +MgardXGenerateSourceAllDevices("Decompress") +MgardXGenerateSourceAllDevices("Bridge") +set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) +set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) +set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) +set(MGARD_X_HIP_SRC ${MGARD_X_HIP_SRC} PARENT_SCOPE) +set(MGARD_X_SYCL_SRC ${MGARD_X_SYCL_SRC} PARENT_SCOPE) + +set(MGARD_X_SRC ${MGARD_X_SRC} PARENT_SCOPE) diff --git a/src/mgard-x-160/CompressionHighLevel/Decompress.cpp.in b/src/mgard-x-160/CompressionHighLevel/Decompress.cpp.in new file mode 100644 index 000000000..679a58b78 --- /dev/null +++ b/src/mgard-x-160/CompressionHighLevel/Decompress.cpp.in @@ -0,0 +1,39 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/CompressionHighLevel/CompressionHighLevel.hpp" +// clang-format off +namespace mgard_x_160 { + +template enum compress_status_type decompress<@DEVICE_TYPE@>(const void *compressed_data, + size_t compressed_size, + void *&decompressed_data, + Config config, + bool output_pre_allocated); + +template enum compress_status_type decompress<@DEVICE_TYPE@>(const void *compressed_data, + size_t compressed_size, + void *&decompressed_data, + bool output_pre_allocated); + +template enum compress_status_type +decompress<@DEVICE_TYPE@>(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, data_type &dtype, + std::vector &shape, Config config, + bool output_pre_allocated); + +template enum compress_status_type +decompress<@DEVICE_TYPE@>(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, data_type &dtype, + std::vector &shape, bool output_pre_allocated); + + +template enum compress_status_type +release_cache<@DEVICE_TYPE@>(); + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/CompressionLowLevel/CMakeLists.txt b/src/mgard-x-160/CompressionLowLevel/CMakeLists.txt new file mode 100644 index 000000000..3a95257c8 --- /dev/null +++ b/src/mgard-x-160/CompressionLowLevel/CMakeLists.txt @@ -0,0 +1,6 @@ +MgardXGenerateSourceAllCombinations("Compressor") +set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) +set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) +set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) +set(MGARD_X_HIP_SRC ${MGARD_X_HIP_SRC} PARENT_SCOPE) +set(MGARD_X_SYCL_SRC ${MGARD_X_SYCL_SRC} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mgard-x-160/CompressionLowLevel/Compressor.cpp.in b/src/mgard-x-160/CompressionLowLevel/Compressor.cpp.in new file mode 100644 index 000000000..4d3706dc1 --- /dev/null +++ b/src/mgard-x-160/CompressionLowLevel/Compressor.cpp.in @@ -0,0 +1,18 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/CompressionLowLevel/Compressor.hpp" + +// clang-format off +namespace mgard_x_160 { + +template class Compressor<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>; + +template class CompressorCache<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@, Compressor<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>>; + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/Config/CMakeLists.txt b/src/mgard-x-160/Config/CMakeLists.txt new file mode 100644 index 000000000..ae305d9d3 --- /dev/null +++ b/src/mgard-x-160/Config/CMakeLists.txt @@ -0,0 +1,4 @@ +list(APPEND MGARD_X_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/Config.cpp) + +set(MGARD_X_SRC ${MGARD_X_SRC} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mgard-x-160/Config/Config.cpp b/src/mgard-x-160/Config/Config.cpp new file mode 100644 index 000000000..d8cf0d317 --- /dev/null +++ b/src/mgard-x-160/Config/Config.cpp @@ -0,0 +1,73 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include +#include + +#include "mgard-x-160/Config/Config.h" + +namespace { +// CUDA 12 defaults to lazy module loading, which pulls each kernel's code from +// the (large) fatbin on its first launch. For backends with many/large kernels +// (e.g. the rANS lossless path) those scattered first-launch loads add hundreds +// of ms per process. Eager loading front-loads the module once at context +// creation, making launches instant (wall time unchanged or lower). The CUDA +// runtime reads this env var while registering fatbins in static initializers, +// so we must set it from an early-priority constructor (before those run) -- a +// setenv in main() is too late. Only set when the user hasn't chosen, so it +// stays overridable. +__attribute__((constructor(101))) void mgard_x_set_cuda_module_loading() { + if (std::getenv("CUDA_MODULE_LOADING") == nullptr) { + setenv("CUDA_MODULE_LOADING", "EAGER", 0); + } +} +} // namespace + +namespace mgard_x_160 { + +Config::Config() { + dev_type = device_type::AUTO; + dev_id = 0; + compressor = compressor_type::MGARD; + domain_decomposition = domain_decomposition_type::MaxDim; + decomposition = decomposition_type::MultiDim; + estimate_outlier_ratio = 1.0; + huff_dict_size = 8192; + huff_block_size = 1024 * 20; // MGARD 1.6.0 default + block_delta_block_size = 256; + block_delta_mode = block_delta_mode_type::Delta; + lz4_block_size = 1 << 15; + zstd_compress_level = 3; + normalize_coordinates = true; + lossless = lossless_type::Huffman; + reorder = 0; + log_level = log::ERR; + auto_pin_host_buffers = true; + max_memory_footprint = std::numeric_limits::max(); // no limit + total_num_bitplanes = 32; + block_size = 256; + domain_decomposition_dim = 0; + domain_decomposition_sizes = std::vector(); + mdr_adaptive_resolution = false; + adjust_shape = false; + compress_with_dryrun = false; + num_local_refactoring_level = 1; + // MGARD 1.6.0 default (its max_larget_level): no limit. + num_global_refactoring_level = std::numeric_limits::max(); + enable_roi = false; + projection_mode = compression_projection_mode_type::Auto; + auto_cache_release = false; + cpu_mode = cpu_parallelization_mode::INTER_BLOCK; + mdr_qoi_mode = false; + mdr_qoi_num_variables = 3; + fuse_decompose_quantize = true; + fuse_dequantize_recompose = true; +} + +void Config::apply() { log::level = log_level; } + +} // namespace mgard_x_160 diff --git a/src/mgard-x-160/DataRefactoring/CMakeLists.txt b/src/mgard-x-160/DataRefactoring/CMakeLists.txt new file mode 100644 index 000000000..98d24db79 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/CMakeLists.txt @@ -0,0 +1,7 @@ +add_subdirectory (MultiDimension) +add_subdirectory (SingleDimension) +set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) +set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) +set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) +set(MGARD_X_HIP_SRC ${MGARD_X_HIP_SRC} PARENT_SCOPE) +set(MGARD_X_SYCL_SRC ${MGARD_X_SYCL_SRC} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/CMakeLists.txt b/src/mgard-x-160/DataRefactoring/MultiDimension/CMakeLists.txt new file mode 100644 index 000000000..6f186f86e --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/CMakeLists.txt @@ -0,0 +1,10 @@ +add_subdirectory (Coefficient) +add_subdirectory (Correction) +add_subdirectory (CopyND) +MgardXGenerateSourceAllCombinations("Decompose") +MgardXGenerateSourceAllCombinations("Recompose") +set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) +set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) +set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) +set(MGARD_X_HIP_SRC ${MGARD_X_HIP_SRC} PARENT_SCOPE) +set(MGARD_X_SYCL_SRC ${MGARD_X_SYCL_SRC} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CMakeLists.txt b/src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CMakeLists.txt new file mode 100644 index 000000000..ce0242906 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CMakeLists.txt @@ -0,0 +1,10 @@ +MgardXGenerateSource3D("CalcCoefficients3D") +MgardXGenerateSource3D("CoefficientsRestore3D") +MgardXGenerateSourceND("CalcCoefficientsND") +MgardXGenerateSourceND("CoefficientsRestoreND") + +set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) +set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) +set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) +set(MGARD_X_HIP_SRC ${MGARD_X_HIP_SRC} PARENT_SCOPE) +set(MGARD_X_SYCL_SRC ${MGARD_X_SYCL_SRC} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficients3D.cpp.in b/src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficients3D.cpp.in new file mode 100644 index 000000000..6c3f8b96d --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficients3D.cpp.in @@ -0,0 +1,27 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficients3D.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template void CalcCoefficients3D<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + Hierarchy<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &hierarchy, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> dinput, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &doutput, SIZE l, + int queue_idx); + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsND.cpp.in b/src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsND.cpp.in new file mode 100644 index 000000000..07a3422b6 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsND.cpp.in @@ -0,0 +1,28 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CalcCoefficientsND.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template void CalcCoefficientsND<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + Hierarchy<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &hierarchy, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> dinput1, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> dinput2, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &doutput, SIZE l, + int queue_idx); + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestore3D.cpp.in b/src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestore3D.cpp.in new file mode 100644 index 000000000..ad1bc39db --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestore3D.cpp.in @@ -0,0 +1,27 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestore3D.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template void CoefficientsRestore3D<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + Hierarchy<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &hierarchy, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> dinput, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &doutput, SIZE l, + int queue_idx); + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestoreND.cpp.in b/src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestoreND.cpp.in new file mode 100644 index 000000000..d421e4fde --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestoreND.cpp.in @@ -0,0 +1,28 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/MultiDimension/Coefficient/CoefficientsRestoreND.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template void CoefficientsRestoreND<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + Hierarchy<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &hierarchy, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> dinput1, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> dinput2, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &doutput, SIZE l, + int queue_idx); + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/AddND.cpp.in b/src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/AddND.cpp.in new file mode 100644 index 000000000..e857447d9 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/AddND.cpp.in @@ -0,0 +1,25 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/MultiDimension/CopyND/AddND.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template void AddND<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> dinput, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> doutput, int queue_idx); + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/CMakeLists.txt b/src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/CMakeLists.txt new file mode 100644 index 000000000..81e8c0524 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/CMakeLists.txt @@ -0,0 +1,9 @@ +MgardXGenerateSourceAllCombinations("CopyND") +MgardXGenerateSourceAllCombinations("Copy3D") +MgardXGenerateSourceAllCombinations("AddND") +MgardXGenerateSourceAllCombinations("SubtractND") +set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) +set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) +set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) +set(MGARD_X_HIP_SRC ${MGARD_X_HIP_SRC} PARENT_SCOPE) +set(MGARD_X_SYCL_SRC ${MGARD_X_SYCL_SRC} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/Copy3D.cpp.in b/src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/Copy3D.cpp.in new file mode 100644 index 000000000..c2d632384 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/Copy3D.cpp.in @@ -0,0 +1,25 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/MultiDimension/CopyND/Copy3D.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template void Copy3D<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> dinput, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> doutput, bool padding, int queue_idx); + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/CopyND.cpp.in b/src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/CopyND.cpp.in new file mode 100644 index 000000000..5b0583f7f --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/CopyND.cpp.in @@ -0,0 +1,25 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/MultiDimension/CopyND/CopyND.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template void CopyND<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> dinput, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> doutput, int queue_idx); + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/SubtractND.cpp.in b/src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/SubtractND.cpp.in new file mode 100644 index 000000000..0f42df6c2 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/CopyND/SubtractND.cpp.in @@ -0,0 +1,25 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/MultiDimension/CopyND/SubtractND.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template void SubtractND<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> dinput, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> doutput, int queue_idx); + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/Correction/CMakeLists.txt b/src/mgard-x-160/DataRefactoring/MultiDimension/Correction/CMakeLists.txt new file mode 100644 index 000000000..2051b0f41 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/Correction/CMakeLists.txt @@ -0,0 +1,7 @@ +MgardXGenerateSource3D("CalcCorrection3D") +MgardXGenerateSourceND("CalcCorrectionND") +set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) +set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) +set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) +set(MGARD_X_HIP_SRC ${MGARD_X_HIP_SRC} PARENT_SCOPE) +set(MGARD_X_SYCL_SRC ${MGARD_X_SYCL_SRC} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrection3D.cpp.in b/src/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrection3D.cpp.in new file mode 100644 index 000000000..7273804fe --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrection3D.cpp.in @@ -0,0 +1,27 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrection3D.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template void CalcCorrection3D<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + Hierarchy<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &hierarchy, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> dcoeff, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &dcorrection, SIZE l, + int queue_idx); + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrectionND.cpp.in b/src/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrectionND.cpp.in new file mode 100644 index 000000000..8ec5766d1 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrectionND.cpp.in @@ -0,0 +1,27 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/MultiDimension/Correction/CalcCorrectionND.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template void CalcCorrectionND<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + Hierarchy<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &hierarchy, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> dcoeff, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &dcorrection, SIZE l, + int queue_idx); + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/Decompose.cpp.in b/src/mgard-x-160/DataRefactoring/MultiDimension/Decompose.cpp.in new file mode 100644 index 000000000..ed423a863 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/Decompose.cpp.in @@ -0,0 +1,27 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/MultiDimension/DataRefactoring.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template void decompose<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + Hierarchy<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &hierarchy, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &v, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> w, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> b, + int start_level, int stop_level, bool orthogonal_projection, int queue_idx); + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/MultiDimension/Recompose.cpp.in b/src/mgard-x-160/DataRefactoring/MultiDimension/Recompose.cpp.in new file mode 100644 index 000000000..5f38ba89f --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/MultiDimension/Recompose.cpp.in @@ -0,0 +1,27 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/MultiDimension/DataRefactoring.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace multi_dimension { + +template void recompose<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + Hierarchy<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &hierarchy, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &v, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> w, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> b, + int start_level, int stop_level, bool orthogonal_projection, int queue_idx); + +} // namespace multi_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/SingleDimension/CMakeLists.txt b/src/mgard-x-160/DataRefactoring/SingleDimension/CMakeLists.txt new file mode 100644 index 000000000..4e1a5910a --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/SingleDimension/CMakeLists.txt @@ -0,0 +1,9 @@ +add_subdirectory(Coefficient) +add_subdirectory(Correction) +MgardXGenerateSourceAllCombinations("Decompose") +MgardXGenerateSourceAllCombinations("Recompose") +set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) +set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) +set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) +set(MGARD_X_HIP_SRC ${MGARD_X_HIP_SRC} PARENT_SCOPE) +set(MGARD_X_SYCL_SRC ${MGARD_X_SYCL_SRC} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CMakeLists.txt b/src/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CMakeLists.txt new file mode 100644 index 000000000..5c39daa98 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CMakeLists.txt @@ -0,0 +1,7 @@ +MgardXGenerateSourceAllCombinations("CalcCoefficients") +MgardXGenerateSourceAllCombinations("CoefficientsRestore") +set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) +set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) +set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) +set(MGARD_X_HIP_SRC ${MGARD_X_HIP_SRC} PARENT_SCOPE) +set(MGARD_X_SYCL_SRC ${MGARD_X_SYCL_SRC} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CalcCoefficients.cpp.in b/src/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CalcCoefficients.cpp.in new file mode 100644 index 000000000..7f39a8223 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CalcCoefficients.cpp.in @@ -0,0 +1,27 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CalcCoefficients.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace single_dimension { + +template void CalcCoefficients<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + DIM current_dim, SubArray<1, @DATA_TYPE@, @DEVICE_TYPE@> ratio, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> v, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> coarse, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> coeff, int queue_idx); + +} // namespace single_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CoefficientsRestore.cpp.in b/src/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CoefficientsRestore.cpp.in new file mode 100644 index 000000000..062df0ed4 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CoefficientsRestore.cpp.in @@ -0,0 +1,27 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/SingleDimension/Coefficient/CoefficientsRestore.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace single_dimension { + +template void CoefficientsRestore<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + DIM current_dim, SubArray<1, @DATA_TYPE@, @DEVICE_TYPE@> ratio, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> v, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> coarse, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> coeff, int queue_idx); + +} // namespace single_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/SingleDimension/Correction/CMakeLists.txt b/src/mgard-x-160/DataRefactoring/SingleDimension/Correction/CMakeLists.txt new file mode 100644 index 000000000..b32759b53 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/SingleDimension/Correction/CMakeLists.txt @@ -0,0 +1,6 @@ +MgardXGenerateSourceAllCombinations("CalcCorrection") +set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) +set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) +set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) +set(MGARD_X_HIP_SRC ${MGARD_X_HIP_SRC} PARENT_SCOPE) +set(MGARD_X_SYCL_SRC ${MGARD_X_SYCL_SRC} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/SingleDimension/Correction/CalcCorrection.cpp.in b/src/mgard-x-160/DataRefactoring/SingleDimension/Correction/CalcCorrection.cpp.in new file mode 100644 index 000000000..8ccc409d4 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/SingleDimension/Correction/CalcCorrection.cpp.in @@ -0,0 +1,27 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/SingleDimension/Correction/CalcCorrection.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace single_dimension { + +template void CalcCorrection<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + Hierarchy<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &hierarchy, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &coeff, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &correction, + SIZE curr_dim, SIZE l, int queue_idx); + +} // namespace single_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 +// clang-format off \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/SingleDimension/Decompose.cpp.in b/src/mgard-x-160/DataRefactoring/SingleDimension/Decompose.cpp.in new file mode 100644 index 000000000..efb3e57ec --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/SingleDimension/Decompose.cpp.in @@ -0,0 +1,26 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/SingleDimension/DataRefactoring.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace single_dimension { + +template void decompose<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + Hierarchy<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &hierarchy, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &v, int start_level, int stop_level, + int queue_idx); + +} // namespace single_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/DataRefactoring/SingleDimension/Recompose.cpp.in b/src/mgard-x-160/DataRefactoring/SingleDimension/Recompose.cpp.in new file mode 100644 index 000000000..a9e11d298 --- /dev/null +++ b/src/mgard-x-160/DataRefactoring/SingleDimension/Recompose.cpp.in @@ -0,0 +1,26 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/DataRefactoring/SingleDimension/DataRefactoring.hpp" +// clang-format off +namespace mgard_x_160 { + +namespace data_refactoring { + +namespace single_dimension { + +template void recompose<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>( + Hierarchy<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &hierarchy, + SubArray<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@> &v, int start_level, int stop_level, + int queue_idx); + +} // namespace single_dimension + +} // namespace data_refactoring + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/Hierarchy/CMakeLists.txt b/src/mgard-x-160/Hierarchy/CMakeLists.txt new file mode 100644 index 000000000..9c505a344 --- /dev/null +++ b/src/mgard-x-160/Hierarchy/CMakeLists.txt @@ -0,0 +1,6 @@ +MgardXGenerateSourceAllCombinations("Hierarchy") +set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) +set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) +set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) +set(MGARD_X_HIP_SRC ${MGARD_X_HIP_SRC} PARENT_SCOPE) +set(MGARD_X_SYCL_SRC ${MGARD_X_SYCL_SRC} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mgard-x-160/Hierarchy/Hierarchy.cpp.in b/src/mgard-x-160/Hierarchy/Hierarchy.cpp.in new file mode 100644 index 000000000..1ccfe3036 --- /dev/null +++ b/src/mgard-x-160/Hierarchy/Hierarchy.cpp.in @@ -0,0 +1,15 @@ +/* + * Copyright 2022, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (chenj3@ornl.gov) + * Date: March 17, 2022 + */ + +#include "mgard-x-160/Hierarchy/Hierarchy.hpp" +// clang-format off +namespace mgard_x_160 { + +template class Hierarchy<@NUM_DIM@, @DATA_TYPE@, @DEVICE_TYPE@>; + +} // namespace mgard_x_160 +// clang-format on \ No newline at end of file diff --git a/src/mgard-x-160/Metadata/CMakeLists.txt b/src/mgard-x-160/Metadata/CMakeLists.txt new file mode 100644 index 000000000..1ef19c88c --- /dev/null +++ b/src/mgard-x-160/Metadata/CMakeLists.txt @@ -0,0 +1,4 @@ +list(APPEND MGARD_X_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/Metadata.cpp) + +set(MGARD_X_SRC ${MGARD_X_SRC} PARENT_SCOPE) diff --git a/src/mgard-x-160/Metadata/Metadata.cpp b/src/mgard-x-160/Metadata/Metadata.cpp new file mode 100644 index 000000000..25cea2abc --- /dev/null +++ b/src/mgard-x-160/Metadata/Metadata.cpp @@ -0,0 +1,1035 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Kenneth Moreland (morelandkd@ornl.gov) + * Date: January 5, 2026 + */ + +#include "mgard-x-160/Metadata/Metadata.hpp" +#include "MGARDConfig.hpp" +#include "mgard-x-160/RuntimeX/Utilities/Exceptions.h" +#include "proto/mgard.pb.h" + +#include + +#include + +using mgard_x_160::SERIALIZED_TYPE; + +using SerializedIter = std::vector::const_iterator; + +namespace { + +template bool big_endian() { + static_assert(std::is_integral::value, + "can only check endianness of integral types"); + const Int n = 1; + return not *reinterpret_cast(&n); +} + +// The width of the quantized-coefficient type is a build-time choice +// (QUANTIZED_INT in DataTypes.h) that the reader must agree with, so it is +// recorded in the header rather than assumed. Writing it means the width can +// be changed later without turning existing files into garbage: a mismatch is +// then a clean error instead of a misparse. +mgard::pb::Quantization::Type QuantizationTypeForWidth(std::size_t width) { + switch (width) { + case 1: + return mgard::pb::Quantization::INT8_T; + case 2: + return mgard::pb::Quantization::INT16_T; + case 4: + return mgard::pb::Quantization::INT32_T; + case 8: + return mgard::pb::Quantization::INT64_T; + default: + throw mgard_x_160::InvalidDataException("unsupported quantized integer width."); + } +} + +std::size_t WidthForQuantizationType(mgard::pb::Quantization::Type type) { + switch (type) { + case mgard::pb::Quantization::INT8_T: + return 1; + case mgard::pb::Quantization::INT16_T: + return 2; + case mgard::pb::Quantization::INT32_T: + return 4; + case mgard::pb::Quantization::INT64_T: + return 8; + default: + throw mgard_x_160::InvalidDataException("unrecognized quantization type."); + } +} + +uint32_t ComputeCRC32(const std::vector &data, + std::size_t start = 0) { + // `crc32_z` takes a `z_size_t`. + if (data.size() - start > std::numeric_limits::max()) { + std::cout << mgard_x_160::log::log_err + << "buffer is too large (size would overflow.\n"; + } + uLong crc32_ = crc32_z(0, Z_NULL, 0); + crc32_ = crc32_z(crc32_, static_cast(data.data() + start), + data.size() - start); + return crc32_; +} + +std::vector SerializeProtoBuf(mgard::pb::Header &header) { + size_t header_size = header.ByteSize(); + std::vector header_bytes(header_size); + header.SerializeToArray(header_bytes.data(), header_size); + return header_bytes; +} + +mgard::pb::Header +DeserializeProtoBuf(const std::vector header_bytes, + uint64_t offset) { + uint64_t header_size = header_bytes.size() - offset; + + // The `CodedInputStream` constructor takes an `int`. + if (header_size > std::numeric_limits::max()) { + std::cout << mgard_x_160::log::log_err + << "header is too large (size would overflow).\n"; + } + mgard::pb::Header header; + google::protobuf::io::CodedInputStream stream( + static_cast(header_bytes.data() + + offset), + header_size); + if (not header.ParseFromCodedStream(&stream)) { + throw std::runtime_error("header parsing encountered read or format error"); + } + if (not stream.ConsumedEntireMessage()) { + throw std::runtime_error("part of header left unparsed"); + } + return header; +} + +template +void Serialize(const T &item, std::vector &vec) { + if constexpr (std::is_integral::value) { + T in = item; + for (int i = 0; i < sizeof(T); i++) { + vec.push_back(in); + in = in >> 8; + } + } else { + const SERIALIZED_TYPE *in = + reinterpret_cast(&item); + vec.insert(vec.end(), in, in + sizeof(item)); + } +} + +template void Deserialize(T &item, SerializedIter &iter) { + if constexpr (std::is_integral::value) { + T out = 0; + for (int i = sizeof(T) - 1; i >= 0; i--) { + out = out << 8; + out = out + *(iter + i); + } + item = out; + } else { + SERIALIZED_TYPE *out = reinterpret_cast(&item); + std::copy(iter, iter + sizeof(T), out); + } + iter += sizeof(T); +} + +void SerializeSignature(std::vector &vec) { + for (char c : mgard::SIGNATURE) { + Serialize(c, vec); + } +} + +// This function does not assign to a signature data member. Instead, it just +// checks that the deserialized signature matches `mgard::SIGNATURE`. +void DeserializeSignature(SerializedIter &iter) { + for (const char c : mgard::SIGNATURE) { + char c_; + Deserialize(c_, iter); + if (c_ != c) { + throw mgard_x_160::InvalidDataException("signature mismatch."); + } + } +} + +void SerializeBytes(const std::vector &data, + std::vector &vec) { + vec.insert(vec.end(), data.begin(), data.end()); +} + +// Write `tolerances` into `roi` using whichever of the two encodings is +// smaller. ROI maps are piecewise constant in practice (a few distinct +// tolerances over long runs of blocks), where run-length is dramatically +// smaller; a fully irregular map is bigger under run-length, so both are +// built and the smaller one wins. Both decode to the same vector. +void SerializeROITolerances(const std::vector &tolerances, + mgard::pb::RegionOfInterest &roi) { + mgard::pb::PackedTolerances packed; + google::protobuf::RepeatedField &packed_values = + *packed.mutable_tolerances(); + packed_values.Reserve(tolerances.size()); + for (double t : tolerances) { + packed_values.Add(t); + } + + mgard::pb::RunLengthTolerances run_length; + google::protobuf::RepeatedField &rl_values = + *run_length.mutable_values(); + google::protobuf::RepeatedField &rl_lengths = + *run_length.mutable_run_lengths(); + for (std::size_t i = 0; i < tolerances.size();) { + std::size_t j = i; + // Bit-exact comparison: the decoded tolerances must reproduce the + // compressor's quantization steps exactly, so runs may only merge values + // that are identical, not merely close. + while (j < tolerances.size() && tolerances[j] == tolerances[i]) { + j++; + } + rl_values.Add(tolerances[i]); + rl_lengths.Add(static_cast(j - i)); + i = j; + } + + if (run_length.ByteSizeLong() < packed.ByteSizeLong()) { + *roi.mutable_run_length() = run_length; + } else { + *roi.mutable_packed() = packed; + } +} + +// Inverse of SerializeROITolerances. +std::vector +DeserializeROITolerances(const mgard::pb::RegionOfInterest &roi) { + std::vector tolerances; + if (roi.has_packed()) { + const google::protobuf::RepeatedField &values = + roi.packed().tolerances(); + tolerances.assign(values.begin(), values.end()); + } else if (roi.has_run_length()) { + const mgard::pb::RunLengthTolerances &run_length = roi.run_length(); + if (run_length.values_size() != run_length.run_lengths_size()) { + throw mgard_x_160::InvalidDataException( + "ROI tolerance map has mismatched run values and run lengths."); + } + std::size_t total = 0; + for (int i = 0; i < run_length.run_lengths_size(); i++) { + total += run_length.run_lengths(i); + } + tolerances.reserve(total); + for (int i = 0; i < run_length.values_size(); i++) { + tolerances.insert(tolerances.end(), run_length.run_lengths(i), + run_length.values(i)); + } + } else { + throw mgard_x_160::InvalidDataException( + "ROI tolerance map is present but carries no tolerance encoding."); + } + return tolerances; +} + +} // anonymous namespace + +namespace mgard_x_160 { + +void MetadataBase::InitializeConfig(Config &config) { + config.domain_decomposition = ddtype; + config.domain_decomposition_dim = domain_decomposed_dim; + config.decomposition = decomposition; + config.lossless = ltype; + config.reorder = reorder; + // Only overwrite a backend's parameters when the metadata actually carries + // them (non-zero). A file compressed with a non-Huffman backend leaves the + // Huffman fields at 0; copying those zeros would make + // HuffmanWorkspace::resize divide by a zero block size. Leaving config + // defaults intact avoids that (the unused backend's workspace is harmlessly + // sized with defaults). + if (huff_dict_size != 0) { + config.huff_dict_size = huff_dict_size; + } + if (huff_block_size != 0) { + config.huff_block_size = huff_block_size; + } + if (block_delta_block_size != 0) { + config.block_delta_block_size = block_delta_block_size; + } + // The decomposition has to be rebuilt with the levels it was made with. + // Streams from before the level cap was recorded used all levels. + config.num_global_refactoring_level = + has_num_global_refactoring_level + ? (SIZE)num_global_refactoring_level + : std::numeric_limits::max(); + // The hybrid parameters are only meaningful for a hybrid file, and + // Deserialize refuses to produce a hybrid file without them, so a Hybrid + // decomposition here always carries a full set. For non-hybrid files leave + // the caller's Config alone -- those fields are unused and overwriting them + // with zeros would break a subsequent hybrid compression through the same + // Config. + if (decomposition == decomposition_type::Hybrid) { + config.num_local_refactoring_level = (SIZE)hybrid_num_local_levels; + config.num_global_refactoring_level = (SIZE)hybrid_num_global_levels; + config.projection_mode = hybrid_projection_mode; + config.enable_roi = hybrid_enable_roi; + if (hybrid_enable_roi) { + config.roi_tolerance_map = hybrid_roi_tolerance_map; + } + } +} + +void MetadataBase::PrintSummary() { + std::cout << "=======Metadata Summary=======\n"; + std::cout << "Signature: "; + for (const char c : mgard::SIGNATURE) + std::cout << c; + std::cout << "\n"; + std::cout << "MGARD version: " << (int)software_version[0] << "." + << (int)software_version[1] << "." << (int)software_version[2] + << "\n"; + std::cout << "File format version: " << (int)file_version[0] << "." + << (int)file_version[1] << "." << (int)file_version[2] << "\n"; + std::cout << "Metadata size: " << metadata_size << "\n"; + std::cout << "Metadata crc32: " << metadata_crc32 << "\n"; + std::cout << "Endiness: "; + if (etype == endiness_type::Big_Endian) { + std::cout << "Big Endian\n"; + } else { + std::cout << "Little Endian\n"; + } + std::cout << "Data type: "; + if (dtype == data_type::Float) { + std::cout << "Float\n"; + } else if (dtype == data_type::Double) { + std::cout << "Double\n"; + } + std::cout << "Topology: "; + if (dstype == data_structure_type::Cartesian_Grid_Uniform) { + std::cout << "Uniform Grid\n"; + } else if (dstype == data_structure_type::Cartesian_Grid_Non_Uniform) { + std::cout << "Non-uniform Grid\n"; + } + std::cout << "Shape: "; + for (uint64_t &c : shape) + std::cout << c << " "; + std::cout << "\n"; + std::cout << "Function Decomposition: "; + if (decomposition == decomposition_type::MultiDim) { + std::cout << "MultiDim\n"; + } else if (decomposition == decomposition_type::SingleDim) { + std::cout << "SingleDim\n"; + } else if (decomposition == decomposition_type::Hybrid) { + std::cout << "Hybrid\n"; + std::cout << "Local refactoring levels: " << hybrid_num_local_levels + << "\n"; + std::cout << "Global refactoring levels: " << hybrid_num_global_levels + << "\n"; + std::cout << "Local block size: " << hybrid_local_block_size << "\n"; + std::cout << "Projection: " + << (hybrid_projection_mode == + compression_projection_mode_type::Hierarchical + ? "hierarchical" + : "orthogonal") + << "\n"; + std::cout << "ROI: "; + if (hybrid_enable_roi) { + std::cout << "enabled (" << hybrid_roi_tolerance_map.size() + << " level-0 blocks"; + if (!hybrid_roi_block_dimensions.empty()) { + std::cout << ", grid"; + for (uint64_t n : hybrid_roi_block_dimensions) + std::cout << " " << n; + } + std::cout << ")\n"; + } else { + std::cout << "disabled\n"; + } + } + std::cout << "Domain Decomposition: "; + if (domain_decomposed) { + if (ddtype == domain_decomposition_type::MaxDim) { + std::cout << "MaxDim\n"; + } else if (ddtype == domain_decomposition_type::Variable) { + std::cout << "Variable\n"; + } else { + std::cout << "Block\n"; + } + std::cout << "Decomposed Dim: " << domain_decomposed_dim << "\n"; + std::cout << "Decomposed Size: " << domain_decomposed_size << "\n"; + } else { + std::cout << "No\n"; + } + std::cout << "Error bound mode: "; + if (ebtype == error_bound_type::REL) { + std::cout << "REL\n"; + } else if (ebtype == error_bound_type::ABS) { + std::cout << "ABS\n"; + } + std::cout << "Norm type: "; + if (ntype == norm_type::L_Inf) { + std::cout << "L_Inf\n"; + } else if (ntype == norm_type::L_2) { + std::cout << "L_2\n"; + } + std::cout << "Norm: " << norm << "\n"; + std::cout << "tol: " << tol << "\n"; + std::cout << "s: " << s << "\n"; + + std::cout << "Lossless: "; + if (ltype == mgard_x_160::lossless_type::Huffman) { + std::cout << "Huffman\n"; + std::cout << "Huffman dictionary size: " << huff_dict_size << "\n"; + std::cout << "Huffman block size: " << huff_block_size << "\n"; + } else if (ltype == mgard_x_160::lossless_type::Huffman_LZ4) { + std::cout << "Huffman_LZ4\n"; + std::cout << "Huffman dictionary size: " << huff_dict_size << "\n"; + std::cout << "Huffman block size: " << huff_block_size << "\n"; + } else if (ltype == mgard_x_160::lossless_type::LZ4) { + std::cout << "LZ4\n"; + } else if (ltype == mgard_x_160::lossless_type::Huffman_Zstd) { + std::cout << "Huffman_Zstd\n"; + std::cout << "Huffman dictionary size: " << huff_dict_size << "\n"; + std::cout << "Huffman block size: " << huff_block_size << "\n"; + } else if (ltype == mgard_x_160::lossless_type::CPU_Lossless) { + std::cout << "CPU_Lossless\n"; + } else if (ltype == mgard_x_160::lossless_type::BlockDelta) { + std::cout << "BlockDelta\n"; + std::cout << "BlockDelta block size: " << block_delta_block_size << "\n"; + } + + std::cout << "Backend: "; + if (ptype == processor_type::X_SERIAL) { + std::cout << "X_SERIAL\n"; + } else if (ptype == processor_type::X_CUDA) { + std::cout << "X_OPENMP\n"; + } else if (ptype == processor_type::X_OPENMP) { + std::cout << "X_CUDA\n"; + } else if (ptype == processor_type::X_HIP) { + std::cout << "X_HIP\n"; + } else if (ptype == processor_type::X_SYCL) { + std::cout << "X_SYCL\n"; + } +} + +std::uint64_t MetadataBase::SerializePreambleSize() { + std::uint64_t preamble_size = 0; + preamble_size += mgard::SIGNATURE.size(); + preamble_size += sizeof(uint64_t); // header size + preamble_size += sizeof(uint32_t); // crc32 size + return preamble_size; +} + +std::vector MetadataBase::Serialize() { + mgard::pb::Header header; + + { // Version Number + software_version[0] = MGARD_VERSION_MAJOR; + software_version[1] = MGARD_VERSION_MINOR; + software_version[2] = MGARD_VERSION_PATCH; + + mgard::pb::VersionNumber &mgard_version_number = + *header.mutable_mgard_version(); + mgard_version_number.set_major_(MGARD_VERSION_MAJOR); + mgard_version_number.set_minor_(MGARD_VERSION_MINOR); + mgard_version_number.set_patch_(MGARD_VERSION_PATCH); + + file_version[0] = MGARD_FILE_VERSION_MAJOR; + file_version[1] = MGARD_FILE_VERSION_MINOR; + file_version[2] = MGARD_FILE_VERSION_PATCH; + + mgard::pb::VersionNumber &format_version_number = + *header.mutable_file_format_version(); + mgard_version_number.set_major_(MGARD_FILE_VERSION_MAJOR); + mgard_version_number.set_minor_(MGARD_FILE_VERSION_MINOR); + mgard_version_number.set_patch_(MGARD_FILE_VERSION_PATCH); + } + + { // Domain + mgard::pb::Domain &domain = *header.mutable_domain(); + domain.set_topology(mgard::pb::Domain::CARTESIAN_GRID); + mgard::pb::CartesianGridTopology &cartesian_grid_topology = + *domain.mutable_cartesian_grid_topology(); + cartesian_grid_topology.set_dimension(total_dims); + google::protobuf::RepeatedField &shape_ = + *cartesian_grid_topology.mutable_shape(); + shape_.Resize(total_dims, 0); + std::copy(shape.begin(), shape.end(), shape_.mutable_data()); + mgard::pb::Domain::Geometry geometry; + if (dstype == data_structure_type::Cartesian_Grid_Uniform) { + geometry = mgard::pb::Domain::UNIT_CUBE; + } else { + geometry = mgard::pb::Domain::EXPLICIT_CUBE; + mgard::pb::ExplicitCubeGeometry &explicit_cube_geometry = + *domain.mutable_explicit_cube_geometry(); + google::protobuf::RepeatedField &coordinates_ = + *explicit_cube_geometry.mutable_coordinates(); + + uint64_t totel_len = 0; + for (DIM d = 0; d < total_dims; d++) + totel_len += shape[d]; + coordinates_.Resize(totel_len, 0); + double *p = coordinates_.mutable_data(); + for (DIM d = 0; d < total_dims; d++) { + std::copy(coords[d].begin(), coords[d].end(), p); + p += shape[d]; + } + } + domain.set_geometry(geometry); + } + + { // Dataset + mgard::pb::Dataset &dataset = *header.mutable_dataset(); + if (dtype == data_type::Float) { + dataset.set_type(mgard::pb::Dataset::FLOAT); + } else if (dtype == data_type::Double) { + dataset.set_type(mgard::pb::Dataset::DOUBLE); + } + dataset.set_dimension(1); + } + + { // Error control + mgard::pb::ErrorControl &error = *header.mutable_error_control(); + if (ebtype == error_bound_type::ABS) { + error.set_mode(mgard::pb::ErrorControl::ABSOLUTE); + } else if (ebtype == error_bound_type::REL) { + error.set_mode(mgard::pb::ErrorControl::RELATIVE); + error.set_norm_of_original_data(norm); + } + if (ntype == norm_type::L_Inf) { + error.set_norm(mgard::pb::ErrorControl::L_INFINITY); + error.set_s(s); + } else { + error.set_norm(mgard::pb::ErrorControl::S_NORM); + error.set_s(s); + } + error.set_tolerance(tol); + } + + { // Domain Decomposition + mgard::pb::DomainDecomposition &domainDecomposition = + *header.mutable_domain_decomposition(); + if (domain_decomposed) { + if (ddtype == domain_decomposition_type::MaxDim) { + domainDecomposition.set_method( + mgard::pb::DomainDecomposition::MAX_DIMENSION); + } else if (ddtype == domain_decomposition_type::Block) { + domainDecomposition.set_method(mgard::pb::DomainDecomposition::BLOCK); + } else if (ddtype == domain_decomposition_type::Variable) { + domainDecomposition.set_method( + mgard::pb::DomainDecomposition::VARIABLE); + } + } else { + domainDecomposition.set_method( + mgard::pb::DomainDecomposition::NOOP_METHOD); + } + domainDecomposition.set_decomposition_dimension(domain_decomposed_dim); + domainDecomposition.set_decomposition_size(domain_decomposed_size); + } + + { // Function Decomposition + mgard::pb::FunctionDecomposition &function_decomposition = + *header.mutable_function_decomposition(); + function_decomposition.set_transform( + mgard::pb::FunctionDecomposition::MULTILEVEL_COEFFICIENTS); + if (decomposition == decomposition_type::MultiDim) { + function_decomposition.set_hierarchy( + mgard::pb::FunctionDecomposition::MULTIDIMENSION_WITH_GHOST_NODES); + } else if (decomposition == decomposition_type::SingleDim) { + function_decomposition.set_hierarchy( + mgard::pb::FunctionDecomposition::ONE_DIM_AT_A_TIME_WITH_GHOST_NODES); + } else if (decomposition == decomposition_type::Hybrid) { + function_decomposition.set_hierarchy( + mgard::pb::FunctionDecomposition::HYBRID_HIERARCHY); + // The hybrid hierarchy cannot be reconstructed from the hierarchy enum + // alone: the local/global level counts fix the layout of the decomposed + // buffer and the ROI map fixes the per-block quantization step. Without + // them the decompressor would have to be handed the same parameters out + // of band, and silently produce garbage when it was not. + mgard::pb::HybridHierarchy &hybrid = + *function_decomposition.mutable_hybrid_hierarchy(); + hybrid.set_num_local_levels(hybrid_num_local_levels); + hybrid.set_num_global_levels(hybrid_num_global_levels); + hybrid.set_local_block_size(hybrid_local_block_size); + if (hybrid_projection_mode == + compression_projection_mode_type::Hierarchical) { + hybrid.set_projection(mgard::pb::HybridHierarchy::HIERARCHICAL); + } else if (hybrid_projection_mode == + compression_projection_mode_type::Orthogonal) { + hybrid.set_projection(mgard::pb::HybridHierarchy::ORTHOGONAL); + } else { + throw InvalidDataException( + "cannot serialize an invalid hybrid projection mode."); + } + if (hybrid_enable_roi) { + mgard::pb::RegionOfInterest &roi = *hybrid.mutable_region_of_interest(); + google::protobuf::RepeatedField + &block_dimensions = *roi.mutable_block_dimensions(); + block_dimensions.Reserve(hybrid_roi_block_dimensions.size()); + for (uint64_t n : hybrid_roi_block_dimensions) { + block_dimensions.Add(n); + } + SerializeROITolerances(hybrid_roi_tolerance_map, roi); + } + } + // Not populated by MGARD-X: the reader rebuilds the hierarchy (and hence + // the level count) from the shape, and under domain decomposition there is + // no single value to record. Written for format compatibility only. + function_decomposition.set_l_target(l_target); + if (has_num_global_refactoring_level) { + function_decomposition.set_num_global_refactoring_levels( + num_global_refactoring_level); + } + } + + { // Quantization + mgard::pb::Quantization &quantization = *header.mutable_quantization(); + if (otype == operation_type::Compression) { + quantization.set_method(mgard::pb::Quantization::COEFFICIENTWISE_LINEAR); + quantization.set_bin_widths(mgard::pb::Quantization::PER_COEFFICIENT); + quantization.set_type( + ::QuantizationTypeForWidth(sizeof(mgard_x_160::QUANTIZED_INT))); + quantization.set_big_endian(big_endian()); + if (big_endian()) { + etype = endiness_type::Big_Endian; + } else { + etype = endiness_type::Little_Endian; + } + } else { + quantization.set_method(mgard::pb::Quantization::NOOP_QUANTIZATION); + } + } + + { // MDR + mgard::pb::BitplaneEncoding &bitplane_encoding = + *header.mutable_bitplane_encoding(); + if (otype == operation_type::MDR) { + bitplane_encoding.set_method( + mgard::pb::BitplaneEncoding::GROUPED_BITPLANE_ENCODING); + bitplane_encoding.set_type(mgard::pb::BitplaneEncoding::INT32_T); + bitplane_encoding.set_number_bitplanes(number_bitplanes); + bitplane_encoding.set_big_endian(big_endian()); + } else { + bitplane_encoding.set_method( + mgard::pb::BitplaneEncoding::NOOP_BITPLANE_ENCODING); + } + } + + { // Encoding + mgard::pb::Encoding &encoding = *header.mutable_encoding(); + encoding.set_preprocessor(reorder ? mgard::pb::Encoding::SHUFFLE + : mgard::pb::Encoding::NOOP_PREPROCESSOR); + if (ltype == mgard_x_160::lossless_type::Huffman) { + encoding.set_compressor(mgard::pb::Encoding::X_HUFFMAN); + encoding.set_huffman_dictionary_size(huff_dict_size); + encoding.set_huffman_block_size(huff_block_size); + } else if (ltype == mgard_x_160::lossless_type::Huffman_LZ4) { + encoding.set_compressor(mgard::pb::Encoding::X_HUFFMAN_LZ4); + encoding.set_huffman_dictionary_size(huff_dict_size); + encoding.set_huffman_block_size(huff_block_size); + } else if (ltype == mgard_x_160::lossless_type::LZ4) { + encoding.set_compressor(mgard::pb::Encoding::X_LZ4); + } else if (ltype == mgard_x_160::lossless_type::Huffman_Zstd) { + encoding.set_compressor(mgard::pb::Encoding::X_HUFFMAN_ZSTD); + encoding.set_huffman_dictionary_size(huff_dict_size); + encoding.set_huffman_block_size(huff_block_size); + } else if (ltype == mgard_x_160::lossless_type::CPU_Lossless) { + encoding.set_compressor(mgard::pb::Encoding::CPU_HUFFMAN_ZSTD); + } else if (ltype == mgard_x_160::lossless_type::BlockDelta) { + encoding.set_compressor(mgard::pb::Encoding::X_BLOCK_DELTA); + encoding.set_block_delta_block_size(block_delta_block_size); + } else if (ltype == mgard_x_160::lossless_type::SymbolRans) { + encoding.set_compressor(mgard::pb::Encoding::X_SYMBOL_RANS); + encoding.set_huffman_dictionary_size(huff_dict_size); + } else if (ltype == mgard_x_160::lossless_type::ZeroRLE_Rans) { + encoding.set_compressor(mgard::pb::Encoding::X_ZERORLE_RANS); + } + } + + { // Device + mgard::pb::Device &device = *header.mutable_device(); + if (ptype == processor_type::X_SERIAL) { + device.set_backend(mgard::pb::Device::X_SERIAL); + } else if (ptype == processor_type::X_OPENMP) { + device.set_backend(mgard::pb::Device::X_OPENMP); + } else if (ptype == processor_type::X_CUDA) { + device.set_backend(mgard::pb::Device::X_CUDA); + } else if (ptype == processor_type::X_HIP) { + device.set_backend(mgard::pb::Device::X_HIP); + } else if (ptype == processor_type::X_SYCL) { + device.set_backend(mgard::pb::Device::X_SYCL); + } + } + + // Serialize protobuf + std::vector header_bytes = SerializeProtoBuf(header); + uint64_t header_size = header_bytes.size(); + uint32_t header_crc32 = ComputeCRC32(header_bytes); + + metadata_size = SerializePreambleSize() + header_size; + + // start serializing + std::vector serialized_data; + serialized_data.reserve(metadata_size); + + ::SerializeSignature(serialized_data); + ::Serialize(header_size, serialized_data); + ::Serialize(header_crc32, serialized_data); + ::SerializeBytes(header_bytes, serialized_data); + + return serialized_data; +} + +uint64_t MetadataBase::DeserializeSize( + std::vector::const_iterator &iter) { + uint64_t header_size = 0; + ::DeserializeSignature(iter); + ::Deserialize(header_size, iter); + + header_size += SerializePreambleSize(); + + return header_size; +} + +void MetadataBase::Deserialize( + const std::vector &serialized_data) { + metadata_size = serialized_data.size(); + + SerializedIter iter = serialized_data.cbegin(); + DeserializeSize(iter); + + uint32_t header_crc32 = 0; + ::Deserialize(header_crc32, iter); + + // Actual serialized data is after header starting at index offset. + uint64_t offset = std::distance(serialized_data.begin(), iter); + + if (header_crc32 != ComputeCRC32(serialized_data, offset)) { + throw InvalidDataException("header CRC32 mismatch."); + } + + mgard::pb::Header header = DeserializeProtoBuf(serialized_data, offset); + + { // Version Number + const mgard::pb::VersionNumber mgard_version_number = + header.mgard_version(); + software_version[0] = mgard_version_number.major_(); + software_version[1] = mgard_version_number.minor_(); + software_version[2] = mgard_version_number.patch_(); + if (software_version[0] > MGARD_VERSION_MAJOR) { + throw InvalidDataException("MGARD version mismatch."); + } + + const mgard::pb::VersionNumber format_version_number = + header.file_format_version(); + file_version[0] = format_version_number.major_(); + file_version[1] = format_version_number.minor_(); + file_version[2] = format_version_number.patch_(); + if (file_version[0] > MGARD_FILE_VERSION_MAJOR) { + throw InvalidDataException("MGARD file format version mismatch."); + } + } + + { // Domain + const mgard::pb::Domain &domain = header.domain(); + const mgard::pb::CartesianGridTopology cartesian_grid_topology = + domain.cartesian_grid_topology(); + total_dims = cartesian_grid_topology.dimension(); + const google::protobuf::RepeatedField shape_ = + cartesian_grid_topology.shape(); + if (total_dims != shape_.size()) { + throw InvalidDataException("grid shape does not match given dimension."); + } + shape = std::vector(total_dims); + std::copy(shape_.begin(), shape_.end(), shape.begin()); + + const mgard::pb::Domain::Geometry geometry = domain.geometry(); + if (geometry == mgard::pb::Domain::UNIT_CUBE) { + dstype = data_structure_type::Cartesian_Grid_Uniform; + } else if (geometry == mgard::pb::Domain::EXPLICIT_CUBE) { + dstype = data_structure_type::Cartesian_Grid_Non_Uniform; + const mgard::pb::ExplicitCubeGeometry explicit_cube_geometry = + domain.explicit_cube_geometry(); + const google::protobuf::RepeatedField coordinates = + explicit_cube_geometry.coordinates(); + uint64_t totel_len = 0; + for (DIM d = 0; d < total_dims; d++) + totel_len += shape[d]; + if (totel_len != coordinates.size()) { + throw InvalidDataException( + "mismatch between number of node coordinates and grid shape."); + } + using It = google::protobuf::RepeatedField::const_iterator; + It p = coordinates.begin(); + coords = std::vector>(total_dims); + for (size_t d = 0; d < total_dims; d++) { + const It q = p + shape[d]; + coords[d] = std::vector(shape[d]); + std::copy(p, q, coords[d].begin()); + p = q; + } + assert(p == coordinates.end()); + } + } + + { // Dataset + const mgard::pb::Dataset dataset = header.dataset(); + if (dataset.type() == mgard::pb::Dataset::FLOAT) { + dtype = data_type::Float; + } else if (dataset.type() == mgard::pb::Dataset::DOUBLE) { + dtype = data_type::Double; + } + assert(dataset.dimension() == 1); + } + + { // Error control + const mgard::pb::ErrorControl error = header.error_control(); + if (error.mode() == mgard::pb::ErrorControl::ABSOLUTE) { + ebtype = error_bound_type::ABS; + } else if (error.mode() == mgard::pb::ErrorControl::RELATIVE) { + ebtype = error_bound_type::REL; + norm = error.norm_of_original_data(); + } + + if (error.norm() == mgard::pb::ErrorControl::L_INFINITY) { + ntype = norm_type::L_Inf; + s = std::numeric_limits::infinity(); + } else if (error.norm() == mgard::pb::ErrorControl::S_NORM) { + ntype = norm_type::L_2; + s = error.s(); + } + tol = error.tolerance(); + } + + { // Domain Decomposition + const mgard::pb::DomainDecomposition domainDecomposition = + header.domain_decomposition(); + if (domainDecomposition.method() != + mgard::pb::DomainDecomposition::NOOP_METHOD) { + domain_decomposed = true; + if (domainDecomposition.method() == + mgard::pb::DomainDecomposition::MAX_DIMENSION) { + ddtype = domain_decomposition_type::MaxDim; + } else if (domainDecomposition.method() == + mgard::pb::DomainDecomposition::BLOCK) { + ddtype = domain_decomposition_type::Block; + } else if (domainDecomposition.method() == + mgard::pb::DomainDecomposition::VARIABLE) { + ddtype = domain_decomposition_type::Variable; + } + + domain_decomposed_dim = domainDecomposition.decomposition_dimension(); + domain_decomposed_size = domainDecomposition.decomposition_size(); + } else { + domain_decomposed = false; + } + } + + { // Function Decomposition + const mgard::pb::FunctionDecomposition function_decomposition = + header.function_decomposition(); + assert(function_decomposition.transform() == + mgard::pb::FunctionDecomposition::MULTILEVEL_COEFFICIENTS); + if (function_decomposition.hierarchy() == + mgard::pb::FunctionDecomposition::MULTIDIMENSION_WITH_GHOST_NODES) { + decomposition = decomposition_type::MultiDim; + } else if (function_decomposition.hierarchy() == + mgard::pb::FunctionDecomposition:: + ONE_DIM_AT_A_TIME_WITH_GHOST_NODES) { + decomposition = decomposition_type::SingleDim; + } else if (function_decomposition.hierarchy() == + mgard::pb::FunctionDecomposition::HYBRID_HIERARCHY) { + decomposition = decomposition_type::Hybrid; + if (!function_decomposition.has_hybrid_hierarchy()) { + // Written by a build from before the hybrid parameters were added to + // the header. Its level counts and ROI map only ever existed in the + // caller's Config, so we cannot reconstruct it here. Failing loudly is + // the point: silently falling back to the Config defaults is what + // produced wrong output without any error. + throw InvalidDataException( + "this file uses the hybrid hierarchy but predates the hybrid " + "parameters being recorded in the header; it can only be " + "decompressed by supplying the original local/global level counts " + "and ROI tolerance map through Config."); + } + const mgard::pb::HybridHierarchy &hybrid = + function_decomposition.hybrid_hierarchy(); + hybrid_num_local_levels = hybrid.num_local_levels(); + hybrid_num_global_levels = hybrid.num_global_levels(); + hybrid_local_block_size = hybrid.local_block_size(); + switch (hybrid.projection()) { + case mgard::pb::HybridHierarchy::HIERARCHICAL: + hybrid_projection_mode = compression_projection_mode_type::Hierarchical; + break; + case mgard::pb::HybridHierarchy::PROJECTION_UNSPECIFIED: + case mgard::pb::HybridHierarchy::ORTHOGONAL: + // A missing field denotes a legacy BlockMGARD stream, all of which + // used the orthogonal basis. + hybrid_projection_mode = compression_projection_mode_type::Orthogonal; + break; + default: + throw InvalidDataException( + "hybrid hierarchy records an unknown projection mode."); + } + if (hybrid_num_local_levels == 0 && hybrid_num_global_levels == 0) { + throw InvalidDataException( + "hybrid hierarchy records zero local and zero global levels."); + } + if (hybrid_local_block_size != MGARDX_HYBRID_LOCAL_BLOCK_SIZE) { + throw InvalidDataException( + "hybrid hierarchy was written with local block size " + + std::to_string(hybrid_local_block_size) + + ", but this build of MGARD-X only implements block size " + + std::to_string(MGARDX_HYBRID_LOCAL_BLOCK_SIZE) + "."); + } + hybrid_enable_roi = hybrid.has_region_of_interest(); + if (hybrid_enable_roi) { + const mgard::pb::RegionOfInterest &roi = hybrid.region_of_interest(); + const google::protobuf::RepeatedField + &block_dimensions = roi.block_dimensions(); + hybrid_roi_block_dimensions.assign(block_dimensions.begin(), + block_dimensions.end()); + hybrid_roi_tolerance_map = ::DeserializeROITolerances(roi); + if (hybrid_roi_block_dimensions.size() != total_dims) { + throw InvalidDataException( + "ROI block grid has a different dimension than the data."); + } + std::size_t expected = 1; + for (uint64_t n : hybrid_roi_block_dimensions) { + expected *= n; + } + if (hybrid_roi_tolerance_map.size() != expected) { + throw InvalidDataException( + "ROI tolerance map holds " + + std::to_string(hybrid_roi_tolerance_map.size()) + + " entries but its block grid describes " + + std::to_string(expected) + "."); + } + } + } else { + throw InvalidDataException( + "this decomposition hierarchy mismatch the hierarchy used " + "in MGARD-X."); + } + l_target = function_decomposition.l_target(); + has_num_global_refactoring_level = + function_decomposition.has_num_global_refactoring_levels(); + num_global_refactoring_level = + has_num_global_refactoring_level + ? function_decomposition.num_global_refactoring_levels() + : 0; + } + + { // Quantization + const mgard::pb::Quantization quantization = header.quantization(); + if (quantization.method() != mgard::pb::Quantization::NOOP_QUANTIZATION) { + assert(quantization.bin_widths() == + mgard::pb::Quantization::PER_COEFFICIENT); + // Checked rather than asserted: this decides how the payload is read, so + // a mismatch must fail in release builds too. It fires when a file + // written by a build with a different QUANTIZED_INT width is read. + const std::size_t file_width = + ::WidthForQuantizationType(quantization.type()); + if (file_width != sizeof(mgard_x_160::QUANTIZED_INT)) { + throw InvalidDataException( + "this file quantizes to " + std::to_string(file_width * 8) + + "-bit integers, but this build of MGARD-X uses " + + std::to_string(sizeof(mgard_x_160::QUANTIZED_INT) * 8) + "-bit."); + } + if (quantization.big_endian() != big_endian()) { + throw InvalidDataException( + "this file was written with the opposite endianness."); + } + if (big_endian()) { + etype = endiness_type::Big_Endian; + } else { + etype = endiness_type::Little_Endian; + } + } + } + + { // MDR + const mgard::pb::BitplaneEncoding bitplane_encoding = + header.bitplane_encoding(); + if (bitplane_encoding.method() != + mgard::pb::BitplaneEncoding::NOOP_BITPLANE_ENCODING) { + number_bitplanes = bitplane_encoding.number_bitplanes(); + assert(bitplane_encoding.big_endian() == big_endian()); + } + } + + { + const mgard::pb::Quantization quantization = header.quantization(); + const mgard::pb::BitplaneEncoding bitplane_encoding = + header.bitplane_encoding(); + if (quantization.method() != mgard::pb::Quantization::NOOP_QUANTIZATION && + bitplane_encoding.method() != + mgard::pb::BitplaneEncoding::NOOP_BITPLANE_ENCODING || + quantization.method() == mgard::pb::Quantization::NOOP_QUANTIZATION && + bitplane_encoding.method() == + mgard::pb::BitplaneEncoding::NOOP_BITPLANE_ENCODING) { + throw InvalidDataException("cannot determine whether this is compressed " + "or refactored data."); + } + } + + if (otype == operation_type::MDR) { // MDR + mgard::pb::BitplaneEncoding &bitplane_encoding = + *header.mutable_bitplane_encoding(); + bitplane_encoding.set_method( + mgard::pb::BitplaneEncoding::GROUPED_BITPLANE_ENCODING); + bitplane_encoding.set_type(mgard::pb::BitplaneEncoding::INT32_T); + bitplane_encoding.set_number_bitplanes(number_bitplanes); + bitplane_encoding.set_big_endian(big_endian()); + } + + { // Encoding + const mgard::pb::Encoding encoding = header.encoding(); + reorder = + encoding.preprocessor() == mgard::pb::Encoding::SHUFFLE ? 1 : 0; + if (encoding.compressor() == mgard::pb::Encoding::X_HUFFMAN) { + ltype = mgard_x_160::lossless_type::Huffman; + huff_dict_size = encoding.huffman_dictionary_size(); + huff_block_size = encoding.huffman_block_size(); + } else if (encoding.compressor() == mgard::pb::Encoding::X_HUFFMAN_LZ4) { + ltype = mgard_x_160::lossless_type::Huffman_LZ4; + huff_dict_size = encoding.huffman_dictionary_size(); + huff_block_size = encoding.huffman_block_size(); + } else if (encoding.compressor() == mgard::pb::Encoding::X_LZ4) { + ltype = mgard_x_160::lossless_type::LZ4; + } else if (encoding.compressor() == mgard::pb::Encoding::X_HUFFMAN_ZSTD) { + ltype = mgard_x_160::lossless_type::Huffman_Zstd; + huff_dict_size = encoding.huffman_dictionary_size(); + huff_block_size = encoding.huffman_block_size(); + } else if (encoding.compressor() == mgard::pb::Encoding::CPU_HUFFMAN_ZSTD) { + ltype = mgard_x_160::lossless_type::CPU_Lossless; + } else if (encoding.compressor() == mgard::pb::Encoding::X_BLOCK_DELTA) { + ltype = mgard_x_160::lossless_type::BlockDelta; + block_delta_block_size = encoding.block_delta_block_size(); + } else if (encoding.compressor() == mgard::pb::Encoding::X_SYMBOL_RANS) { + ltype = mgard_x_160::lossless_type::SymbolRans; + huff_dict_size = encoding.huffman_dictionary_size(); + } else if (encoding.compressor() == mgard::pb::Encoding::X_ZERORLE_RANS) { + ltype = mgard_x_160::lossless_type::ZeroRLE_Rans; + } else { + throw InvalidDataException("unknown lossless compressor type."); + } + } + + { // Device + const mgard::pb::Device device = header.device(); + if (device.backend() == mgard::pb::Device::X_SERIAL) { + ptype = processor_type::X_SERIAL; + } else if (device.backend() == mgard::pb::Device::X_OPENMP) { + ptype = processor_type::X_OPENMP; + } else if (device.backend() == mgard::pb::Device::X_CUDA) { + ptype = processor_type::X_CUDA; + } else if (device.backend() == mgard::pb::Device::X_HIP) { + ptype = processor_type::X_HIP; + } else if (device.backend() == mgard::pb::Device::X_SYCL) { + ptype = processor_type::X_SYCL; + } else if (device.backend() == mgard::pb::Device::CPU) { + throw InvalidDataException("this data was not compressed with MGARD-X."); + } + } +} + +} // namespace mgard_x_160 diff --git a/src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerCuda.cu b/src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerCuda.cu new file mode 100644 index 000000000..fae43b372 --- /dev/null +++ b/src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerCuda.cu @@ -0,0 +1,17 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ +#include "mgard-x-160/RuntimeX/RuntimeX.h" +namespace mgard_x_160 { + +template void BeginAutoTuning(); +template void EndAutoTuning(); + +AutoTuningTable AutoTuner::autoTuningTable; +bool AutoTuner::ProfileKernels = false; +bool AutoTuner::WriteToTable = false; + +} // namespace mgard_x_160 \ No newline at end of file diff --git a/src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerHip.cpp b/src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerHip.cpp new file mode 100644 index 000000000..5edd3fe24 --- /dev/null +++ b/src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerHip.cpp @@ -0,0 +1,17 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ +#include "mgard-x-160/RuntimeX/RuntimeX.h" +namespace mgard_x_160 { + +template void BeginAutoTuning(); +template void EndAutoTuning(); + +AutoTuningTable AutoTuner::autoTuningTable; +bool AutoTuner::ProfileKernels = false; +bool AutoTuner::WriteToTable = false; + +} // namespace mgard_x_160 \ No newline at end of file diff --git a/src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerOpenmp.cpp b/src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerOpenmp.cpp new file mode 100644 index 000000000..807ee3ab9 --- /dev/null +++ b/src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerOpenmp.cpp @@ -0,0 +1,17 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ +#include "mgard-x-160/RuntimeX/RuntimeX.h" +namespace mgard_x_160 { + +template void BeginAutoTuning(); +template void EndAutoTuning(); + +AutoTuningTable AutoTuner::autoTuningTable; +bool AutoTuner::ProfileKernels = false; +bool AutoTuner::WriteToTable = false; + +} // namespace mgard_x_160 \ No newline at end of file diff --git a/src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSerial.cpp b/src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSerial.cpp new file mode 100644 index 000000000..29b8de03a --- /dev/null +++ b/src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSerial.cpp @@ -0,0 +1,17 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ +#include "mgard-x-160/RuntimeX/RuntimeX.h" +namespace mgard_x_160 { + +template void BeginAutoTuning(); +template void EndAutoTuning(); + +AutoTuningTable AutoTuner::autoTuningTable; +bool AutoTuner::ProfileKernels = false; +bool AutoTuner::WriteToTable = false; + +} // namespace mgard_x_160 \ No newline at end of file diff --git a/src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSycl.cpp b/src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSycl.cpp new file mode 100644 index 000000000..0f9136072 --- /dev/null +++ b/src/mgard-x-160/RuntimeX/AutoTuners/AutoTunerSycl.cpp @@ -0,0 +1,17 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ +#include "mgard-x-160/RuntimeX/RuntimeX.h" +namespace mgard_x_160 { + +template void BeginAutoTuning(); +template void EndAutoTuning(); + +AutoTuningTable AutoTuner::autoTuningTable; +bool AutoTuner::ProfileKernels = false; +bool AutoTuner::WriteToTable = false; + +} // namespace mgard_x_160 \ No newline at end of file diff --git a/src/mgard-x-160/RuntimeX/AutoTuners/CMakeLists.txt b/src/mgard-x-160/RuntimeX/AutoTuners/CMakeLists.txt new file mode 100644 index 000000000..1d34da23f --- /dev/null +++ b/src/mgard-x-160/RuntimeX/AutoTuners/CMakeLists.txt @@ -0,0 +1,32 @@ +if(MGARD_ENABLE_SERIAL) + list(APPEND MGARD_X_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/AutoTunerSerial.cpp) + set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) +endif() +if(MGARD_ENABLE_OPENMP) + list(APPEND MGARD_X_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/AutoTunerOpenmp.cpp) + set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) +endif() +if(MGARD_ENABLE_CUDA) + list(APPEND MGARD_X_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/AutoTunerCuda.cu) + set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) +endif() +if(MGARD_ENABLE_HIP) + list(APPEND MGARD_X_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/AutoTunerHip.cpp) + set(MGARD_X_HIP_SRC ${MGARD_X_HIP_SRC} PARENT_SCOPE) +endif() +if(MGARD_ENABLE_SYCL) + list(APPEND MGARD_X_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/AutoTunerSycl.cpp) + set(MGARD_X_SYCL_SRC ${MGARD_X_SYCL_SRC} PARENT_SCOPE) +endif() +if(MGARD_ENABLE_KOKKOS) + list(APPEND MGARD_X_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/AutoTunerKokkos.cpp) + set(MGARD_X_KOKKOS_SRC ${MGARD_X_KOKKOS_SRC} PARENT_SCOPE) +endif() + +set(MGARD_X_SRC ${MGARD_X_SRC} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mgard-x-160/RuntimeX/CMakeLists.txt b/src/mgard-x-160/RuntimeX/CMakeLists.txt new file mode 100644 index 000000000..a0fc4efbf --- /dev/null +++ b/src/mgard-x-160/RuntimeX/CMakeLists.txt @@ -0,0 +1,9 @@ +add_subdirectory (AutoTuners) +add_subdirectory (DeviceAdapters) +add_subdirectory (Utilities) +set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) +set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) +set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) +set(MGARD_X_HIP_SRC ${MGARD_X_HIP_SRC} PARENT_SCOPE) +set(MGARD_X_SYCL_SRC ${MGARD_X_SYCL_SRC} PARENT_SCOPE) +set(MGARD_X_SRC ${MGARD_X_SRC} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mgard-x-160/RuntimeX/DeviceAdapters/CMakeLists.txt b/src/mgard-x-160/RuntimeX/DeviceAdapters/CMakeLists.txt new file mode 100644 index 000000000..e1d417dc6 --- /dev/null +++ b/src/mgard-x-160/RuntimeX/DeviceAdapters/CMakeLists.txt @@ -0,0 +1,30 @@ +if(MGARD_ENABLE_SERIAL) + list(APPEND MGARD_X_SERIAL_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/DeviceAdapterSerial.cpp) + set(MGARD_X_SERIAL_SRC ${MGARD_X_SERIAL_SRC} PARENT_SCOPE) +endif() +if(MGARD_ENABLE_OPENMP) + list(APPEND MGARD_X_OPENMP_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/DeviceAdapterOpenmp.cpp) + set(MGARD_X_OPENMP_SRC ${MGARD_X_OPENMP_SRC} PARENT_SCOPE) +endif() +if(MGARD_ENABLE_CUDA) + list(APPEND MGARD_X_CUDA_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/DeviceAdapterCuda.cu) + set(MGARD_X_CUDA_SRC ${MGARD_X_CUDA_SRC} PARENT_SCOPE) +endif() +if(MGARD_ENABLE_HIP) + list(APPEND MGARD_X_HIP_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/DeviceAdapterHip.cpp) + set(MGARD_X_HIP_SRC ${MGARD_X_HIP_SRC} PARENT_SCOPE) +endif() +if(MGARD_ENABLE_SYCL) + list(APPEND MGARD_X_SYCL_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/DeviceAdapterSycl.cpp) + set(MGARD_X_SYCL_SRC ${MGARD_X_SYCL_SRC} PARENT_SCOPE) +endif() +if(MGARD_ENABLE_KOKKOS) + list(APPEND MGARD_X_KOKKOS_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/DeviceAdapterKokkos.cpp) + set(MGARD_X_KOKKOS_SRC ${MGARD_X_KOKKOS_SRC} PARENT_SCOPE) +endif() \ No newline at end of file diff --git a/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterCuda.cu b/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterCuda.cu new file mode 100644 index 000000000..586de7ea8 --- /dev/null +++ b/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterCuda.cu @@ -0,0 +1,24 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "mgard-x-160/RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +int cuda_dev_id = 0; +DeviceQueues DeviceRuntime::queues; +DeviceSpecification DeviceRuntime::DeviceSpecs; + +bool DeviceRuntime::SyncAllKernelsAndCheckErrors = false; +bool DeviceRuntime::TimingAllKernels = false; +bool DeviceRuntime::PrintKernelConfig = false; + +template <> bool deviceAvailable() { + return DeviceRuntime::GetDeviceCount() > 0; +} + +} // namespace mgard_x_160 \ No newline at end of file diff --git a/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterHip.cpp b/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterHip.cpp new file mode 100644 index 000000000..742bd0901 --- /dev/null +++ b/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterHip.cpp @@ -0,0 +1,24 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "mgard-x-160/RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +int hip_dev_id = 0; +DeviceQueues DeviceRuntime::queues; +DeviceSpecification DeviceRuntime::DeviceSpecs; + +bool DeviceRuntime::SyncAllKernelsAndCheckErrors = false; +bool DeviceRuntime::TimingAllKernels = false; +bool DeviceRuntime::PrintKernelConfig = false; + +template <> bool deviceAvailable() { + return DeviceRuntime::GetDeviceCount() > 0; +} + +} // namespace mgard_x_160 \ No newline at end of file diff --git a/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.cpp b/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.cpp new file mode 100644 index 000000000..d3599bd52 --- /dev/null +++ b/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.cpp @@ -0,0 +1,23 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "mgard-x-160/RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +int DeviceRuntime::curr_dev_id = 0; +// DeviceQueues DeviceRuntime::queues; +DeviceSpecification DeviceRuntime::DeviceSpecs; + +bool DeviceRuntime::SyncAllKernelsAndCheckErrors = false; +// bool MemoryManager::ReduceMemoryFootprint = false; + +// KernelConfigs AutoTuner::kernelConfigs; +// AutoTuningTable AutoTuner::autoTuningTable; +// bool AutoTuner::ProfileKernels = false; + +} // namespace mgard_x_160 \ No newline at end of file diff --git a/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.cpp b/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.cpp new file mode 100644 index 000000000..2381ff499 --- /dev/null +++ b/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.cpp @@ -0,0 +1,24 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "mgard-x-160/RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +int openmp_dev_id = 0; +DeviceQueues DeviceRuntime::queues; +DeviceSpecification DeviceRuntime::DeviceSpecs; + +bool DeviceRuntime::SyncAllKernelsAndCheckErrors = false; +bool DeviceRuntime::TimingAllKernels = false; +bool DeviceRuntime::PrintKernelConfig = false; + +template <> bool deviceAvailable() { + return DeviceRuntime::GetDeviceCount() > 0; +} + +} // namespace mgard_x_160 \ No newline at end of file diff --git a/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSerial.cpp b/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSerial.cpp new file mode 100644 index 000000000..19d9f7225 --- /dev/null +++ b/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSerial.cpp @@ -0,0 +1,24 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "mgard-x-160/RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +int serial_dev_id = 0; +DeviceQueues DeviceRuntime::queues; +DeviceSpecification DeviceRuntime::DeviceSpecs; + +bool DeviceRuntime::SyncAllKernelsAndCheckErrors = false; +bool DeviceRuntime::TimingAllKernels = false; +bool DeviceRuntime::PrintKernelConfig = false; + +template <> bool deviceAvailable() { + return DeviceRuntime::GetDeviceCount() > 0; +} + +} // namespace mgard_x_160 \ No newline at end of file diff --git a/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSycl.cpp b/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSycl.cpp new file mode 100644 index 000000000..4e24a4e3f --- /dev/null +++ b/src/mgard-x-160/RuntimeX/DeviceAdapters/DeviceAdapterSycl.cpp @@ -0,0 +1,25 @@ +/* + * Copyright 2026, Oak Ridge National Laboratory. + * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs + * Author: Jieyang Chen (jieyang@uoregon.edu) + * Date: September 21, 2026 + */ + +#include "mgard-x-160/RuntimeX/RuntimeX.h" + +namespace mgard_x_160 { + +int sycl_dev_id = 0; +DeviceQueues DeviceRuntime::queues; +DeviceSpecification DeviceRuntime::DeviceSpecs; + +// SyncAllKernelsAndCheckErrors needs to be always ON for SYCL +bool DeviceRuntime::SyncAllKernelsAndCheckErrors = true; +bool DeviceRuntime::TimingAllKernels = false; +bool DeviceRuntime::PrintKernelConfig = false; + +template <> bool deviceAvailable() { + return DeviceRuntime::GetDeviceCount() > 0; +} + +} // namespace mgard_x_160 \ No newline at end of file diff --git a/src/mgard-x-160/RuntimeX/Utilities/CMakeLists.txt b/src/mgard-x-160/RuntimeX/Utilities/CMakeLists.txt new file mode 100644 index 000000000..ed8a1d524 --- /dev/null +++ b/src/mgard-x-160/RuntimeX/Utilities/CMakeLists.txt @@ -0,0 +1,3 @@ +list(APPEND MGARD_X_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/Log.cpp) +set(MGARD_X_SRC ${MGARD_X_SRC} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mgard-x-160/RuntimeX/Utilities/Log.cpp b/src/mgard-x-160/RuntimeX/Utilities/Log.cpp new file mode 100644 index 000000000..dca633655 --- /dev/null +++ b/src/mgard-x-160/RuntimeX/Utilities/Log.cpp @@ -0,0 +1,77 @@ +#include +#include +#include + +#include "mgard-x-160/RuntimeX/Utilities/Log.h" + +using std::string; + +namespace mgard_x_160 { + +namespace log { + +int INFO = 1; +int TIME = 2; +int DBG = 4; +int WARN = 8; +int ERR = 16; + +int level = ERR; + +const string log_null = " "; +const string log_info = "\e[32m[info]\e[0m "; +const string log_time = "\e[34m[time]\e[0m "; +const string log_dbg = "\e[34m[dbg]\e[0m "; +const string log_warn = "\e[31m[WARN]\e[0m "; +const string log_err = "\e[31m[ERR]\e[0m "; + +void info(std::string msg, bool override) { + if (level & INFO || override) { + std::cout << log_info << msg << std::endl; + } +} + +void time(std::string msg, bool override) { + if (level & TIME || override) { + std::cout << log_time << msg << std::endl; + } +} + +void dbg(std::string msg, bool override) { + if (level & DBG || override) { + std::cout << log_dbg << msg << std::endl; + } +} + +void warn(std::string msg, bool override) { + if (level & WARN || override) { + std::cout << log_warn << msg << std::endl; + } +} + +void err(std::string msg, bool override) { + if (level & ERR || override) { + std::cout << log_err << msg << std::endl; + } +} + +// https://stackoverflow.com/a/26080768/8740097 +template void build(std::ostream &o, T t) { o << t << std::endl; } + +template +void build(std::ostream &o, T t, + Args... args) // recursive variadic function +{ + build(o, t); + build(o, args...); +} + +template void print(string log_head, Args... args) { + std::ostringstream oss; + build(oss, args...); + std::cout << log_head << oss.str(); +} + +} // namespace log + +} // namespace mgard_x_160 \ No newline at end of file diff --git a/src/mgard-x/CompressionHighLevel/Decompress.cpp.in b/src/mgard-x/CompressionHighLevel/Decompress.cpp.in index d8dcb1ff7..75b6da954 100644 --- a/src/mgard-x/CompressionHighLevel/Decompress.cpp.in +++ b/src/mgard-x/CompressionHighLevel/Decompress.cpp.in @@ -34,6 +34,7 @@ decompress<@DEVICE_TYPE@>(const void *compressed_data, size_t compressed_size, template enum compress_status_type release_cache<@DEVICE_TYPE@>(); +template bool compressed_by_v160<@DEVICE_TYPE@>(const void *compressed_data); } // namespace mgard_x // clang-format on \ No newline at end of file diff --git a/src/mgard-x/CompressionHighLevel/DynamicAPI.cpp b/src/mgard-x/CompressionHighLevel/DynamicAPI.cpp index 2f578d4b3..4e7e6c050 100644 --- a/src/mgard-x/CompressionHighLevel/DynamicAPI.cpp +++ b/src/mgard-x/CompressionHighLevel/DynamicAPI.cpp @@ -14,6 +14,7 @@ #include "compress_x.hpp" #include "mgard-x/CompressionHighLevel/CompressionHighLevel.h" +#include "mgard-x/CompressionHighLevel/DataCompatibility.h" #include "mgard-x/Config/Config.h" #include "mgard-x/RuntimeX/AutoTuners/AutoTuner.h" #include "mgard-x/RuntimeX/DataTypes.h" @@ -50,6 +51,86 @@ enum device_type auto_detect_device() { return dev_type; } +// Decompression entry shared by the public overloads below: streams written by +// MGARD 1.6.x or earlier go to the separate 1.6.0-compatible pipeline +// (src/mgard-x-160), everything else to this version's. +namespace { + +void report_v160_disabled() { + log::err("this data was compressed by MGARD 1.6.x or earlier; decompressing " + "it requires MGARD built with -DMGARD_ENABLE_DATA_COMPATIBILITY=ON."); +} + +template +enum compress_status_type +decompress_routed(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, Config config, + bool output_pre_allocated) { + if (compressed_by_v160(compressed_data)) { +#if MGARD_ENABLE_DATA_COMPATIBILITY + return v160::decompress(compressed_data, compressed_size, + decompressed_data, config, + output_pre_allocated); +#else + report_v160_disabled(); + return compress_status_type::DataCompatibilityNotEnabledFailure; +#endif + } + return decompress(compressed_data, compressed_size, + decompressed_data, config, + output_pre_allocated); +} + +template +enum compress_status_type +decompress_routed(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, bool output_pre_allocated) { + return decompress_routed(compressed_data, compressed_size, + decompressed_data, Config(), + output_pre_allocated); +} + +template +enum compress_status_type +decompress_routed(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, data_type &dtype, + std::vector &shape, Config config, + bool output_pre_allocated) { + if (compressed_by_v160(compressed_data)) { +#if MGARD_ENABLE_DATA_COMPATIBILITY + return v160::decompress(compressed_data, compressed_size, + decompressed_data, dtype, shape, + config, output_pre_allocated); +#else + report_v160_disabled(); + return compress_status_type::DataCompatibilityNotEnabledFailure; +#endif + } + return decompress(compressed_data, compressed_size, + decompressed_data, dtype, shape, config, + output_pre_allocated); +} + +template +enum compress_status_type +decompress_routed(const void *compressed_data, size_t compressed_size, + void *&decompressed_data, data_type &dtype, + std::vector &shape, bool output_pre_allocated) { + return decompress_routed(compressed_data, compressed_size, + decompressed_data, dtype, shape, + Config(), output_pre_allocated); +} + +template +enum compress_status_type release_cache_routed() { +#if MGARD_ENABLE_DATA_COMPATIBILITY + v160::release_cache(); +#endif + return release_cache(); +} + +} // namespace + enum compress_status_type compress(DIM D, data_type dtype, std::vector shape, double tol, double s, enum error_bound_type mode, const void *original_data, @@ -281,35 +362,35 @@ enum compress_status_type decompress(const void *compressed_data, if (dev_type == device_type::SERIAL) { #if MGARD_ENABLE_SERIAL - return decompress(compressed_data, compressed_size, + return decompress_routed(compressed_data, compressed_size, decompressed_data, config, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP - return decompress(compressed_data, compressed_size, + return decompress_routed(compressed_data, compressed_size, decompressed_data, config, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA - return decompress(compressed_data, compressed_size, decompressed_data, + return decompress_routed(compressed_data, compressed_size, decompressed_data, config, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP - return decompress(compressed_data, compressed_size, decompressed_data, + return decompress_routed(compressed_data, compressed_size, decompressed_data, config, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL - return decompress(compressed_data, compressed_size, decompressed_data, + return decompress_routed(compressed_data, compressed_size, decompressed_data, config, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; @@ -328,35 +409,35 @@ enum compress_status_type decompress(const void *compressed_data, if (dev_type == device_type::SERIAL) { #if MGARD_ENABLE_SERIAL - return decompress(compressed_data, compressed_size, + return decompress_routed(compressed_data, compressed_size, decompressed_data, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP - return decompress(compressed_data, compressed_size, + return decompress_routed(compressed_data, compressed_size, decompressed_data, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA - return decompress(compressed_data, compressed_size, decompressed_data, + return decompress_routed(compressed_data, compressed_size, decompressed_data, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP - return decompress(compressed_data, compressed_size, decompressed_data, + return decompress_routed(compressed_data, compressed_size, decompressed_data, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL - return decompress(compressed_data, compressed_size, decompressed_data, + return decompress_routed(compressed_data, compressed_size, decompressed_data, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; @@ -378,7 +459,7 @@ decompress(const void *compressed_data, size_t compressed_size, if (dev_type == device_type::SERIAL) { #if MGARD_ENABLE_SERIAL - return decompress(compressed_data, compressed_size, + return decompress_routed(compressed_data, compressed_size, decompressed_data, dtype, shape, config, output_pre_allocated); #else @@ -386,7 +467,7 @@ decompress(const void *compressed_data, size_t compressed_size, #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP - return decompress(compressed_data, compressed_size, + return decompress_routed(compressed_data, compressed_size, decompressed_data, dtype, shape, config, output_pre_allocated); #else @@ -394,21 +475,21 @@ decompress(const void *compressed_data, size_t compressed_size, #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA - return decompress(compressed_data, compressed_size, decompressed_data, + return decompress_routed(compressed_data, compressed_size, decompressed_data, dtype, shape, config, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP - return decompress(compressed_data, compressed_size, decompressed_data, + return decompress_routed(compressed_data, compressed_size, decompressed_data, dtype, shape, config, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL - return decompress(compressed_data, compressed_size, decompressed_data, + return decompress_routed(compressed_data, compressed_size, decompressed_data, dtype, shape, config, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; @@ -427,7 +508,7 @@ decompress(const void *compressed_data, size_t compressed_size, if (dev_type == device_type::SERIAL) { #if MGARD_ENABLE_SERIAL - return decompress(compressed_data, compressed_size, + return decompress_routed(compressed_data, compressed_size, decompressed_data, dtype, shape, output_pre_allocated); #else @@ -436,7 +517,7 @@ decompress(const void *compressed_data, size_t compressed_size, } if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP - return decompress(compressed_data, compressed_size, + return decompress_routed(compressed_data, compressed_size, decompressed_data, dtype, shape, output_pre_allocated); #else @@ -444,21 +525,21 @@ decompress(const void *compressed_data, size_t compressed_size, #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA - return decompress(compressed_data, compressed_size, decompressed_data, + return decompress_routed(compressed_data, compressed_size, decompressed_data, dtype, shape, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP - return decompress(compressed_data, compressed_size, decompressed_data, + return decompress_routed(compressed_data, compressed_size, decompressed_data, dtype, shape, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL - return decompress(compressed_data, compressed_size, decompressed_data, + return decompress_routed(compressed_data, compressed_size, decompressed_data, dtype, shape, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; @@ -477,31 +558,31 @@ enum compress_status_type release_cache(Config config) { if (dev_type == device_type::SERIAL) { #if MGARD_ENABLE_SERIAL - return release_cache(); + return release_cache_routed(); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP - return release_cache(); + return release_cache_routed(); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA - return release_cache(); + return release_cache_routed(); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP - return release_cache(); + return release_cache_routed(); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL - return release_cache(); + return release_cache_routed(); #else return compress_status_type::BackendNotAvailableFailure; #endif diff --git a/src/mgard-x/Metadata/Metadata.cpp b/src/mgard-x/Metadata/Metadata.cpp index 823f9fdbf..c9a32fa4c 100644 --- a/src/mgard-x/Metadata/Metadata.cpp +++ b/src/mgard-x/Metadata/Metadata.cpp @@ -687,6 +687,32 @@ uint64_t MetadataBase::DeserializeSize( return header_size; } +void MetadataBase::DeserializeVersion( + const std::vector &serialized_data) { + SerializedIter iter = serialized_data.cbegin(); + DeserializeSize(iter); + + uint32_t header_crc32 = 0; + ::Deserialize(header_crc32, iter); + uint64_t offset = std::distance(serialized_data.begin(), iter); + if (header_crc32 != ComputeCRC32(serialized_data, offset)) { + throw InvalidDataException("header CRC32 mismatch."); + } + + mgard::pb::Header header = DeserializeProtoBuf(serialized_data, offset); + software_version[0] = header.mgard_version().major_(); + software_version[1] = header.mgard_version().minor_(); + software_version[2] = header.mgard_version().patch_(); + file_version[0] = header.file_format_version().major_(); + file_version[1] = header.file_format_version().minor_(); + file_version[2] = header.file_format_version().patch_(); +} + +bool MetadataBase::WrittenByV160OrEarlier() const { + return software_version[0] < 1 || + (software_version[0] == 1 && software_version[1] <= 6); +} + void MetadataBase::Deserialize( const std::vector &serialized_data) { metadata_size = serialized_data.size(); From b18c006fa5aa8dbc6f31b6536fcd886e57a3d3da Mon Sep 17 00:00:00 2001 From: Jieyang Chen Date: Sat, 10 Oct 2026 10:55:39 -0700 Subject: [PATCH 08/11] mgard-x: replace MaxDim/Block/Variable with a generic block domain decomposition The three domain decomposition types become one: the domain is cut into a grid of blocks of a common, possibly non-cubic shape, so a 400x300x200 array with 100x300x200 blocks is the old MaxDim split and cubic blocks the old Block split. Along each dimension block i starts at i * block_shape[d]; a remainder narrower than 3 nodes is merged into the block before it, so every block is a valid hierarchy. Config::domain_decomposition (and block_size, domain_decomposition_dim, domain_decomposition_sizes) is replaced by: - domain_decomposition_mode = Auto (default): one block when the data fits the memory limit; otherwise the largest block extent (the slowest dimension on ties) is halved until the largest block, merged remainder included, fits. It fails if even 3-node blocks do not fit. - domain_decomposition_mode = Manual with domain_decomposition_block_size: extents larger than the data are clamped; compression fails with an error if the size count does not match the dimensions, any extent is below 3, or the largest block does not fit the memory limit. Shape adjustment is rejected in Manual mode. DomainDecomposer is rewritten around the block grid (subdomain_start / subdomain_shape / block_shape replace the per-type branches and dim_subdomain_offset), and its per-dimension copy loops are replaced by one generic N-D box copy: dimensions contiguous on both sides are merged, then the box is a single Copy1D/Copy2D/Copy3D, looping only over dimensions beyond the third. This also fixes the old MaxDim offsets for a dimension other than the slowest, which copied past the end of the array. RuntimeX: MemoryManager::CopyND is renamed Copy2D, and Copy3D is added (cudaMemcpy3DAsync / hipMemcpy3DAsync; a loop of Copy2D on the other backends). SYCL and Kokkos Copy2D now honour the leading dimensions. Format: DomainDecomposition gains repeated block_shape and is written as BLOCK (NOOP when not decomposed). Older MAX_DIMENSION and BLOCK headers are read as the equivalent block shape; VARIABLE is no longer supported. Metadata stores domain_decomposition_block_shape instead of ddtype/dim/size. CLIs: mgard-x and mdr-x take -dd auto|manual and -dd-size ... . The MDR QoI tools always used one subdomain per variable (now Manual blocks of shape[0] / 3) and drop -dd. Tested on SERIAL and CUDA: compression of 1D-4D data with Auto (fitting and memory-limited), MaxDim-like, cubic, merged-remainder, last-dimension, clamped, 2D and 4D blocks (incl. uncompressed blocks) stays within the error bound, also when decompressed standalone; MDR refactor/reconstruct of the same layouts stays within 1e-2/1e-4/1e-6; files with the previous MaxDim and Block headers decode within the bound; Manual-mode errors are reported; unit tests pass. Co-Authored-By: Claude Opus 5.5 --- doc/MGARD-X.md | 10 +- doc/MGARD-format.md | 9 + .../CompressionHighLevel/CPUPipelines.hpp | 8 +- .../CompressionHighLevel.hpp | 28 +- .../CompressionHighLevel/GPUPipelines.hpp | 4 +- .../CompressionHighLevel/ShapeAdjustment.hpp | 15 +- .../CompressionLowLevel/NormCalculator.hpp | 2 +- include/mgard-x/Config/Config.h | 9 +- .../DomainDecomposer/DomainDecomposer.hpp | 1122 ++++++----------- include/mgard-x/MDRHighLevel/MDRHighLevel.hpp | 25 +- .../MDRHighLevel/ReconstructPipeline.hpp | 2 +- include/mgard-x/Metadata/Metadata.hpp | 47 +- .../mgard-x/RuntimeX/DataStructures/Array.hpp | 6 +- .../RuntimeX/DataStructures/SubArrayCopy.hpp | 2 +- .../RuntimeX/DeviceAdapters/DeviceAdapter.h | 8 +- .../DeviceAdapters/DeviceAdapterCuda.h | 52 +- .../DeviceAdapters/DeviceAdapterHip.h | 52 +- .../DeviceAdapters/DeviceAdapterKokkos.h | 30 +- .../DeviceAdapters/DeviceAdapterOpenmp.h | 19 +- .../DeviceAdapters/DeviceAdapterSerial.h | 19 +- .../DeviceAdapters/DeviceAdapterSycl.h | 32 +- .../RuntimeX/Utilities/SubArrayPrinter.hpp | 18 +- include/mgard-x/Utilities/Types.h | 8 +- src/mgard-x/Config/Config.cpp | 6 +- src/mgard-x/Executables/ArgumentParser.h | 22 + src/mgard-x/Executables/mdr-x-qoi.cpp | 41 +- src/mgard-x/Executables/mdr-x.cpp | 53 +- src/mgard-x/Executables/mgard-x.cpp | 57 +- src/mgard-x/Executables/pmdr-x-qoi-io.cpp | 43 +- src/mgard-x/Executables/pmdr-x-qoi.cpp | 43 +- src/mgard-x/Metadata/Metadata.cpp | 80 +- src/mgard.proto | 8 + tests/mgard-x/src/test_HybridMetadata.cpp.in | 3 +- 33 files changed, 809 insertions(+), 1074 deletions(-) diff --git a/doc/MGARD-X.md b/doc/MGARD-X.md index 5e3cb37da..ec2dffd4d 100644 --- a/doc/MGARD-X.md +++ b/doc/MGARD-X.md @@ -115,7 +115,8 @@ An executable ```mgard-x``` will be built after building the MGARD-X library. To + ```-e ``` error bound + ```-s ``` smoothness parameter + ```-r <0|1>``` internal data layout (0: Higher throughput | 1: Higher compression ratio) - + ```-b <0|1>``` domain decomposition type (0: 1D max dimension (default) | 1: N-D block) + + ```-dd ``` domain decomposition mode (auto: decompose only when the data does not fit the memory limit (default) | manual: use the block size given by ```-dd-size```) + + ```-dd-size ... ``` domain decomposition block size, slowest dimension first (manual mode only) + ```-f ``` maximum memory footprint in bytes (if not specify: no limit) + ```-l / --lossless ``` choose lossless compressor: + ```huffman```: Huffman coding @@ -169,9 +170,10 @@ An executable ```mgard-x``` will be built after building the MGARD-X library. To + ```lz4_block_size```: LZ4 parallelized block size (***Default: 32768***) + ```zstd_compress_level```: Zstd compression level (***Default: 3***) + ```normalize_coordinates```: enable or disable coordinate normalization (***Controlled by CMake option***) - + ```domain_decomposition``` (for high-level APIs only): controls how data can be domain decomposed: - + ```mgard_x::domain_decomposition_type::MaxDim```: 1D decomposition along the largest dimension (***Default***) - + ```mgard_x::domain_decomposition_type::Block```: N-D block based decomposition + + ```domain_decomposition_mode``` (for high-level APIs only): controls how data is split into blocks (subdomains) that are processed one at a time. Blocks may have any shape, e.g. a 400x300x200 array with 100x300x200 blocks is split along the slowest dimension only. Along each dimension a remainder narrower than 3 nodes is merged into the block before it. The block shape is recorded in the compressed header, so decompression needs no setting. + + ```mgard_x::domain_decomposition_mode_type::Auto```: decompose only when the data does not fit the memory limit (device memory capped by ```max_memory_footprint```); the largest block extent is then halved until the largest block fits (***Default***) + + ```mgard_x::domain_decomposition_mode_type::Manual```: use ```domain_decomposition_block_size```; compression fails if any block extent is smaller than 3 or a block would not fit the memory limit + + ```domain_decomposition_block_size```: block shape for the manual mode, one extent per dimension, slowest first; extents larger than the data are clamped to it + ```decomposition```: controls multi-level decomposition: + ```mgard_x::decomposition_type::MultiDim```: N-D decomposition (***Default***) + ```mgard_x::decomposition_type::SingleDim```: 1D-at-a-time decomposition diff --git a/doc/MGARD-format.md b/doc/MGARD-format.md index 00f553433..2dac823a4 100644 --- a/doc/MGARD-format.md +++ b/doc/MGARD-format.md @@ -75,11 +75,20 @@ message DomainDecomposition { NOOP_METHOD = 0; MAX_DIMENSION = 1; BLOCK = 2; + VARIABLE = 3; } Method method = 1; + // MAX_DIMENSION (MGARD-X 1.6 and earlier): slabs of decomposition_size + // nodes along dimension decomposition_dimension. uint64 decomposition_dimension = 2; + // BLOCK without block_shape (MGARD-X 1.6 and earlier): cubic blocks of + // decomposition_size nodes per dimension. uint64 decomposition_size = 3; + // BLOCK: the shape of every block, one extent per dimension. Along each + // dimension block i starts at i * block_shape[d]; a remainder of fewer than + // 3 nodes is merged into the block before it. + repeated uint64 block_shape = 4; } message FunctionDecomposition { diff --git a/include/mgard-x/CompressionHighLevel/CPUPipelines.hpp b/include/mgard-x/CompressionHighLevel/CPUPipelines.hpp index c24b7f50d..fa746d994 100644 --- a/include/mgard-x/CompressionHighLevel/CPUPipelines.hpp +++ b/include/mgard-x/CompressionHighLevel/CPUPipelines.hpp @@ -122,7 +122,7 @@ enum compress_status_type compress_pipeline_cpu( for (DIM d = 0; d < D - 1; d++) linearized_width *= device_subdomain_buffer[curr_subdomain_id].shape(d); - MemoryManager::CopyND( + MemoryManager::Copy2D( device_compressed_buffer[curr_subdomain_id].data(), device_subdomain_buffer[curr_subdomain_id].shape(D - 1) * sizeof(T), (Byte *)device_subdomain_buffer[curr_subdomain_id].data(), @@ -168,7 +168,7 @@ enum compress_status_type compress_pipeline_cpu( for (DIM d = 0; d < D - 1; d++) linearized_width *= device_subdomain_buffer[curr_subdomain_id].shape(d); - MemoryManager::CopyND( + MemoryManager::Copy2D( device_compressed_buffer[curr_subdomain_id].data(), device_subdomain_buffer[curr_subdomain_id].shape(D - 1) * sizeof(T), (Byte *)device_subdomain_buffer[curr_subdomain_id].data(), @@ -410,7 +410,7 @@ enum compress_status_type decompress_pipeline_cpu( for (DIM d = 0; d < D - 1; d++) linearized_width *= device_subdomain_buffer[curr_subdomain_id].shape(d); - MemoryManager::CopyND( + MemoryManager::Copy2D( device_subdomain_buffer[curr_subdomain_id].data(), device_subdomain_buffer[curr_subdomain_id].ld(D - 1), (T *)device_compressed_buffer[curr_subdomain_id].data(), @@ -457,7 +457,7 @@ enum compress_status_type decompress_pipeline_cpu( for (DIM d = 0; d < D - 1; d++) linearized_width *= device_subdomain_buffer[curr_subdomain_id].shape(d); - MemoryManager::CopyND( + MemoryManager::Copy2D( device_subdomain_buffer[curr_subdomain_id].data(), device_subdomain_buffer[curr_subdomain_id].ld(D - 1), (T *)device_compressed_buffer[curr_subdomain_id].data(), diff --git a/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp b/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp index ee55c76c0..75c5fd435 100644 --- a/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp +++ b/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp @@ -296,16 +296,14 @@ general_compress_pipeline(std::vector shape, T tol, T s, m.FillForCompression( ebtype, tol, s, norm, config.decomposition, config.lossless, config.huff_dict_size, config.huff_block_size, shape, - domain_decomposer.domain_decomposed(), config.domain_decomposition, - domain_decomposer.domain_decomposed_dim(), - domain_decomposer.domain_decomposed_size()); + domain_decomposer.domain_decomposed(), + domain_decomposer.block_shape()); } else { m.FillForCompression( ebtype, tol, s, norm, config.decomposition, config.lossless, config.huff_dict_size, config.huff_block_size, shape, - domain_decomposer.domain_decomposed(), config.domain_decomposition, - domain_decomposer.domain_decomposed_dim(), - domain_decomposer.domain_decomposed_size(), coords); + domain_decomposer.domain_decomposed(), + domain_decomposer.block_shape(), coords); } uint32_t metadata_size; @@ -356,16 +354,14 @@ general_compress_pipeline(std::vector shape, T tol, T s, m.FillForCompression( ebtype, tol, s, norm, config.decomposition, config.lossless, config.huff_dict_size, config.huff_block_size, shape, - domain_decomposer.domain_decomposed(), config.domain_decomposition, - domain_decomposer.domain_decomposed_dim(), - domain_decomposer.domain_decomposed_size()); + domain_decomposer.domain_decomposed(), + domain_decomposer.block_shape()); } else { m.FillForCompression( ebtype, tol, s, norm, config.decomposition, config.lossless, config.huff_dict_size, config.huff_block_size, shape, - domain_decomposer.domain_decomposed(), config.domain_decomposition, - domain_decomposer.domain_decomposed_dim(), - domain_decomposer.domain_decomposed_size(), coords); + domain_decomposer.domain_decomposed(), + domain_decomposer.block_shape(), coords); } serizalied_meta = m.Serialize(metadata_size); @@ -588,14 +584,14 @@ general_decompress_pipeline(std::vector shape, // Initialize DomainDecomposer DomainDecomposer domain_decomposer; + std::vector block_shape(m.domain_decomposition_block_shape.begin(), + m.domain_decomposition_block_shape.end()); if (m.dstype == data_structure_type::Cartesian_Grid_Uniform) { domain_decomposer = DomainDecomposer( - shape, m.domain_decomposed, m.domain_decomposed_dim, - m.domain_decomposed_size, config); + shape, block_shape, config); } else { domain_decomposer = DomainDecomposer( - shape, m.domain_decomposed, m.domain_decomposed_dim, - m.domain_decomposed_size, config, coords); + shape, block_shape, config, coords); } domain_decomposer.set_original_data((T *)decompressed_data); diff --git a/include/mgard-x/CompressionHighLevel/GPUPipelines.hpp b/include/mgard-x/CompressionHighLevel/GPUPipelines.hpp index df366a56a..90fb243b4 100644 --- a/include/mgard-x/CompressionHighLevel/GPUPipelines.hpp +++ b/include/mgard-x/CompressionHighLevel/GPUPipelines.hpp @@ -145,7 +145,7 @@ enum compress_status_type compress_pipeline_gpu( SIZE linearized_width = 1; for (DIM d = 0; d < D - 1; d++) linearized_width *= device_subdomain_buffer[current_buffer].shape(d); - MemoryManager::CopyND( + MemoryManager::Copy2D( device_compressed_buffer[current_buffer].data(), device_subdomain_buffer[current_buffer].shape(D - 1) * sizeof(T), (Byte *)device_subdomain_buffer[current_buffer].data(), @@ -476,7 +476,7 @@ enum compress_status_type decompress_pipeline_gpu( SIZE linearized_width = 1; for (DIM d = 0; d < D - 1; d++) linearized_width *= device_subdomain_buffer[current_buffer].shape(d); - MemoryManager::CopyND( + MemoryManager::Copy2D( device_subdomain_buffer[current_buffer].data(), device_subdomain_buffer[current_buffer].ld(D - 1), (T *)device_compressed_buffer[current_buffer].data(), diff --git a/include/mgard-x/CompressionHighLevel/ShapeAdjustment.hpp b/include/mgard-x/CompressionHighLevel/ShapeAdjustment.hpp index 2e6bb7e03..fb34dc0e6 100644 --- a/include/mgard-x/CompressionHighLevel/ShapeAdjustment.hpp +++ b/include/mgard-x/CompressionHighLevel/ShapeAdjustment.hpp @@ -42,13 +42,11 @@ template std::vector find_refactors(T n) { template void adjust_shape(std::vector &shape, Config config) { log::info("Using shape adjustment"); - int num_timesteps; - if (config.domain_decomposition == domain_decomposition_type::Variable) { - // If do shape adjustment with variable domain decomposition - // the temporal dim has to be the first dim - assert(config.domain_decomposition_dim == 0); - num_timesteps = shape[0] / config.domain_decomposition_sizes[0]; - shape[0] = config.domain_decomposition_sizes[0]; + // A manual block size refers to the shape the caller gave. + if (config.domain_decomposition_mode == + domain_decomposition_mode_type::Manual) { + throw Exception("Shape adjustment cannot be combined with manual domain " + "decomposition."); } int max_d = max_dim(shape); SIZE max_n = shape[max_d]; @@ -63,9 +61,6 @@ template void adjust_shape(std::vector &shape, Config config) { // std::cout << "multiple " << factors[i] << // " to dim " << min_d << ": " << shape[min_d] << "\n"; } - if (config.domain_decomposition == domain_decomposition_type::Variable) { - shape[0] *= num_timesteps; - } // std::cout << "shape: "; // for (SIZE n : shape) { // std::cout << n << "\n"; diff --git a/include/mgard-x/CompressionLowLevel/NormCalculator.hpp b/include/mgard-x/CompressionLowLevel/NormCalculator.hpp index 3012190a8..02b510ddd 100644 --- a/include/mgard-x/CompressionLowLevel/NormCalculator.hpp +++ b/include/mgard-x/CompressionLowLevel/NormCalculator.hpp @@ -33,7 +33,7 @@ T norm_calculator(Array &original_array, SIZE linearized_width = 1; for (DIM d = 0; d < D - 1; d++) linearized_width *= original_array.shape(d); - MemoryManager::CopyND( + MemoryManager::Copy2D( temp_subarray.data(), original_array.shape(D - 1), original_array.data(), original_array.ld(D - 1), original_array.shape(D - 1), linearized_width, 0); diff --git a/include/mgard-x/Config/Config.h b/include/mgard-x/Config/Config.h index c56b8e462..f3480db19 100644 --- a/include/mgard-x/Config/Config.h +++ b/include/mgard-x/Config/Config.h @@ -11,7 +11,11 @@ struct Config { device_type dev_type; int dev_id; enum compressor_type compressor; - enum domain_decomposition_type domain_decomposition; + enum domain_decomposition_mode_type domain_decomposition_mode; + // Block extent along each dimension, for Manual decomposition. A block may + // be non-cubic (e.g. {100, 300, 200} on 400 x 300 x 200 data splits only the + // first dimension). Extents larger than the data are clamped to it. + std::vector domain_decomposition_block_size; enum decomposition_type decomposition; double estimate_outlier_ratio; SIZE huff_dict_size; @@ -26,9 +30,6 @@ struct Config { bool auto_pin_host_buffers; SIZE max_memory_footprint; SIZE total_num_bitplanes; - SIZE block_size; - SIZE domain_decomposition_dim; - std::vector domain_decomposition_sizes; bool mdr_adaptive_resolution; bool adjust_shape; bool compress_with_dryrun; diff --git a/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp b/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp index 5be08cf7b..93ee29903 100644 --- a/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp +++ b/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp @@ -2,12 +2,15 @@ * Copyright 2026, Oak Ridge National Laboratory. * MGARD-X: MultiGrid Adaptive Reduction of Data Portable across GPUs and CPUs * Author: Jieyang Chen (jieyang@uoregon.edu) - * Date: September 21, 2026 + * Date: October 9, 2026 */ #ifndef MGARD_X_DOMAIN_DECOMPOSER_HPP #define MGARD_X_DOMAIN_DECOMPOSER_HPP +#include +#include + #include "../Config/Config.h" #include "../Hierarchy/Hierarchy.hpp" @@ -18,838 +21,487 @@ enum class subdomain_copy_direction : uint8_t { SubdomainToOriginal }; +// Splits an N-D array into a grid of blocks (subdomains) that are processed +// one at a time. +// +// All blocks share one shape, block_shape. Along each dimension block b starts +// at b * block_shape[d]; the last block takes whatever remains, and a +// remainder narrower than MIN_BLOCK_EXTENT is merged into the block before it, +// so every block is a valid Hierarchy. Subdomain ids enumerate the block grid +// row-major, like the data (last dimension fastest). A single block covering +// the whole array means no decomposition. +// +// Compression picks the block shape from the Config: +// - Auto: one block if the array fits the memory limit; otherwise the largest +// block extent is halved until the largest block fits. +// - Manual: Config::domain_decomposition_block_size, rejected if a block would +// be narrower than MIN_BLOCK_EXTENT or would not fit the memory limit. +// The block shape is recorded in the header, and decompression rebuilds the +// same grid from it. template class DomainDecomposer { public: - size_t EstimateMemoryFootprint(std::vector shape, - bool enable_prefetch) { - size_t estimate_memory_usgae = 0; + // Every block becomes its own Hierarchy, which needs at least this many + // nodes along each dimension. + static constexpr SIZE MIN_BLOCK_EXTENT = 3; - Array<1, T, DeviceType> array_with_pitch({1}); - size_t pitch_size = array_with_pitch.ld(0) * sizeof(T); + DomainDecomposer() {} - size_t input_space = roundup((size_t)shape[D - 1] * sizeof(T), pitch_size); - for (DIM d = 0; d < D - 1; d++) { - input_space *= shape[d]; + // Compression: choose the block shape according to the Config. A non-empty + // `coords` means a non-uniform grid. + DomainDecomposer(std::vector shape, Config config, + std::vector coords = std::vector()) + : shape(shape), config(config), coords(coords), uniform(coords.empty()) { + if (config.domain_decomposition_mode == + domain_decomposition_mode_type::Manual) { + set_block_shape(manual_block_shape()); + } else { + set_block_shape(auto_block_shape()); } + } - SIZE num_elements = 1; - for (int i = 0; i < shape.size(); i++) - num_elements *= shape[i]; - size_t output_space = 0; - output_space += num_elements * sizeof(HUFFMAN_CODE); - output_space += config.estimate_outlier_ratio * sizeof(QUANTIZED_INT); - - estimate_memory_usgae = input_space + output_space; - - log::dbg("Input output space: " + - std::to_string((double)(input_space + output_space) / 1e9) + - " GB"); + // Decompression: rebuild the block grid recorded in the header. + DomainDecomposer(std::vector shape, std::vector block_shape, + Config config, std::vector coords = std::vector()) + : shape(shape), config(config), coords(coords), uniform(coords.empty()) { + if (block_shape.size() != D) { + throw InvalidDataException("domain decomposition block shape has " + + std::to_string(block_shape.size()) + + " dimensions, the data has " + + std::to_string(D) + "."); + } + set_block_shape(block_shape); + } - using HierarchyType = typename OperatorType::HierarchyType; - size_t hierarchy_space = - HierarchyType::EstimateMemoryFootprint(shape, config); - estimate_memory_usgae += hierarchy_space; - log::dbg("Hierarchy space: " + - std::to_string((double)hierarchy_space / 1e9) + " GB"); + // ---------------------------------------------------------------- the grid - // For prefetching - if (enable_prefetch) { - estimate_memory_usgae *= 2; + // Number of blocks of extent `block` along an extent of `n` nodes, with a + // too-narrow remainder merged into the last full block. + static SIZE num_blocks(SIZE n, SIZE block) { + SIZE count = (n - 1) / block + 1; + SIZE remainder = n % block; + if (count > 1 && remainder != 0 && remainder < MIN_BLOCK_EXTENT) { + count--; } - estimate_memory_usgae += - OperatorType::EstimateMemoryFootprint(shape, config); - log::dbg("Operation space: " + - std::to_string( - (double)OperatorType::EstimateMemoryFootprint(shape, config) / - 1e9) + - " GB"); - - return estimate_memory_usgae; + return count; } - bool need_domain_decomposition(std::vector shape, - bool enable_prefetch) { - // using Cache = CompressorCache; - size_t estm = EstimateMemoryFootprint(shape, enable_prefetch); - size_t aval = - std::min((SIZE)DeviceRuntime::GetAvailableMemory(), - config.max_memory_footprint); - log::dbg("Estimated memory usage: " + std::to_string((double)estm / 1e9) + - "GB, Available: " + std::to_string((double)aval / 1e9) + "GB"); - bool need = estm >= aval; - return need; + // Extent of block i along an extent of `n` nodes; the last block takes the + // rest, merged remainder included. + static SIZE block_extent(SIZE n, SIZE block, SIZE i) { + SIZE count = num_blocks(n, block); + return i + 1 < count ? block : n - (count - 1) * block; } - std::vector dim_num_subdomain() { - std::vector _dim_num_subdomain(D, 1); - if (config.domain_decomposition == domain_decomposition_type::MaxDim || - config.domain_decomposition == domain_decomposition_type::Variable) { - _dim_num_subdomain[_domain_decomposed_dim] = this->_num_subdomains; - } else if (config.domain_decomposition == - domain_decomposition_type::Block) { - for (int d = D - 1; d >= 0; d--) { - _dim_num_subdomain[d] = (shape[d] - 1) / _domain_decomposed_size + 1; - } - } - return _dim_num_subdomain; - } + bool domain_decomposed() { return _num_subdomains > 1; } + + SIZE num_subdomains() { return _num_subdomains; } - std::vector dim_subdomain_id(int subdomain_id) { - std::vector _dim_num_subdomain = dim_num_subdomain(); - std::vector _dim_subdomain_id(D); + // Common block shape (clamped to the data); what the header records. + std::vector block_shape() { return _block_shape; } + + // Position of a subdomain in the block grid, per dimension. + std::vector subdomain_grid_index(SIZE subdomain_id) { + check_subdomain_id(subdomain_id); + std::vector index(D); for (int d = D - 1; d >= 0; d--) { - _dim_subdomain_id[d] = subdomain_id % _dim_num_subdomain[d]; - subdomain_id /= _dim_num_subdomain[d]; + index[d] = subdomain_id % _blocks_per_dim[d]; + subdomain_id /= _blocks_per_dim[d]; } - return _dim_subdomain_id; + return index; } - std::vector dim_subdomain_offset(int subdomain_id) { - std::vector _dim_subdomain_id = dim_subdomain_id(subdomain_id); - std::vector _dim_subdomain_offset(D); - for (int d = D - 1; d >= 0; d--) { - _dim_subdomain_offset[d] = _dim_subdomain_id[d] * _domain_decomposed_size; + // First node of a subdomain in the full array, per dimension. + std::vector subdomain_start(SIZE subdomain_id) { + std::vector index = subdomain_grid_index(subdomain_id); + std::vector start(D); + for (DIM d = 0; d < D; d++) { + start[d] = index[d] * _block_shape[d]; } - return _dim_subdomain_offset; + return start; } - std::vector subdomain_shape(int subdomain_id) { - if (subdomain_id >= _num_subdomains) { - throw std::runtime_error( - "DomainDecomposer: subdomain_shape wrong subdomain_id."); - } - if (!_domain_decomposed) { - return shape; - } else { - if (config.domain_decomposition == domain_decomposition_type::MaxDim) { - if (subdomain_id < - shape[_domain_decomposed_dim] / _domain_decomposed_size) { - std::vector chunk_shape = shape; - chunk_shape[_domain_decomposed_dim] = _domain_decomposed_size; - return chunk_shape; - } else { - SIZE leftover_dim_size = - shape[_domain_decomposed_dim] % _domain_decomposed_size; - std::vector leftover_shape = shape; - leftover_shape[_domain_decomposed_dim] = leftover_dim_size; - return leftover_shape; - } - } else if (config.domain_decomposition == - domain_decomposition_type::Block) { - std::vector chunk_shape(D); - std::vector _dim_subdomain_id = dim_subdomain_id(subdomain_id); - for (int d = D - 1; d >= 0; d--) { - if (_dim_subdomain_id[d] < shape[d] / _domain_decomposed_size) { - chunk_shape[d] = _domain_decomposed_size; - } else { - chunk_shape[d] = shape[d] % _domain_decomposed_size; - } - } - return chunk_shape; - } else if (config.domain_decomposition == - domain_decomposition_type::Variable) { - std::vector chunk_shape = shape; - chunk_shape[_domain_decomposed_dim] = - _domain_decomposed_sizes[subdomain_id]; - return chunk_shape; - } else { - throw std::runtime_error("Wrong domain decomposition type."); - return shape; - } + std::vector subdomain_shape(SIZE subdomain_id) { + std::vector index = subdomain_grid_index(subdomain_id); + std::vector extent(D); + for (DIM d = 0; d < D; d++) { + extent[d] = block_extent(shape[d], _block_shape[d], index[d]); } + return extent; } - int largest_subdomain() { - SIZE max_count = 0; - int max_subdomain = 0; - for (int id = 0; id < this->_num_subdomains; id++) { - SIZE count = 1; - std::vector chunk_shape = subdomain_shape(id); - for (int i = 0; i < chunk_shape.size(); i++) { - count *= chunk_shape[i]; - } + // Id of a subdomain with the most nodes (sizes the shared device buffers). + SIZE largest_subdomain() { + SIZE max_count = 0, max_id = 0; + for (SIZE id = 0; id < _num_subdomains; id++) { + SIZE count = num_nodes(subdomain_shape(id)); if (count > max_count) { max_count = count; - max_subdomain = id; + max_id = id; } } - return max_subdomain; + return max_id; } - SIZE subdomain_compressed_buffer_size(int subdomain_id) { - std::vector shape = subdomain_shape(subdomain_id); - SIZE num_elements = 1; - for (int i = 0; i < shape.size(); i++) - num_elements *= shape[i]; + SIZE subdomain_compressed_buffer_size(SIZE subdomain_id) { SIZE size = 0; - size += num_elements * sizeof(HUFFMAN_CODE); + size += num_nodes(subdomain_shape(subdomain_id)) * sizeof(HUFFMAN_CODE); size += config.estimate_outlier_ratio * sizeof(QUANTIZED_INT); return size; } - bool generate_max_dim_domain_decomposition_strategy( - std::vector shape, DIM &_domain_decomposed_dim, - SIZE &_domain_decomposed_size) { - // determine max dimension - SIZE max_dim = 0; + typename OperatorType::HierarchyType subdomain_hierarchy(SIZE subdomain_id) { + using HierarchyType = typename OperatorType::HierarchyType; + std::vector extent = subdomain_shape(subdomain_id); + if (uniform) { + return HierarchyType(extent, config); + } + // Non-uniform: the block's slice of each coordinate array. + std::vector start = subdomain_start(subdomain_id); + std::vector> block_coords(D); + std::vector block_coord_ptrs(D); for (DIM d = 0; d < D; d++) { - if (shape[d] > max_dim) { - max_dim = shape[d]; - _domain_decomposed_dim = d; - } + block_coords[d].resize(extent[d]); + MemoryManager::Copy1D(block_coords[d].data(), + coords[d] + start[d], extent[d], 0); + block_coord_ptrs[d] = block_coords[d].data(); } + DeviceRuntime::SyncQueue(0); + return HierarchyType(extent, block_coord_ptrs, config); + } - // domain decomposition strategy - std::vector chunk_shape = shape; - - SIZE curr_num_subdomains = 1; - - bool need_prefetch = false; - // Then check if each chunk can fit into device memory - while (need_domain_decomposition(chunk_shape, need_prefetch)) { - // Divide by 2 and round up - chunk_shape[_domain_decomposed_dim] = - (chunk_shape[_domain_decomposed_dim] - 1) / 2 + 1; + // ------------------------------------------------------ moving block data - curr_num_subdomains = (shape[_domain_decomposed_dim] - 1) / - chunk_shape[_domain_decomposed_dim] + - 1; - need_prefetch = curr_num_subdomains > 1; - } - _domain_decomposed_size = chunk_shape[_domain_decomposed_dim]; - return true; + void set_original_data(T *original_data) { + this->original_data = original_data; + keep_original_data_decomposed = false; } - bool - generate_block_domain_decomposition_strategy(std::vector shape, - SIZE &_domain_decomposed_size) { - std::vector chunk_shape(D, _domain_decomposed_size); + // The original data is supplied (or wanted) as one buffer per subdomain, + // each holding its block contiguously. + void set_decomposed_original_data(std::vector decomposed_original_data) { + this->decomposed_original_data = decomposed_original_data; + original_data = decomposed_original_data[0]; + keep_original_data_decomposed = true; + } - int curr_num_subdomains = 1; - for (DIM d = 0; d < D; d++) { - curr_num_subdomains *= (shape[d] - 1) / _domain_decomposed_size + 1; + // Copies one subdomain between the original data and a device buffer (which + // is resized to the subdomain when copying into it). + void copy_subdomain(Array &subdomain_data, + SIZE subdomain_id, + enum subdomain_copy_direction direction, int queue_idx) { + std::vector extent = subdomain_shape(subdomain_id); + if (direction == subdomain_copy_direction::OriginalToSubdomain) { + subdomain_data.resize(extent, queue_idx); } - // Need prefetch if there are more subdomains than devices - bool need_prefetch = curr_num_subdomains > 1; - // Then check if each chunk can fit into device memory - while (need_domain_decomposition(chunk_shape, need_prefetch)) { - // Divide by 2 and round up - _domain_decomposed_size = (_domain_decomposed_size - 1) / 2 + 1; - chunk_shape = std::vector(D, _domain_decomposed_size); - - curr_num_subdomains = 1; + T *original; + std::vector original_strides; + if (keep_original_data_decomposed) { + original = decomposed_original_data[subdomain_id]; + original_strides = contiguous_strides(extent); + } else { + original_strides = contiguous_strides(shape); + std::vector start = subdomain_start(subdomain_id); + SIZE offset = 0; for (DIM d = 0; d < D; d++) { - curr_num_subdomains *= (shape[d] - 1) / _domain_decomposed_size + 1; + offset += start[d] * original_strides[d]; } + original = original_data + offset; + } - need_prefetch = curr_num_subdomains > 1; + // The device buffer is pitched: rows of ld(D - 1) elements. + std::vector buffer_strides(D); + buffer_strides[D - 1] = 1; + if (D >= 2) { + buffer_strides[D - 2] = subdomain_data.ld(D - 1); + } + for (int d = (int)D - 3; d >= 0; d--) { + buffer_strides[d] = buffer_strides[d + 1] * subdomain_data.shape(d + 1); } - return true; - } - typename OperatorType::HierarchyType subdomain_hierarchy(int subdomain_id) { - using HierarchyType = typename OperatorType::HierarchyType; - if (uniform) { - return HierarchyType(subdomain_shape(subdomain_id), config); + if (direction == subdomain_copy_direction::OriginalToSubdomain) { + copy_box(subdomain_data.data(), buffer_strides, original, + original_strides, extent, queue_idx); } else { - if (config.domain_decomposition == domain_decomposition_type::MaxDim || - config.domain_decomposition == domain_decomposition_type::Variable) { - std::vector chunk_coords = coords; - std::vector chunk_shape = subdomain_shape(subdomain_id); - T *decompose_dim_coord = new T[chunk_shape[_domain_decomposed_dim]]; - MemoryManager::Copy1D( - decompose_dim_coord, - coords[_domain_decomposed_dim] + - subdomain_id * _domain_decomposed_size, - chunk_shape[_domain_decomposed_dim], 0); - DeviceRuntime::SyncQueue(0); - chunk_coords[_domain_decomposed_dim] = decompose_dim_coord; - HierarchyType hierarchy(chunk_shape, chunk_coords, config); - delete[] decompose_dim_coord; - return hierarchy; - } else if (config.domain_decomposition == - domain_decomposition_type::Block) { - std::vector chunk_coords(D); - std::vector chunk_shape = subdomain_shape(subdomain_id); - std::vector _dim_subdomain_id = dim_subdomain_id(subdomain_id); - for (int d = D - 1; d >= 0; d--) { - chunk_coords[d] = new T[chunk_shape[d]]; - MemoryManager::Copy1D( - chunk_coords[d], - coords[d] + _dim_subdomain_id[d] * _domain_decomposed_size, - chunk_shape[d], 0); - DeviceRuntime::SyncQueue(0); - } - HierarchyType hierarchy(chunk_shape, chunk_coords, config); - for (int d = D - 1; d >= 0; d--) - delete[] chunk_coords[d]; - return hierarchy; - } else { - throw std::runtime_error("Wrong domain decomposition type."); - } + copy_box(original, original_strides, subdomain_data.data(), + buffer_strides, extent, queue_idx); } } - DomainDecomposer() : _domain_decomposed(false) {} - - // Find domain decomposion method - DomainDecomposer(std::vector shape, Config config) - : original_data(nullptr), shape(shape), config(config), - keep_original_data_decomposed(false) { - if (!need_domain_decomposition(shape, false) && - config.domain_decomposition != domain_decomposition_type::Block && - config.domain_decomposition != domain_decomposition_type::Variable) { - this->_domain_decomposed = false; - this->_domain_decomposed_dim = 0; - this->_domain_decomposed_size = this->shape[0]; - this->_num_subdomains = 1; - log::info("DomainDecomposer: no decomposition used"); - } else { - this->_domain_decomposed = true; - if (config.domain_decomposition == domain_decomposition_type::MaxDim) { - generate_max_dim_domain_decomposition_strategy( - shape, this->_domain_decomposed_dim, this->_domain_decomposed_size); - this->_num_subdomains = (shape[this->_domain_decomposed_dim] - 1) / - this->_domain_decomposed_size + - 1; - log::info( - "Domain decomposed dim: " + std::to_string(_domain_decomposed_dim) + - ", Domain decomposed size: " + - std::to_string(_domain_decomposed_size)); - log::info("DomainDecomposer: decomposed into " + - std::to_string(this->_num_subdomains) + - " subdomains using MaxDim method"); - } else if (config.domain_decomposition == - domain_decomposition_type::Block) { - this->_domain_decomposed_size = config.block_size; - generate_block_domain_decomposition_strategy( - shape, this->_domain_decomposed_size); - this->_num_subdomains = 1; - for (DIM d = 0; d < D; d++) { - this->_num_subdomains *= - (shape[d] - 1) / this->_domain_decomposed_size + 1; - } - log::info("Domain decomposed size: " + - std::to_string(_domain_decomposed_size)); - log::info("DomainDecomposer: decomposed into " + - std::to_string(this->_num_subdomains) + - " subdomains using Block method"); - } else if (config.domain_decomposition == - domain_decomposition_type::Variable) { - this->_domain_decomposed_dim = config.domain_decomposition_dim; - this->_domain_decomposed_sizes = config.domain_decomposition_sizes; - this->_num_subdomains = this->_domain_decomposed_sizes.size(); - log::info("DomainDecomposer: decomposed into " + - std::to_string(this->_num_subdomains) + - " subdomains using Variable method"); + // Copies an N-D box of `extent` nodes; strides are in elements, per + // dimension, for each side. Dimensions that are contiguous on both sides are + // merged first, so a box spanning whole rows or planes becomes a single 1D + // or 2D copy. What remains maps onto Copy3D, looping only over dimensions + // beyond the third. + static void copy_box(T *dst, std::vector dst_strides, const T *src, + std::vector src_strides, std::vector extent, + int queue_idx) { + struct Run { + SIZE n, dst_stride, src_stride; + }; + // Innermost first. + std::vector runs; + for (int d = D - 1; d >= 0; d--) { + if (extent[d] == 1) { + continue; + } + if (!runs.empty() && + dst_strides[d] == runs.back().dst_stride * runs.back().n && + src_strides[d] == runs.back().src_stride * runs.back().n) { + runs.back().n *= extent[d]; } else { - throw std::runtime_error("Wrong domain decomposition type."); + runs.push_back({extent[d], dst_strides[d], src_strides[d]}); } } + // The copy primitives move unit-stride rows. + if (runs.empty() || runs[0].dst_stride != 1 || runs[0].src_stride != 1) { + runs.insert(runs.begin(), {1, 1, 1}); + } - uniform = true; - } - - // Find domain decomposion method - DomainDecomposer(std::vector shape, Config config, - std::vector coords) - : original_data(nullptr), shape(shape), config(config), coords(coords), - keep_original_data_decomposed(false) { - if (!need_domain_decomposition(shape, false) && - config.domain_decomposition != domain_decomposition_type::Block && - config.domain_decomposition != domain_decomposition_type::Variable) { - this->_domain_decomposed = false; - this->_domain_decomposed_dim = 0; - this->_domain_decomposed_size = this->shape[0]; - this->_num_subdomains = 1; - log::info("DomainDecomposer: no decomposition used"); + using Mem = MemoryManager; + if (runs.size() == 1) { + Mem::Copy1D(dst, src, runs[0].n, queue_idx); + } else if (runs.size() == 2) { + Mem::Copy2D(dst, runs[1].dst_stride, src, runs[1].src_stride, runs[0].n, + runs[1].n, queue_idx); } else { - this->_domain_decomposed = true; - if (config.domain_decomposition == domain_decomposition_type::MaxDim) { - generate_max_dim_domain_decomposition_strategy( - shape, this->_domain_decomposed_dim, this->_domain_decomposed_size); - this->_num_subdomains = (shape[this->_domain_decomposed_dim] - 1) / - this->_domain_decomposed_size + - 1; - log::info( - "Domain decomposed dim: " + std::to_string(_domain_decomposed_dim) + - ", Domain decomposed size: " + - std::to_string(_domain_decomposed_size)); - log::info("DomainDecomposer: decomposed into " + - std::to_string(this->_num_subdomains) + - " subdomains using MaxDim method"); - } else if (config.domain_decomposition == - domain_decomposition_type::Block) { - this->_domain_decomposed_size = config.block_size; - generate_block_domain_decomposition_strategy( - shape, this->_domain_decomposed_size); - this->_num_subdomains = 1; - for (DIM d = 0; d < D; d++) { - this->_num_subdomains *= - (shape[d] - 1) / this->_domain_decomposed_size + 1; + // One Copy3D per position in the remaining outer runs. + std::vector outer(runs.size() - 3, 0); + while (true) { + SIZE dst_offset = 0, src_offset = 0; + for (size_t i = 0; i < outer.size(); i++) { + dst_offset += outer[i] * runs[3 + i].dst_stride; + src_offset += outer[i] * runs[3 + i].src_stride; + } + Mem::Copy3D(dst + dst_offset, runs[1].dst_stride, runs[2].dst_stride, + src + src_offset, runs[1].src_stride, runs[2].src_stride, + runs[0].n, runs[1].n, runs[2].n, queue_idx); + size_t i = 0; + while (i < outer.size() && ++outer[i] == runs[3 + i].n) { + outer[i++] = 0; + } + if (i == outer.size()) { + break; } - log::info("Domain decomposed size: " + - std::to_string(_domain_decomposed_size)); - log::info("DomainDecomposer: decomposed into " + - std::to_string(this->_num_subdomains) + - " subdomains using Block method"); - } else if (config.domain_decomposition == - domain_decomposition_type::Variable) { - this->_domain_decomposed_dim = config.domain_decomposition_dim; - this->_domain_decomposed_sizes = config.domain_decomposition_sizes; - this->_num_subdomains = this->_domain_decomposed_sizes.size(); - log::info("DomainDecomposer: decomposed into " + - std::to_string(this->_num_subdomains) + - " subdomains using Variable method"); - } else { - throw std::runtime_error("Wrong domain decomposition type."); } } - - uniform = false; } - // Force to use this domain decomposion method - DomainDecomposer(std::vector shape, bool _domain_decomposed, - DIM _domain_decomposed_dim, SIZE _domain_decomposed_size, - Config config) - : original_data(nullptr), shape(shape), - _domain_decomposed_dim(_domain_decomposed_dim), - _domain_decomposed_size(_domain_decomposed_size), - _domain_decomposed(_domain_decomposed), config(config), - keep_original_data_decomposed(false) { - if (!this->_domain_decomposed) { - this->_domain_decomposed_dim = 0; - this->_domain_decomposed_size = this->shape[0]; - this->_num_subdomains = 1; - log::info("DomainDecomposer: no decomposition used"); - } else { - if (config.domain_decomposition == domain_decomposition_type::MaxDim) { - this->_num_subdomains = (shape[this->_domain_decomposed_dim] - 1) / - this->_domain_decomposed_size + - 1; - log::info( - "Domain decomposed dim: " + std::to_string(_domain_decomposed_dim) + - ", Domain decomposed size: " + - std::to_string(_domain_decomposed_size)); - log::info("DomainDecomposer: decomposed into " + - std::to_string(this->_num_subdomains) + - " subdomains using MaxDim method"); - } else if (config.domain_decomposition == - domain_decomposition_type::Block) { - this->_num_subdomains = 1; - for (DIM d = 0; d < D; d++) { - this->_num_subdomains *= - (shape[d] - 1) / this->_domain_decomposed_size + 1; - } - log::info("Domain decomposed size: " + - std::to_string(_domain_decomposed_size)); - log::info("DomainDecomposer: decomposed into " + - std::to_string(this->_num_subdomains) + - " subdomains using Block method"); - } else if (config.domain_decomposition == - domain_decomposition_type::Variable) { - this->_domain_decomposed_dim = config.domain_decomposition_dim; - this->_domain_decomposed_sizes = config.domain_decomposition_sizes; - this->_num_subdomains = this->_domain_decomposed_sizes.size(); - log::info("DomainDecomposer: decomposed into " + - std::to_string(this->_num_subdomains) + - " subdomains using Variable method"); - } else { - throw std::runtime_error("Wrong domain decomposition type."); - } + // ------------------------------------------------------- memory estimates + + size_t EstimateMemoryFootprint(std::vector shape, + bool enable_prefetch) { + size_t estimate_memory_usgae = 0; + + Array<1, T, DeviceType> array_with_pitch({1}); + size_t pitch_size = array_with_pitch.ld(0) * sizeof(T); + + size_t input_space = roundup((size_t)shape[D - 1] * sizeof(T), pitch_size); + for (DIM d = 0; d < D - 1; d++) { + input_space *= shape[d]; } - uniform = true; - } + size_t output_space = 0; + output_space += num_nodes(shape) * sizeof(HUFFMAN_CODE); + output_space += config.estimate_outlier_ratio * sizeof(QUANTIZED_INT); - // Force to use this domain decomposion method - DomainDecomposer(std::vector shape, bool _domain_decomposed, - DIM _domain_decomposed_dim, SIZE _domain_decomposed_size, - Config config, std::vector coords) - : original_data(nullptr), shape(shape), - _domain_decomposed_dim(_domain_decomposed_dim), - _domain_decomposed_size(_domain_decomposed_size), - _domain_decomposed(_domain_decomposed), config(config), coords(coords), - keep_original_data_decomposed(false) { - if (!this->_domain_decomposed) { - this->_domain_decomposed_dim = 0; - this->_domain_decomposed_size = this->shape[0]; - this->_num_subdomains = 1; - log::info("DomainDecomposer: no decomposition used"); - } else { - if (config.domain_decomposition == domain_decomposition_type::MaxDim) { - this->_num_subdomains = (shape[this->_domain_decomposed_dim] - 1) / - this->_domain_decomposed_size + - 1; - log::info( - "Domain decomposed dim: " + std::to_string(_domain_decomposed_dim) + - ", Domain decomposed size: " + - std::to_string(_domain_decomposed_size)); - log::info("DomainDecomposer: decomposed into " + - std::to_string(this->_num_subdomains) + - " subdomains using MaxDim method"); - } else if (config.domain_decomposition == - domain_decomposition_type::Block) { - this->_num_subdomains = 1; - for (DIM d = 0; d < D; d++) { - this->_num_subdomains *= - (shape[d] - 1) / this->_domain_decomposed_size + 1; - } - log::info("Domain decomposed size: " + - std::to_string(_domain_decomposed_size)); - log::info("DomainDecomposer: decomposed into " + - std::to_string(this->_num_subdomains) + - " subdomains using Block method"); - } else if (config.domain_decomposition == - domain_decomposition_type::Variable) { - this->_domain_decomposed_dim = config.domain_decomposition_dim; - this->_domain_decomposed_sizes = config.domain_decomposition_sizes; - this->_num_subdomains = this->_domain_decomposed_sizes.size(); - log::info("DomainDecomposer: decomposed into " + - std::to_string(this->_num_subdomains) + - " subdomains using Variable method"); - } else { - throw std::runtime_error("Wrong domain decomposition type."); - } + estimate_memory_usgae = input_space + output_space; + + log::dbg("Input output space: " + + std::to_string((double)(input_space + output_space) / 1e9) + + " GB"); + + using HierarchyType = typename OperatorType::HierarchyType; + size_t hierarchy_space = + HierarchyType::EstimateMemoryFootprint(shape, config); + estimate_memory_usgae += hierarchy_space; + log::dbg("Hierarchy space: " + + std::to_string((double)hierarchy_space / 1e9) + " GB"); + + // For prefetching + if (enable_prefetch) { + estimate_memory_usgae *= 2; } + estimate_memory_usgae += + OperatorType::EstimateMemoryFootprint(shape, config); + log::dbg("Operation space: " + + std::to_string( + (double)OperatorType::EstimateMemoryFootprint(shape, config) / + 1e9) + + " GB"); - uniform = false; + return estimate_memory_usgae; } - // void calc_domain_decompose_parameter(SIZE subdomain_id, - // std::vector subdomain_shape, - // SIZE &subdomain_ld, SIZE &original_ld, - // SIZE &n1, SIZE &n2) { - // SIZE decomposed_size; - // if (subdomain_id < - // shape[_domain_decomposed_dim] / _domain_decomposed_size) { - // decomposed_size = _domain_decomposed_size; - // } else { - // decomposed_size = shape[_domain_decomposed_dim] % - // _domain_decomposed_size; - // } - - // std::vector original_shape = original_data_shape(subdomain_shape); - - // subdomain_ld = subdomain_shape[_domain_decomposed_dim]; // - // decomposed_size; for (int d = D - 1; d > (int)_domain_decomposed_dim; - // d--) { - // subdomain_ld *= subdomain_shape[d]; - // } - // // std::cout << "dst_ld: " << dst_ld << "\n"; - // original_ld = 1; - // for (int d = D - 1; d >= (int)_domain_decomposed_dim; d--) { - // original_ld *= original_shape[d]; - // } - // // std::cout << "src_ld: " << src_ld << "\n"; - // n1 = subdomain_shape[_domain_decomposed_dim]; - // for (int d = D - 1; d > (int)_domain_decomposed_dim; d--) { - // n1 *= subdomain_shape[d]; - // } - // // std::cout << "n1: " << n1 << "\n"; - // n2 = 1; - // for (int d = 0; d < (int)_domain_decomposed_dim; d++) { - // n2 *= subdomain_shape[d]; - // } - // // std::cout << "n2: " << n2 << "\n"; - // } - - SIZE calc_offset(std::vector shape, - std::vector dim_subdomain_offset) { - SIZE curr_stride = 1; - SIZE ret_idx = 0; - for (int d = D - 1; d >= 0; d--) { - ret_idx += dim_subdomain_offset[d] * curr_stride; - curr_stride *= shape[d]; - } - return ret_idx; + // Memory one block may use: the device's free memory, capped by + // Config::max_memory_footprint. + size_t available_memory() { + return std::min((SIZE)DeviceRuntime::GetAvailableMemory(), + config.max_memory_footprint); } - bool check_shape(Array &subdomain_data, - std::vector shape) { - if (subdomain_data.data() == nullptr) { - return false; - } - for (DIM d = 0; d < D; d++) { - if (subdomain_data.shape(d) != shape[d]) { - return false; - } + bool need_domain_decomposition(std::vector shape, + bool enable_prefetch) { + size_t estm = EstimateMemoryFootprint(shape, enable_prefetch); + size_t aval = available_memory(); + log::dbg("Estimated memory usage: " + std::to_string((double)estm / 1e9) + + "GB, Available: " + std::to_string((double)aval / 1e9) + "GB"); + return estm >= aval; + } + + std::vector shape; + Config config; + std::vector coords; + bool uniform = true; + +private: + static SIZE num_nodes(std::vector shape) { + SIZE count = 1; + for (SIZE n : shape) { + count *= n; } - return true; + return count; } - T *original_data_ptr(int subdomain_id) { - if (config.domain_decomposition == domain_decomposition_type::MaxDim || - config.domain_decomposition == domain_decomposition_type::Variable) { - if (!keep_original_data_decomposed) { - SIZE offset = 0; - for (int i = 0; i < subdomain_id; i++) { - SIZE count = 1; - std::vector chunk_shape = subdomain_shape(i); - for (int d = D - 1; d >= 0; d--) { - count *= chunk_shape[d]; - } - offset += count; - } - // SIZE offset = _domain_decomposed_size; - // for (int d = D - 1; d > (int)_domain_decomposed_dim; d--) { - // offset *= shape[d]; - // } - return original_data + offset; // * subdomain_id; - } else { - return decomposed_original_data[subdomain_id]; - } - } else if (config.domain_decomposition == - domain_decomposition_type::Block) { - if (!keep_original_data_decomposed) { - return original_data + - calc_offset(shape, dim_subdomain_offset(subdomain_id)); - } else { - return decomposed_original_data[subdomain_id]; - } - } else { - throw std::runtime_error("Wrong domain decomposition type."); + static std::vector contiguous_strides(std::vector shape) { + std::vector strides(D); + strides[D - 1] = 1; + for (int d = (int)D - 2; d >= 0; d--) { + strides[d] = strides[d + 1] * shape[d + 1]; } + return strides; } - std::vector original_data_shape(std::vector subdomain_shape) { - if (!keep_original_data_decomposed) { - return shape; - } else { - return subdomain_shape; + static std::string shape_string(std::vector shape) { + std::stringstream ss; + for (size_t d = 0; d < shape.size(); d++) { + ss << (d ? "x" : "") << shape[d]; } + return ss.str(); } - void copy_subdomain(Array &subdomain_data, int subdomain_id, - enum subdomain_copy_direction direction, int queue_idx) { + void check_subdomain_id(SIZE subdomain_id) { if (subdomain_id >= _num_subdomains) { - throw std::runtime_error( - "DomainDecomposer::copy_subdomain wrong subdomain_id."); + throw std::runtime_error("DomainDecomposer: subdomain id " + + std::to_string(subdomain_id) + " out of range."); } + } - if (!_domain_decomposed) { - // if (keep_original_data_decomposed) { - // throw std::runtime_error("Do not support restoring to decomposed data - // when no domain decomposition was used."); - // } + // Number of blocks of shape `block`, and the shape of the largest one (a + // merged remainder makes the last block along a dimension the widest), + // which bounds the memory any block needs. + SIZE count_blocks(std::vector block) { + SIZE count = 1; + for (DIM d = 0; d < D; d++) { + count *= num_blocks(shape[d], block[d]); + } + return count; + } - if (direction == subdomain_copy_direction::OriginalToSubdomain) { - subdomain_data.resize(shape); - } - SIZE linearized_width = 1; - for (DIM d = 0; d < D - 1; d++) - linearized_width *= subdomain_data.shape(d); - if (direction == subdomain_copy_direction::OriginalToSubdomain) { - MemoryManager::CopyND( - subdomain_data.data(), subdomain_data.ld(D - 1), original_data, - subdomain_data.shape(D - 1), subdomain_data.shape(D - 1), - linearized_width, queue_idx); - } else { - MemoryManager::CopyND( - original_data, subdomain_data.shape(D - 1), subdomain_data.data(), - subdomain_data.ld(D - 1), subdomain_data.shape(D - 1), - linearized_width, queue_idx); - } - } else { - if (config.domain_decomposition == domain_decomposition_type::MaxDim || - config.domain_decomposition == domain_decomposition_type::Variable) { - if (keep_original_data_decomposed) { - throw std::runtime_error( - "Do not support restoring to decomposed data when using " - "MaxDim or Variable"); - } - T *data = original_data_ptr(subdomain_id); - if (direction == subdomain_copy_direction::OriginalToSubdomain) { - subdomain_data.resize(subdomain_shape(subdomain_id), queue_idx); - } - SIZE subdomain_ld, original_ld, n1, n2; - // calc_domain_decompose_parameter(subdomain_id, subdomain_data.shape(), - // subdomain_ld, original_ld, n1, n2); SIZE decomposed_size; if - // (subdomain_id < - // shape[_domain_decomposed_dim] / _domain_decomposed_size) { - // decomposed_size = _domain_decomposed_size; - // } else { - // decomposed_size = shape[_domain_decomposed_dim] % - // _domain_decomposed_size; - // } - - std::vector original_shape = - original_data_shape(subdomain_data.shape()); - - subdomain_ld = subdomain_data.shape(_domain_decomposed_dim); - for (int d = D - 1; d > (int)_domain_decomposed_dim; d--) { - subdomain_ld *= subdomain_data.shape(d); - } - // std::cout << "subdomain_ld: " << subdomain_ld << "\n"; - original_ld = 1; - for (int d = D - 1; d >= (int)_domain_decomposed_dim; d--) { - original_ld *= original_shape[d]; - } - // std::cout << "original_ld: " << original_ld << "\n"; - n1 = subdomain_data.shape(_domain_decomposed_dim); - for (int d = D - 1; d > (int)_domain_decomposed_dim; d--) { - n1 *= subdomain_data.shape(d); - } - // std::cout << "n1: " << n1 << "\n"; - n2 = 1; - for (int d = 0; d < (int)_domain_decomposed_dim; d++) { - n2 *= subdomain_data.shape(d); - } - // std::cout << "n2: " << n2 << "\n"; - - if (direction == subdomain_copy_direction::OriginalToSubdomain) { - MemoryManager::CopyND(subdomain_data.data(), subdomain_ld, - data, original_ld, n1, n2, - queue_idx); - } else { - MemoryManager::CopyND(data, original_ld, - subdomain_data.data(), subdomain_ld, - n1, n2, queue_idx); - } - } else if (config.domain_decomposition == - domain_decomposition_type::Block) { - std::vector chunk_shape = subdomain_shape(subdomain_id); - if (direction == subdomain_copy_direction::OriginalToSubdomain) { - subdomain_data.resize(chunk_shape, queue_idx); - } + std::vector largest_block_shape(std::vector block) { + std::vector largest(D); + for (DIM d = 0; d < D; d++) { + largest[d] = block_extent(shape[d], block[d], + num_blocks(shape[d], block[d]) - 1); + } + return largest; + } - T *data = original_data_ptr(subdomain_id); - std::vector shape = original_data_shape(subdomain_data.shape()); - - if (D == 1) { - if (direction == subdomain_copy_direction::OriginalToSubdomain) { - MemoryManager::CopyND( - subdomain_data.data(), subdomain_data.ld(D - 1), data, - shape[D - 1], subdomain_data.shape(D - 1), 1, queue_idx); - } else { - MemoryManager::CopyND( - data, shape[D - 1], subdomain_data.data(), - subdomain_data.ld(D - 1), subdomain_data.shape(D - 1), 1, - queue_idx); - } - } else if (D == 2) { - if (direction == subdomain_copy_direction::OriginalToSubdomain) { - MemoryManager::CopyND( - subdomain_data.data(), subdomain_data.ld(D - 1), data, - shape[D - 1], subdomain_data.shape(D - 1), - subdomain_data.shape(D - 2), queue_idx); - } else { - MemoryManager::CopyND( - data, shape[D - 1], subdomain_data.data(), - subdomain_data.ld(D - 1), subdomain_data.shape(D - 1), - subdomain_data.shape(D - 2), queue_idx); - } - } else if (D == 3) { - for (SIZE i = 0; i < subdomain_data.shape(D - 3); i++) { - if (direction == subdomain_copy_direction::OriginalToSubdomain) { - MemoryManager::CopyND( - subdomain_data.data() + - calc_offset(subdomain_data.shape(), {i, 0, 0}), - subdomain_data.ld(D - 1), - data + calc_offset(shape, {i, 0, 0}), shape[D - 1], - subdomain_data.shape(D - 1), subdomain_data.shape(D - 2), - queue_idx); - } else { - MemoryManager::CopyND( - data + calc_offset(shape, {i, 0, 0}), shape[D - 1], - subdomain_data.data() + - calc_offset(subdomain_data.shape(), {i, 0, 0}), - subdomain_data.ld(D - 1), subdomain_data.shape(D - 1), - subdomain_data.shape(D - 2), queue_idx); - } - } - } else if (D == 4) { - for (SIZE j = 0; j < subdomain_data.shape(D - 4); j++) { - for (SIZE i = 0; i < subdomain_data.shape(D - 3); i++) { - if (direction == subdomain_copy_direction::OriginalToSubdomain) { - MemoryManager::CopyND( - subdomain_data.data() + - calc_offset(subdomain_data.shape(), {j, i, 0, 0}), - subdomain_data.ld(D - 1), - data + calc_offset(shape, {j, i, 0, 0}), shape[D - 1], - subdomain_data.shape(D - 1), subdomain_data.shape(D - 2), - queue_idx); - } else { - MemoryManager::CopyND( - data + calc_offset(shape, {j, i, 0, 0}), shape[D - 1], - subdomain_data.data() + - calc_offset(subdomain_data.shape(), {j, i, 0, 0}), - subdomain_data.ld(D - 1), subdomain_data.shape(D - 1), - subdomain_data.shape(D - 2), queue_idx); - } - } - } - } else if (D == 5) { - for (SIZE k = 0; k < subdomain_data.shape(D - 5); k++) { - for (SIZE j = 0; j < subdomain_data.shape(D - 4); j++) { - for (SIZE i = 0; i < subdomain_data.shape(D - 3); i++) { - if (direction == - subdomain_copy_direction::OriginalToSubdomain) { - MemoryManager::CopyND( - subdomain_data.data() + - calc_offset(subdomain_data.shape(), {k, j, i, 0, 0}), - subdomain_data.ld(D - 1), - data + calc_offset(shape, {k, j, i, 0, 0}), shape[D - 1], - subdomain_data.shape(D - 1), subdomain_data.shape(D - 2), - queue_idx); - } else { - MemoryManager::CopyND( - data + calc_offset(shape, {k, j, i, 0, 0}), shape[D - 1], - subdomain_data.data() + - calc_offset(subdomain_data.shape(), {k, j, i, 0, 0}), - subdomain_data.ld(D - 1), subdomain_data.shape(D - 1), - subdomain_data.shape(D - 2), queue_idx); - } - } - } - } - } else { - throw std::runtime_error( - "Copy subdomain does not support higher than 5D data."); + bool fits_memory(std::vector block) { + return !need_domain_decomposition(largest_block_shape(block), + count_blocks(block) > 1); + } + + std::vector auto_block_shape() { + std::vector block = shape; + while (!fits_memory(block)) { + // Halve the largest extent that can still shrink. Ties go to the slowest + // dimension, whose blocks are the most contiguous in memory. + int split = -1; + for (DIM d = 0; d < D; d++) { + if (block[d] > MIN_BLOCK_EXTENT && + (split < 0 || block[d] > block[split])) { + split = d; } - } else { - throw std::runtime_error("Wrong domain decomposition type."); } + if (split < 0) { + throw Exception("Automatic domain decomposition: even the smallest " + "blocks (" + + shape_string(block) + + ") do not fit the memory limit of " + + std::to_string(available_memory()) + " bytes."); + } + block[split] = std::max((block[split] - 1) / 2 + 1, MIN_BLOCK_EXTENT); } + return block; } - bool domain_decomposed() { return _domain_decomposed; } - - DIM domain_decomposed_dim() { return _domain_decomposed_dim; } - - SIZE domain_decomposed_size() { return _domain_decomposed_size; } - - SIZE num_subdomains() { return _num_subdomains; } - - void set_original_data(T *original_data) { - this->original_data = original_data; - keep_original_data_decomposed = false; + std::vector manual_block_shape() { + const std::vector &requested = config.domain_decomposition_block_size; + if (requested.size() != D) { + throw Exception("Manual domain decomposition: needs a block size for " + "each of the " + + std::to_string(D) + " dimensions, got " + + std::to_string(requested.size()) + "."); + } + std::vector block(D); + for (DIM d = 0; d < D; d++) { + if (requested[d] < MIN_BLOCK_EXTENT) { + throw Exception("Manual domain decomposition: block size " + + std::to_string(requested[d]) + " along dimension " + + std::to_string(d) + " is smaller than " + + std::to_string(MIN_BLOCK_EXTENT) + "."); + } + block[d] = std::min(requested[d], shape[d]); + } + if (!fits_memory(block)) { + std::vector largest = largest_block_shape(block); + throw Exception( + "Manual domain decomposition: blocks of " + shape_string(block) + + " need about " + + std::to_string( + EstimateMemoryFootprint(largest, count_blocks(block) > 1)) + + " bytes (largest block " + shape_string(largest) + "), more than " + + "the " + std::to_string(available_memory()) + " bytes available."); + } + return block; } - void set_decomposed_original_data(std::vector decomposed_original_data) { - this->decomposed_original_data = decomposed_original_data; - original_data = decomposed_original_data[0]; - keep_original_data_decomposed = true; + void set_block_shape(std::vector block) { + _block_shape.resize(D); + _blocks_per_dim.resize(D); + _num_subdomains = 1; + for (DIM d = 0; d < D; d++) { + _block_shape[d] = std::min(block[d], shape[d]); + // A dimension narrower than MIN_BLOCK_EXTENT can only be kept whole. + if (_block_shape[d] < std::min(MIN_BLOCK_EXTENT, shape[d])) { + throw InvalidDataException( + "domain decomposition block extent " + + std::to_string(_block_shape[d]) + " along dimension " + + std::to_string(d) + " is smaller than " + + std::to_string(MIN_BLOCK_EXTENT) + "."); + } + _blocks_per_dim[d] = num_blocks(shape[d], _block_shape[d]); + _num_subdomains *= _blocks_per_dim[d]; + } + if (domain_decomposed()) { + log::info("DomainDecomposer: decomposed into " + + std::to_string(_num_subdomains) + " subdomains of " + + shape_string(_block_shape)); + } else { + log::info("DomainDecomposer: no decomposition used"); + } } - std::vector shape; - bool _domain_decomposed; - DIM _domain_decomposed_dim; - SIZE _domain_decomposed_size; - std::vector _domain_decomposed_sizes; - SIZE _num_subdomains; - std::vector> subdomain_hierarchies; - T *original_data; + std::vector _block_shape; + std::vector _blocks_per_dim; + SIZE _num_subdomains = 1; + T *original_data = nullptr; std::vector decomposed_original_data; - bool keep_original_data_decomposed; - Config config; - bool uniform; - std::vector coords; + bool keep_original_data_decomposed = false; }; } // namespace mgard_x -#endif \ No newline at end of file +#endif diff --git a/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp b/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp index 73446a0be..c5aa6c31a 100644 --- a/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp +++ b/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp @@ -46,9 +46,15 @@ SIZE get_max_output_data_size( return size; } +// Domain decomposition block shape recorded in the header. +template +std::vector block_shape(Metadata &metadata) { + return std::vector(metadata.domain_decomposition_block_shape.begin(), + metadata.domain_decomposition_block_shape.end()); +} + template void load(Config &config, Metadata &metadata) { - config.domain_decomposition = metadata.ddtype; config.decomposition = metadata.decomposition; config.lossless = metadata.ltype; config.huff_dict_size = metadata.huff_dict_size; @@ -139,14 +145,12 @@ void MDRefactor(std::vector shape, const void *original_data, m.FillForMDR( (T)0.0, config.decomposition, config.lossless, config.huff_dict_size, config.huff_block_size, shape, domain_decomposer.domain_decomposed(), - config.domain_decomposition, domain_decomposer.domain_decomposed_dim(), - domain_decomposer.domain_decomposed_size(), config.total_num_bitplanes); + domain_decomposer.block_shape(), config.total_num_bitplanes); } else { m.FillForMDR( (T)0.0, config.decomposition, config.lossless, config.huff_dict_size, config.huff_block_size, shape, domain_decomposer.domain_decomposed(), - config.domain_decomposition, domain_decomposer.domain_decomposed_dim(), - domain_decomposer.domain_decomposed_size(), config.total_num_bitplanes, + domain_decomposer.block_shape(), config.total_num_bitplanes, coords); } @@ -213,8 +217,7 @@ void MDRequest(std::vector shape, RefactoredMetadata &refactored_metadata, domain_decomposer; domain_decomposer = DomainDecomposer, DeviceType>( - shape, m.domain_decomposed, m.domain_decomposed_dim, - m.domain_decomposed_size, config); + shape, block_shape(m), config); generate_request(domain_decomposer, config, refactored_metadata); DeviceRuntime::Finalize(); } @@ -276,15 +279,11 @@ void MDReconstruct(std::vector shape, if (m.dstype == data_structure_type::Cartesian_Grid_Uniform) { domain_decomposer = DomainDecomposer, - DeviceType>(shape, m.domain_decomposed, - m.domain_decomposed_dim, - m.domain_decomposed_size, config); + DeviceType>(shape, block_shape(m), config); } else { domain_decomposer = DomainDecomposer, - DeviceType>(shape, m.domain_decomposed, - m.domain_decomposed_dim, - m.domain_decomposed_size, config, coords); + DeviceType>(shape, block_shape(m), config, coords); } if (!config.mdr_adaptive_resolution) { // Should not re-allocate if the data is already allocated diff --git a/include/mgard-x/MDRHighLevel/ReconstructPipeline.hpp b/include/mgard-x/MDRHighLevel/ReconstructPipeline.hpp index ccca1b348..632516320 100644 --- a/include/mgard-x/MDRHighLevel/ReconstructPipeline.hpp +++ b/include/mgard-x/MDRHighLevel/ReconstructPipeline.hpp @@ -159,7 +159,7 @@ void reconstruct_pipeline( reconstructed_data.shape[curr_subdomain_id] = device_subdomain_buffer[current_buffer].shape(); reconstructed_data.offset[curr_subdomain_id] = - domain_decomposer.dim_subdomain_offset(curr_subdomain_id); + domain_decomposer.subdomain_start(curr_subdomain_id); } current_buffer = next_buffer; current_queue = next_queue; diff --git a/include/mgard-x/Metadata/Metadata.hpp b/include/mgard-x/Metadata/Metadata.hpp index f45c75ade..6941ba426 100644 --- a/include/mgard-x/Metadata/Metadata.hpp +++ b/include/mgard-x/Metadata/Metadata.hpp @@ -60,9 +60,9 @@ struct MetadataBase { std::vector hybrid_roi_block_dimensions; bool domain_decomposed = false; - enum domain_decomposition_type ddtype; - uint8_t domain_decomposed_dim; - uint64_t domain_decomposed_size; + // Shape of the domain decomposition blocks (see DomainDecomposer); equal to + // the data shape when the domain is not decomposed. + std::vector domain_decomposition_block_shape; enum operation_type otype; @@ -109,9 +109,7 @@ template struct Metadata : MetadataBase { enum lossless_type ltype, uint32_t huff_dict_size, uint32_t huff_block_size, std::vector shape, bool domain_decomposed, - domain_decomposition_type ddtype, - uint8_t domain_decomposed_dim, - uint64_t domain_decomposed_size) { + std::vector domain_decomposition_block_shape) { otype = operation_type::Compression; if (std::is_same::value) { @@ -148,9 +146,9 @@ template struct Metadata : MetadataBase { this->shape[d] = (uint64_t)shape[d]; } this->domain_decomposed = domain_decomposed; - this->ddtype = ddtype; - this->domain_decomposed_dim = domain_decomposed_dim; - this->domain_decomposed_size = domain_decomposed_size; + this->domain_decomposition_block_shape.assign( + domain_decomposition_block_shape.begin(), + domain_decomposition_block_shape.end()); } template @@ -159,13 +157,12 @@ template struct Metadata : MetadataBase { enum decomposition_type decomposition, enum lossless_type ltype, uint32_t huff_dict_size, uint32_t huff_block_size, std::vector shape, - bool domain_decomposed, domain_decomposition_type ddtype, - uint8_t domain_decomposed_dim, - uint64_t domain_decomposed_size, std::vector coords) { + bool domain_decomposed, + std::vector domain_decomposition_block_shape, + std::vector coords) { FillForCompression(ebtype, tol, s, norm, decomposition, ltype, huff_dict_size, huff_block_size, shape, - domain_decomposed, ddtype, domain_decomposed_dim, - domain_decomposed_size); + domain_decomposed, domain_decomposition_block_shape); for (int d = 0; d < this->total_dims; d++) { std::vector coord(shape[d]); T *coord_h = new T[shape[d]]; @@ -183,9 +180,9 @@ template struct Metadata : MetadataBase { void FillForMDR(T norm, enum decomposition_type decomposition, enum lossless_type ltype, uint32_t huff_dict_size, uint32_t huff_block_size, std::vector shape, - bool domain_decomposed, domain_decomposition_type ddtype, - uint8_t domain_decomposed_dim, - uint64_t domain_decomposed_size, uint64_t number_bitplanes) { + bool domain_decomposed, + std::vector domain_decomposition_block_shape, + uint64_t number_bitplanes) { otype = operation_type::MDR; if (std::is_same::value) { this->ptype = processor_type::X_SERIAL; @@ -212,9 +209,9 @@ template struct Metadata : MetadataBase { this->shape[d] = (uint64_t)shape[d]; } this->domain_decomposed = domain_decomposed; - this->ddtype = ddtype; - this->domain_decomposed_dim = domain_decomposed_dim; - this->domain_decomposed_size = domain_decomposed_size; + this->domain_decomposition_block_shape.assign( + domain_decomposition_block_shape.begin(), + domain_decomposition_block_shape.end()); this->betype = bitplane_encoding_type::GroupedBitplaneEncoding; this->number_bitplanes = number_bitplanes; } @@ -223,13 +220,13 @@ template struct Metadata : MetadataBase { void FillForMDR(T norm, enum decomposition_type decomposition, enum lossless_type ltype, uint32_t huff_dict_size, uint32_t huff_block_size, std::vector shape, - bool domain_decomposed, domain_decomposition_type ddtype, - uint8_t domain_decomposed_dim, - uint64_t domain_decomposed_size, uint64_t number_bitplanes, + bool domain_decomposed, + std::vector domain_decomposition_block_shape, + uint64_t number_bitplanes, std::vector coords) { FillForMDR(norm, decomposition, ltype, huff_dict_size, huff_block_size, - shape, domain_decomposed, ddtype, domain_decomposed_dim, - domain_decomposed_size, number_bitplanes); + shape, domain_decomposed, domain_decomposition_block_shape, + number_bitplanes); for (int d = 0; d < this->total_dims; d++) { std::vector coord(shape[d]); T *coord_h = new T[shape[d]]; diff --git a/include/mgard-x/RuntimeX/DataStructures/Array.hpp b/include/mgard-x/RuntimeX/DataStructures/Array.hpp index 2775758d7..70dc91e9a 100644 --- a/include/mgard-x/RuntimeX/DataStructures/Array.hpp +++ b/include/mgard-x/RuntimeX/DataStructures/Array.hpp @@ -96,7 +96,7 @@ void Array::copy( initialize(array.__shape); if (array.device_allocated) { allocate(queue_idx); - MemoryManager::CopyND(dv, __ldvs[D - 1], array.dv, + MemoryManager::Copy2D(dv, __ldvs[D - 1], array.dv, array.__ldvs[D - 1], array.__shape[D - 1], array.linearized_width, queue_idx); } @@ -196,7 +196,7 @@ void Array::load(const T *data, SIZE ld, if (ld == 0) { ld = __shape[D - 1]; } - MemoryManager::CopyND(dv, __ldvs[D - 1], data, ld, __shape[D - 1], + MemoryManager::Copy2D(dv, __ldvs[D - 1], data, ld, __shape[D - 1], linearized_width, queue_idx); } template @@ -226,7 +226,7 @@ T *Array::hostCopy(bool keep, queue_idx); host_allocated = true; } - MemoryManager::CopyND(hv, __shape[D - 1], dv, __ldvs[D - 1], + MemoryManager::Copy2D(hv, __shape[D - 1], dv, __ldvs[D - 1], __shape[D - 1], linearized_width, queue_idx); keepHostCopy = keep; diff --git a/include/mgard-x/RuntimeX/DataStructures/SubArrayCopy.hpp b/include/mgard-x/RuntimeX/DataStructures/SubArrayCopy.hpp index 158b641ed..c92979314 100644 --- a/include/mgard-x/RuntimeX/DataStructures/SubArrayCopy.hpp +++ b/include/mgard-x/RuntimeX/DataStructures/SubArrayCopy.hpp @@ -45,7 +45,7 @@ void SubArrayCopyND(SubArrayType1 subArray1, SubArrayType2 subArray2, using DeviceType = typename CompatibleDeviceType< typename SubArrayType1::DevType, typename SubArrayType2::DevType>::DeviceType; - MemoryManager::CopyND(subArray1.data(), subArray1.getLd(0), + MemoryManager::Copy2D(subArray1.data(), subArray1.getLd(0), subArray2.data(), subArray2.getLd(0), count1, count2, queue_idx); } diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapter.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapter.h index 8ca9a090a..2834b6340 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapter.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapter.h @@ -286,10 +286,16 @@ template class MemoryManager { int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); template - MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, + MGARDX_CONT static void Copy2D(T *dst_ptr, SIZE dst_ld, const T *src_ptr, SIZE src_ld, SIZE n1, SIZE n2, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + template + MGARDX_CONT static void Copy3D(T *dst_ptr, SIZE dst_ld1, SIZE dst_ld2, + const T *src_ptr, SIZE src_ld1, SIZE src_ld2, + SIZE n1, SIZE n2, SIZE n3, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); + template MGARDX_CONT static void MallocHost(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE); diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h index 6e97f662f..e054fb9ea 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h @@ -978,10 +978,10 @@ template <> class MemoryManager { } template - MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, + MGARDX_CONT static void Copy2D(T *dst_ptr, SIZE dst_ld, const T *src_ptr, SIZE src_ld, SIZE n1, SIZE n2, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { - log::dbg("Calling MemoryManager::CopyND"); + log::dbg("Calling MemoryManager::Copy2D"); if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { DeviceRuntime::SyncQueue(queue_idx); } @@ -997,6 +997,54 @@ template <> class MemoryManager { } } + // Copies n3 slices of n2 rows of n1 elements. Element (k, j, i) sits at + // ptr + k * ld2 + j * ld1 + i: ld1 is the row stride and ld2 the slice + // stride, both in elements. + template + MGARDX_CONT static void Copy3D(T *dst_ptr, SIZE dst_ld1, SIZE dst_ld2, + const T *src_ptr, SIZE src_ld1, SIZE src_ld2, + SIZE n1, SIZE n2, SIZE n3, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Copy3D"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + if (n1 == 0 || n2 == 0 || n3 == 0) { + return; + } + // A pitched 3D copy addresses a slice as whole rows, so the slice stride + // must be a multiple of the row stride. A single row per slice is a 2D + // copy in disguise. Otherwise fall back to one 2D copy per slice. + if (n2 == 1) { + Copy2D(dst_ptr, dst_ld2, src_ptr, src_ld2, n1, n3, queue_idx); + return; + } + if (dst_ld2 % dst_ld1 != 0 || src_ld2 % src_ld1 != 0) { + for (SIZE k = 0; k < n3; k++) { + Copy2D(dst_ptr + k * dst_ld2, dst_ld1, src_ptr + k * src_ld2, src_ld1, + n1, n2, queue_idx); + } + return; + } + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); + cudaMemcpy3DParms p = {}; + p.srcPtr = make_cudaPitchedPtr((void *)src_ptr, src_ld1 * sizeof(converted_T), + n1 * sizeof(converted_T), src_ld2 / src_ld1); + p.dstPtr = make_cudaPitchedPtr((void *)dst_ptr, dst_ld1 * sizeof(converted_T), + n1 * sizeof(converted_T), dst_ld2 / dst_ld1); + p.extent = make_cudaExtent(n1 * sizeof(converted_T), n2, n3); + p.kind = cudaMemcpyDefault; + gpuErrchk(cudaMemcpy3DAsync(&p, stream)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + template MGARDX_CONT static void MallocHost(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.h index 68a60dc0b..90b578976 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.h @@ -898,10 +898,10 @@ template <> class MemoryManager { } template - MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, + MGARDX_CONT static void Copy2D(T *dst_ptr, SIZE dst_ld, const T *src_ptr, SIZE src_ld, SIZE n1, SIZE n2, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { - log::dbg("Calling MemoryManager::CopyND"); + log::dbg("Calling MemoryManager::Copy2D"); if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { DeviceRuntime::SyncQueue(queue_idx); } @@ -926,6 +926,54 @@ template <> class MemoryManager { } } + // Copies n3 slices of n2 rows of n1 elements. Element (k, j, i) sits at + // ptr + k * ld2 + j * ld1 + i: ld1 is the row stride and ld2 the slice + // stride, both in elements. + template + MGARDX_CONT static void Copy3D(T *dst_ptr, SIZE dst_ld1, SIZE dst_ld2, + const T *src_ptr, SIZE src_ld1, SIZE src_ld2, + SIZE n1, SIZE n2, SIZE n3, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Copy3D"); + using converted_T = + typename std::conditional::value, Byte, T>::type; + if (n1 == 0 || n2 == 0 || n3 == 0) { + return; + } + // A pitched 3D copy addresses a slice as whole rows, so the slice stride + // must be a multiple of the row stride. A single row per slice is a 2D + // copy in disguise. Otherwise fall back to one 2D copy per slice. + if (n2 == 1) { + Copy2D(dst_ptr, dst_ld2, src_ptr, src_ld2, n1, n3, queue_idx); + return; + } + if (dst_ld2 % dst_ld1 != 0 || src_ld2 % src_ld1 != 0) { + for (SIZE k = 0; k < n3; k++) { + Copy2D(dst_ptr + k * dst_ld2, dst_ld1, src_ptr + k * src_ld2, src_ld1, + n1, n2, queue_idx); + } + return; + } + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + hipStream_t stream = DeviceRuntime::GetQueue(queue_idx); + hipMemcpy3DParms p = {}; + p.srcPtr = make_hipPitchedPtr((void *)src_ptr, src_ld1 * sizeof(converted_T), + n1 * sizeof(converted_T), src_ld2 / src_ld1); + p.dstPtr = make_hipPitchedPtr((void *)dst_ptr, dst_ld1 * sizeof(converted_T), + n1 * sizeof(converted_T), dst_ld2 / dst_ld1); + p.extent = make_hipExtent(n1 * sizeof(converted_T), n2, n3); + p.kind = hipMemcpyDefault; + gpuErrchk(hipMemcpy3DAsync(&p, stream)); + if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { + DeviceRuntime::SyncQueue(queue_idx); + } + if (DeviceRuntime::SyncAllKernelsAndCheckErrors) { + DeviceRuntime::SyncDevice(); + } + } + template MGARDX_CONT static void MallocHost(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h index 510f12d88..805f5f29b 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h @@ -334,12 +334,36 @@ template <> class MemoryManager { } template - MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, + MGARDX_CONT static void Copy2D(T *dst_ptr, SIZE dst_ld, const T *src_ptr, SIZE src_ld, SIZE n1, SIZE n2, int queue_idx) { using converted_T = typename std::conditional::value, Byte, T>::type; - Kokkos::Impl::DeepCopy( - dst_ptr, src_ptr, n1 * n2 * sizeof(converted_T)); + if (dst_ld == n1 && src_ld == n1) { + Kokkos::Impl::DeepCopy( + dst_ptr, src_ptr, n1 * n2 * sizeof(converted_T)); + } else { + // Rows are strided: copy them one by one. + for (SIZE row = 0; row < n2; row++) { + Kokkos::Impl::DeepCopy( + (converted_T *)dst_ptr + row * dst_ld, + (converted_T *)src_ptr + row * src_ld, n1 * sizeof(converted_T)); + } + } + } + + // Copies n3 slices of n2 rows of n1 elements. Element (k, j, i) sits at + // ptr + k * ld2 + j * ld1 + i: ld1 is the row stride and ld2 the slice + // stride, both in elements. + template + MGARDX_CONT static void Copy3D(T *dst_ptr, SIZE dst_ld1, SIZE dst_ld2, + const T *src_ptr, SIZE src_ld1, SIZE src_ld2, + SIZE n1, SIZE n2, SIZE n3, + int queue_idx) { + log::dbg("Calling MemoryManager::Copy3D"); + for (SIZE k = 0; k < n3; k++) { + Copy2D(dst_ptr + k * dst_ld2, dst_ld1, src_ptr + k * src_ld2, src_ld1, + n1, n2, queue_idx); + } } template diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h index 6fcb21a7c..b31958c14 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h @@ -401,10 +401,10 @@ template <> class MemoryManager { } template - MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, + MGARDX_CONT static void Copy2D(T *dst_ptr, SIZE dst_ld, const T *src_ptr, SIZE src_ld, SIZE n1, SIZE n2, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { - log::dbg("Calling MemoryManager::CopyND"); + log::dbg("Calling MemoryManager::Copy2D"); using converted_T = typename std::conditional::value, Byte, T>::type; if ((dst_ld == n1) && (src_ld == n1)) { @@ -420,6 +420,21 @@ template <> class MemoryManager { } } + // Copies n3 slices of n2 rows of n1 elements. Element (k, j, i) sits at + // ptr + k * ld2 + j * ld1 + i: ld1 is the row stride and ld2 the slice + // stride, both in elements. + template + MGARDX_CONT static void Copy3D(T *dst_ptr, SIZE dst_ld1, SIZE dst_ld2, + const T *src_ptr, SIZE src_ld1, SIZE src_ld2, + SIZE n1, SIZE n2, SIZE n3, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Copy3D"); + for (SIZE k = 0; k < n3; k++) { + Copy2D(dst_ptr + k * dst_ld2, dst_ld1, src_ptr + k * src_ld2, src_ld1, + n1, n2, queue_idx); + } + } + template MGARDX_CONT static void MallocHost(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h index 91a1e62c2..bf0a1b47c 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h @@ -889,10 +889,10 @@ template <> class MemoryManager { } template - MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, + MGARDX_CONT static void Copy2D(T *dst_ptr, SIZE dst_ld, const T *src_ptr, SIZE src_ld, SIZE n1, SIZE n2, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { - log::dbg("Calling MemoryManager::CopyND"); + log::dbg("Calling MemoryManager::Copy2D"); using converted_T = typename std::conditional::value, Byte, T>::type; if ((dst_ld == n1) && (src_ld == n1)) { @@ -908,6 +908,21 @@ template <> class MemoryManager { } } + // Copies n3 slices of n2 rows of n1 elements. Element (k, j, i) sits at + // ptr + k * ld2 + j * ld1 + i: ld1 is the row stride and ld2 the slice + // stride, both in elements. + template + MGARDX_CONT static void Copy3D(T *dst_ptr, SIZE dst_ld1, SIZE dst_ld2, + const T *src_ptr, SIZE src_ld1, SIZE src_ld2, + SIZE n1, SIZE n2, SIZE n3, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Copy3D"); + for (SIZE k = 0; k < n3; k++) { + Copy2D(dst_ptr + k * dst_ld2, dst_ld1, src_ptr + k * src_ld2, src_ld1, + n1, n2, queue_idx); + } + } + template MGARDX_CONT static void MallocHost(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h index 239e6ad95..534e55f4d 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h @@ -627,18 +627,27 @@ template <> class MemoryManager { } template - MGARDX_CONT static void CopyND(T *dst_ptr, SIZE dst_ld, const T *src_ptr, + MGARDX_CONT static void Copy2D(T *dst_ptr, SIZE dst_ld, const T *src_ptr, SIZE src_ld, SIZE n1, SIZE n2, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { - log::dbg("Calling MemoryManager::CopyND"); + log::dbg("Calling MemoryManager::Copy2D"); if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { DeviceRuntime::SyncQueue(queue_idx); } sycl::queue q = DeviceRuntime::GetQueue(queue_idx); using converted_T = typename std::conditional::value, Byte, T>::type; - q.memcpy((converted_T *)dst_ptr, (converted_T *)src_ptr, - n1 * n2 * sizeof(converted_T)); + if (dst_ld == n1 && src_ld == n1) { + q.memcpy((converted_T *)dst_ptr, (converted_T *)src_ptr, + n1 * n2 * sizeof(converted_T)); + } else { + // Rows are strided: copy them one by one. + for (SIZE row = 0; row < n2; row++) { + q.memcpy((converted_T *)dst_ptr + row * dst_ld, + (converted_T *)src_ptr + row * src_ld, + n1 * sizeof(converted_T)); + } + } if (queue_idx == MGARDX_SYNCHRONIZED_QUEUE) { DeviceRuntime::SyncQueue(queue_idx); } @@ -647,6 +656,21 @@ template <> class MemoryManager { } } + // Copies n3 slices of n2 rows of n1 elements. Element (k, j, i) sits at + // ptr + k * ld2 + j * ld1 + i: ld1 is the row stride and ld2 the slice + // stride, both in elements. + template + MGARDX_CONT static void Copy3D(T *dst_ptr, SIZE dst_ld1, SIZE dst_ld2, + const T *src_ptr, SIZE src_ld1, SIZE src_ld2, + SIZE n1, SIZE n2, SIZE n3, + int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { + log::dbg("Calling MemoryManager::Copy3D"); + for (SIZE k = 0; k < n3; k++) { + Copy2D(dst_ptr + k * dst_ld2, dst_ld1, src_ptr + k * src_ld2, src_ld1, + n1, n2, queue_idx); + } + } + template MGARDX_CONT static void MallocHost(T *&ptr, SIZE n, int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { diff --git a/include/mgard-x/RuntimeX/Utilities/SubArrayPrinter.hpp b/include/mgard-x/RuntimeX/Utilities/SubArrayPrinter.hpp index 26273a105..5166c6b79 100644 --- a/include/mgard-x/RuntimeX/Utilities/SubArrayPrinter.hpp +++ b/include/mgard-x/RuntimeX/Utilities/SubArrayPrinter.hpp @@ -58,7 +58,7 @@ void PrintSubarray(std::string name, SubArrayType subArray) { // ncol, nrow, D2H, 0); DeviceRuntime::SyncQueue(0); for (SIZE i = 0; i < nrow; i++) { - MemoryManager::CopyND( + MemoryManager::Copy2D( v + ncol * nfib * i, nfib, subArray.data() + subArray.lddv1() * subArray.lddv2() * i, subArray.lddv1(), nfib, ncol, 0); @@ -120,11 +120,11 @@ void CompareSubarray(std::string name, SubArrayType subArray1, T *v2 = new T[nrow * ncol * nfib]; for (SIZE i = 0; i < nrow; i++) { - MemoryManager::CopyND( + MemoryManager::Copy2D( v1 + ncol * nfib * i, nfib, subArray1.data() + subArray1.lddv1() * subArray1.lddv2() * i, subArray1.lddv1(), nfib, ncol, 0); - MemoryManager::CopyND( + MemoryManager::Copy2D( v2 + ncol * nfib * i, nfib, subArray2.data() + subArray2.lddv1() * subArray2.lddv2() * i, subArray2.lddv1(), nfib, ncol, 0); @@ -201,11 +201,11 @@ void CompareSubarray(std::string name, SubArrayType1 subArray1, T *v1 = new T[nrow * ncol * nfib]; T *v2 = new T[nrow * ncol * nfib]; for (SIZE i = 0; i < nrow; i++) { - MemoryManager::CopyND( + MemoryManager::Copy2D( v1 + ncol * nfib * i, nfib, subArray1.data() + subArray1.lddv1() * subArray1.lddv2() * i, subArray1.lddv1(), nfib, ncol, 0); - MemoryManager::CopyND(v2 + ncol * nfib * i, nfib, + MemoryManager::Copy2D(v2 + ncol * nfib * i, nfib, subArray2.data() + subArray2.lddv1 * subArray2.lddv2 * i, subArray2.lddv1, nfib, ncol, 0); @@ -377,7 +377,7 @@ void verify_matrix_cuda(SIZE nrow, SIZE ncol, SIZE nfib, T *dv, SIZE lddv1, // ncol, dv, lddv1 * sizeof(T), sizex * sizeof(T), // lddv2, nfib * sizeof(T), ncol, nrow, D2H, // queue_idx); - // MemoryManager::CopyND(v, nfib, dv, lddv1, nfib, ncol * nrow, + // MemoryManager::Copy2D(v, nfib, dv, lddv1, nfib, ncol * nrow, // 0); DeviceRuntime::SyncQueue(0); tmp_handle->sync(queue_idx); verify_matrix(nrow, ncol, nfib, v, nfib, ncol, file_prefix, store, verify); delete[] v; @@ -421,7 +421,7 @@ void DumpSubArray(std::string name, SubArray subArray) { T *v = new T[nrow * ncol * nfib]; DeviceRuntime::SyncQueue(0); - MemoryManager::CopyND(v, nfib, subArray.data(), + MemoryManager::Copy2D(v, nfib, subArray.data(), subArray.ld(D - 1), nfib, ncol * nrow, 0); DeviceRuntime::SyncQueue(0); std::fstream myfile; @@ -466,7 +466,7 @@ void LoadSubArray(std::string name, SubArray subArray) { printf("Error occurred at read time!\n"); return; } - MemoryManager::CopyND(subArray.data(), subArray.ld(D - 1), v, + MemoryManager::Copy2D(subArray.data(), subArray.ld(D - 1), v, nfib, nfib, ncol * nrow, 0); delete[] v; } @@ -505,7 +505,7 @@ void VerifySubArray(std::string name, SubArray subArray, return; } - MemoryManager::CopyND(v2, nfib, subArray.data(), + MemoryManager::Copy2D(v2, nfib, subArray.data(), subArray.ld(D - 1), nfib, ncol * nrow, 0); DeviceRuntime::SyncQueue(0); diff --git a/include/mgard-x/Utilities/Types.h b/include/mgard-x/Utilities/Types.h index d19f34ef4..888cb3d14 100644 --- a/include/mgard-x/Utilities/Types.h +++ b/include/mgard-x/Utilities/Types.h @@ -106,7 +106,13 @@ enum class endiness_type : uint8_t { Little_Endian, Big_Endian }; enum class coordinate_location : uint8_t { Embedded, External }; -enum class domain_decomposition_type : uint8_t { MaxDim, Block, Variable }; +// How the input is split into blocks that are compressed one at a time (see +// DomainDecomposer): +// - Auto: decide whether splitting is needed to fit the memory limit and, if +// so, pick the block shape. +// - Manual: use Config::domain_decomposition_block_size as given; it is +// checked (every extent >= 3, fits the memory limit) and rejected otherwise. +enum class domain_decomposition_mode_type : uint8_t { Auto, Manual }; enum class operation_type : uint8_t { Compression, MDR }; diff --git a/src/mgard-x/Config/Config.cpp b/src/mgard-x/Config/Config.cpp index 0bd5a3a01..2eb09e161 100644 --- a/src/mgard-x/Config/Config.cpp +++ b/src/mgard-x/Config/Config.cpp @@ -33,7 +33,8 @@ Config::Config() { dev_type = device_type::AUTO; dev_id = 0; compressor = compressor_type::MGARD; - domain_decomposition = domain_decomposition_type::MaxDim; + domain_decomposition_mode = domain_decomposition_mode_type::Auto; + domain_decomposition_block_size = std::vector(); decomposition = decomposition_type::MultiDim; estimate_outlier_ratio = 1.0; huff_dict_size = 8192; @@ -48,9 +49,6 @@ Config::Config() { auto_pin_host_buffers = true; max_memory_footprint = std::numeric_limits::max(); // no limit total_num_bitplanes = 32; - block_size = 256; - domain_decomposition_dim = 0; - domain_decomposition_sizes = std::vector(); mdr_adaptive_resolution = false; adjust_shape = false; compress_with_dryrun = false; diff --git a/src/mgard-x/Executables/ArgumentParser.h b/src/mgard-x/Executables/ArgumentParser.h index cc072f130..194452f1d 100644 --- a/src/mgard-x/Executables/ArgumentParser.h +++ b/src/mgard-x/Executables/ArgumentParser.h @@ -119,6 +119,28 @@ enum mgard_x::error_bound_type get_error_bound_mode(int argc, char *argv[]) { return mode; } +// -dd / --domain-decomposition (default: auto) and, for manual, +// -dd-size / --domain-decomposition-size ... : the block +// shape, slowest dimension first. +void get_domain_decomposition(int argc, char *argv[], mgard_x::Config &config) { + std::string dd = "auto"; + if (has_arg(argc, argv, "-dd", "--domain-decomposition")) { + dd = get_arg(argc, argv, "Domain decomposition", "-dd", + "--domain-decomposition"); + } + if (dd == "auto") { + config.domain_decomposition_mode = mgard_x::domain_decomposition_mode_type::Auto; + } else if (dd == "manual") { + config.domain_decomposition_mode = + mgard_x::domain_decomposition_mode_type::Manual; + config.domain_decomposition_block_size = get_args( + argc, argv, "Domain decomposition block size", "-dd-size", + "--domain-decomposition-size"); + } else { + throw std::runtime_error("Wrong domain decomposition mode."); + } +} + enum mgard_x::device_type get_device_type(int argc, char *argv[]) { enum mgard_x::device_type dev_type = mgard_x::device_type::AUTO; std::string dev = diff --git a/src/mgard-x/Executables/mdr-x-qoi.cpp b/src/mgard-x/Executables/mdr-x-qoi.cpp index 22b17492c..4395b8691 100644 --- a/src/mgard-x/Executables/mdr-x-qoi.cpp +++ b/src/mgard-x/Executables/mdr-x-qoi.cpp @@ -41,8 +41,6 @@ void print_usage_message(std::string error) { \t\t -d / --device : device type\n\ \t\t (optional) -v / --verbose <0|1|2|3> 0: error; 1: error+info; 2: error+timing; 3: all\n\ \t\t (optional) -m / --max-memory \n\ -\t\t (optional) -dd / --domain-decomposition \n\ -\t\t\t (optional) -dd-size / --domain-decomposition-size (for block domain decomposition only) \n\ \n\ \t -x / --reconstruct: reconstruct data\n\ \t\t -i / --input \n\ @@ -353,7 +351,6 @@ template int launch_refactor(mgard_x::DIM D, enum mgard_x::data_type dtype, std::string input_file, std::string output_file, std::vector shape, - std::string domain_decomposition, mgard_x::SIZE block_size, enum mgard_x::device_type dev_type, int verbose, mgard_x::SIZE max_memory_footprint) { @@ -361,19 +358,11 @@ int launch_refactor(mgard_x::DIM D, enum mgard_x::data_type dtype, config.normalize_coordinates = false; config.log_level = verbose_to_log_level(verbose); config.decomposition = mgard_x::decomposition_type::MultiDim; - if (domain_decomposition == "max-dim") { - config.domain_decomposition = mgard_x::domain_decomposition_type::MaxDim; - } else if (domain_decomposition == "block") { - config.domain_decomposition = mgard_x::domain_decomposition_type::Block; - config.block_size = block_size; - } else if (domain_decomposition == "variable") { - config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; - } - - config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; - config.domain_decomposition_dim = 0; - config.domain_decomposition_sizes = {shape[0] / 3, shape[0] / 3, - shape[0] / 3}; + // One subdomain per variable: the variables are stacked along dimension 0. + config.domain_decomposition_mode = + mgard_x::domain_decomposition_mode_type::Manual; + config.domain_decomposition_block_size = shape; + config.domain_decomposition_block_size[0] = shape[0] / 3; config.dev_type = dev_type; config.max_memory_footprint = max_memory_footprint; @@ -493,10 +482,6 @@ int launch_reconstruct(std::string input_file, std::string output_file, config.mdr_qoi_mode = true; config.mdr_qoi_num_variables = shape.size(); - config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; - config.domain_decomposition_dim = 0; - config.domain_decomposition_sizes = {shape[0] / 3, shape[0] / 3, - shape[0] / 3}; mgard_x::Byte *original_data; size_t in_size = 0; @@ -741,25 +726,13 @@ bool try_refactoring(int argc, char *argv[]) { max_memory_footprint = (mgard_x::SIZE)get_arg( argc, argv, "Max memory", "-m", "--max-memory"); } - std::string domain_decomposition = "max-dim"; - mgard_x::SIZE block_size = 0; - if (has_arg(argc, argv, "-dd", "--domain-decomposition")) { - domain_decomposition = get_arg( - argc, argv, "Domain decomposition", "-dd", "--domain-decomposition"); - if (domain_decomposition == "block") { - block_size = get_arg(argc, argv, "Block size", "-dd-size", - "--domain-decomposition-size"); - } - } if (dtype == mgard_x::data_type::Double) { launch_refactor(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, domain_decomposition, - block_size, dev_type, verbose, + output_file.c_str(), shape, dev_type, verbose, max_memory_footprint); } else if (dtype == mgard_x::data_type::Float) { launch_refactor(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, domain_decomposition, - block_size, dev_type, verbose, max_memory_footprint); + output_file.c_str(), shape, dev_type, verbose, max_memory_footprint); } return true; } diff --git a/src/mgard-x/Executables/mdr-x.cpp b/src/mgard-x/Executables/mdr-x.cpp index 4106f4529..691e68681 100644 --- a/src/mgard-x/Executables/mdr-x.cpp +++ b/src/mgard-x/Executables/mdr-x.cpp @@ -41,8 +41,10 @@ void print_usage_message(std::string error) { \t\t -d / --device : device type\n\ \t\t (optional) -v / --verbose <0|1|2|3> 0: error; 1: error+info; 2: error+timing; 3: all\n\ \t\t (optional) -m / --max-memory \n\ -\t\t (optional) -dd / --domain-decomposition \n\ -\t\t\t (optional) -dd-size / --domain-decomposition-size (for block domain decomposition only) \n\ +\t\t (optional) -dd / --domain-decomposition : domain decomposition\n\ +\t\t\t (default: auto -- split only when the data does not fit the memory limit)\n\ +\t\t (optional) -dd-size / --domain-decomposition-size ...: with -dd manual,\n\ +\t\t\t number of dimensions followed by the block size per dimension (slowest first)\n\ \t\t (optional) -l / --lossless : bitplane lossless stage (default: huffman)\n\ \n\ \t -x / --reconstruct: reconstruct data\n\ @@ -305,7 +307,7 @@ template int launch_refactor(mgard_x::DIM D, enum mgard_x::data_type dtype, std::string input_file, std::string output_file, std::vector shape, - std::string domain_decomposition, mgard_x::SIZE block_size, + mgard_x::Config dd_config, enum mgard_x::device_type dev_type, int verbose, mgard_x::SIZE max_memory_footprint, enum mgard_x::lossless_type lossless) { @@ -315,21 +317,9 @@ int launch_refactor(mgard_x::DIM D, enum mgard_x::data_type dtype, config.lossless = lossless; config.log_level = verbose_to_log_level(verbose); config.decomposition = mgard_x::decomposition_type::MultiDim; - if (domain_decomposition == "max-dim") { - config.domain_decomposition = mgard_x::domain_decomposition_type::MaxDim; - } else if (domain_decomposition == "block") { - config.domain_decomposition = mgard_x::domain_decomposition_type::Block; - config.block_size = block_size; - } else if (domain_decomposition == "variable") { - config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; - } - - // config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; - // config.domain_decomposition_dim = 0; - // config.domain_decomposition_sizes = {512, 512, 512, 512, 512, 512, 512, - // 512}; config.domain_decomposition_sizes = {98, 98, 98, 98, 98, 98, 98, 98}; - // config.domain_decomposition_sizes = std::vector(8, 256); - // config.domain_decomposition_sizes = std::vector(8, 100); + config.domain_decomposition_mode = dd_config.domain_decomposition_mode; + config.domain_decomposition_block_size = + dd_config.domain_decomposition_block_size; config.dev_type = dev_type; config.max_memory_footprint = max_memory_footprint; @@ -404,13 +394,6 @@ int launch_reconstruct(std::string input_file, std::string output_file, config.dev_type = dev_type; config.mdr_adaptive_resolution = adaptive_resolution; - // config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; - // config.domain_decomposition_dim = 0; - // config.domain_decomposition_sizes = {512, 512, 512, 512, 512, 512, 512, - // 512}; config.domain_decomposition_sizes = {98, 98, 98, 98, 98, 98, 98, 98}; - // config.domain_decomposition_sizes = std::vector(8, 256); - // config.domain_decomposition_sizes = std::vector(8, 100); - size_t original_size = 1; T *original_data = nullptr; if (original_file.compare("none") != 0) { @@ -518,25 +501,17 @@ bool try_refactoring(int argc, char *argv[]) { max_memory_footprint = (mgard_x::SIZE)get_arg( argc, argv, "Max memory", "-m", "--max-memory"); } - std::string domain_decomposition = "max-dim"; - mgard_x::SIZE block_size = 0; - if (has_arg(argc, argv, "-dd", "--domain-decomposition")) { - domain_decomposition = get_arg( - argc, argv, "Domain decomposition", "-dd", "--domain-decomposition"); - if (domain_decomposition == "block") { - block_size = get_arg(argc, argv, "Block size", "-dd-size", - "--domain-decomposition-size"); - } - } + mgard_x::Config dd_config; + get_domain_decomposition(argc, argv, dd_config); if (dtype == mgard_x::data_type::Double) { launch_refactor(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, domain_decomposition, - block_size, dev_type, verbose, max_memory_footprint, + output_file.c_str(), shape, dd_config, + dev_type, verbose, max_memory_footprint, lossless); } else if (dtype == mgard_x::data_type::Float) { launch_refactor(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, domain_decomposition, - block_size, dev_type, verbose, max_memory_footprint, + output_file.c_str(), shape, dd_config, + dev_type, verbose, max_memory_footprint, lossless); } return true; diff --git a/src/mgard-x/Executables/mgard-x.cpp b/src/mgard-x/Executables/mgard-x.cpp index 54ba58802..5f7054ea0 100644 --- a/src/mgard-x/Executables/mgard-x.cpp +++ b/src/mgard-x/Executables/mgard-x.cpp @@ -46,6 +46,10 @@ void print_usage_message(std::string error) { \t\t -s / --smoothness : smoothness parameter\n\ \t\t -l / --lossless : lossless compression\n\ \t\t -d / --device : device type\n\ +\t\t (optional) -dd / --domain-decomposition : domain decomposition\n\ +\t\t\t (default: auto -- split only when the data does not fit the memory limit)\n\ +\t\t (optional) -dd-size / --domain-decomposition-size ...: with -dd manual,\n\ +\t\t\t number of dimensions followed by the block size per dimension (slowest first)\n\ \t\t (optional) -hh / --hybrid: use hybrid (block-local + global) hierarchy\n\ \t\t (optional) -ll / --local-levels : number of hybrid block-local refactoring levels (default: 1)\n\ \t\t (optional) -gl / --global-levels : number of global refactoring levels,\n\ @@ -559,7 +563,7 @@ int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, std::vector shape, double tol, std::vector tol_map, bool enable_roi, double s, enum mgard_x::error_bound_type mode, std::string lossless, - std::string domain_decomposition, mgard_x::SIZE block_size, + mgard_x::Config dd_config, enum mgard_x::device_type dev_type, int verbose, mgard_x::SIZE max_memory_footprint, mgard_x::SIZE num_local_levels, @@ -596,37 +600,12 @@ int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, // config.compressor = mgard_x::compressor_type::ZFP; - if (domain_decomposition == "block") { - config.domain_decomposition = mgard_x::domain_decomposition_type::Block; - config.block_size = block_size; - } else { - config.domain_decomposition = mgard_x::domain_decomposition_type::MaxDim; - } + config.domain_decomposition_mode = dd_config.domain_decomposition_mode; + config.domain_decomposition_block_size = + dd_config.domain_decomposition_block_size; config.cpu_mode = mgard_x::cpu_parallelization_mode::INTRA_BLOCK; - // config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; - config.domain_decomposition_dim = 0; - // NYX - // config.domain_d ecomposition_sizes = {512, 512}; - - // config.domain_decomposition_sizes = {512, 512, 512, 512}; - // config.domain_decomposition_sizes = {2048}; - // config.domain_decomposition_sizes = {128, 248, 315, 348, 384, 424, 201}; - // config.domain_decomposition_sizes = std::vector(128, 16); - // config.domain_decomposition_sizes = std::vector(128, 16); - - // XGC - // config.domain_decomposition_sizes = {312, 312, 312, 312}; - // config.domain_decomposition_sizes = {1248}; - // config.domain_decomposition_sizes = {156, 283, 514, 295}; - // config.domain_decomposition_sizes = std::vector(96, 13); - - // E3SM - // config.domain_decomposition_sizes = {720, 720, 720, 720}; - // config.domain_decomposition_sizes = {180, 368, 463, 529, 605, 692, 43}; - // config.domain_decomposition_sizes = std::vector(192, 15); - config.estimate_outlier_ratio = 1.0; config.dev_type = dev_type; @@ -977,28 +956,20 @@ bool try_compression(int argc, char *argv[]) { "-gl", "--global-levels"); } - std::string domain_decomposition = "max-dim"; - mgard_x::SIZE block_size = 0; - if (has_arg(argc, argv, "-dd", "--domain-decomposition")) { - domain_decomposition = get_arg( - argc, argv, "Domain decomposition", "-dd", "--domain-decomposition"); - if (domain_decomposition == "block") { - block_size = get_arg(argc, argv, "Block size", "-dd-size", - "--domain-decomposition-size"); - } - } + mgard_x::Config dd_config; + get_domain_decomposition(argc, argv, dd_config); if (dtype == mgard_x::data_type::Double) { launch_compress( shape.size(), dtype, input_file.c_str(), output_file.c_str(), shape, - tol, tol_map, enable_roi, s, mode, lossless, domain_decomposition, - block_size, dev_type, verbose, max_memory_footprint, num_local_levels, + tol, tol_map, enable_roi, s, mode, lossless, dd_config, + dev_type, verbose, max_memory_footprint, num_local_levels, num_global_levels, use_hybrid, warm_up, kernel_fusion, projection_mode); } else if (dtype == mgard_x::data_type::Float) { launch_compress( shape.size(), dtype, input_file.c_str(), output_file.c_str(), shape, - tol, tol_map, enable_roi, s, mode, lossless, domain_decomposition, - block_size, dev_type, verbose, max_memory_footprint, num_local_levels, + tol, tol_map, enable_roi, s, mode, lossless, dd_config, + dev_type, verbose, max_memory_footprint, num_local_levels, num_global_levels, use_hybrid, warm_up, kernel_fusion, projection_mode); } mgard_x::release_cache(mgard_x::Config()); diff --git a/src/mgard-x/Executables/pmdr-x-qoi-io.cpp b/src/mgard-x/Executables/pmdr-x-qoi-io.cpp index 60816c24a..d0854e796 100644 --- a/src/mgard-x/Executables/pmdr-x-qoi-io.cpp +++ b/src/mgard-x/Executables/pmdr-x-qoi-io.cpp @@ -43,9 +43,7 @@ void print_usage_message(std::string error) { \t\t -d / --device : device type\n\ \t\t (optional) -v / --verbose <0|1|2|3> 0: error; 1: error+info; 2: error+timing; 3: all\n\ \t\t (optional) -m / --max-memory \n\ - \t\t (optional) -dd / --domain-decomposition \n\ - \t\t\t (optional) -dd-size / --domain-decomposition-size (for block domain decomposition only) \n\ - \n\ + \n\ \t -x / --reconstruct: reconstruct data\n\ \t\t -i / --input \n\ \t\t -o / --output \n\ @@ -707,7 +705,6 @@ template int launch_refactor(mgard_x::DIM D, enum mgard_x::data_type dtype, std::string input_file, std::string output_file, std::vector shape, - std::string domain_decomposition, mgard_x::SIZE block_size, enum mgard_x::device_type dev_type, int verbose, mgard_x::SIZE max_memory_footprint, int rank) { @@ -715,19 +712,11 @@ int launch_refactor(mgard_x::DIM D, enum mgard_x::data_type dtype, config.normalize_coordinates = false; config.log_level = verbose_to_log_level(verbose); config.decomposition = mgard_x::decomposition_type::MultiDim; - if (domain_decomposition == "max-dim") { - config.domain_decomposition = mgard_x::domain_decomposition_type::MaxDim; - } else if (domain_decomposition == "block") { - config.domain_decomposition = mgard_x::domain_decomposition_type::Block; - config.block_size = block_size; - } else if (domain_decomposition == "variable") { - config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; - } - - config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; - config.domain_decomposition_dim = 0; - config.domain_decomposition_sizes = {shape[0] / 3, shape[0] / 3, - shape[0] / 3}; + // One subdomain per variable: the variables are stacked along dimension 0. + config.domain_decomposition_mode = + mgard_x::domain_decomposition_mode_type::Manual; + config.domain_decomposition_block_size = shape; + config.domain_decomposition_block_size[0] = shape[0] / 3; config.dev_type = dev_type; config.max_memory_footprint = max_memory_footprint; @@ -842,10 +831,6 @@ int launch_reconstruct(std::string input_file, std::string output_file, config.mdr_qoi_mode = true; config.mdr_qoi_num_variables = shape.size(); - config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; - config.domain_decomposition_dim = 0; - config.domain_decomposition_sizes = {shape[0] / 3, shape[0] / 3, - shape[0] / 3}; mgard_x::Byte *original_data; size_t in_size = 0; @@ -1176,25 +1161,13 @@ bool try_refactoring(int argc, char *argv[]) { max_memory_footprint = (mgard_x::SIZE)get_arg( argc, argv, "Max memory", "-m", "--max-memory"); } - std::string domain_decomposition = "max-dim"; - mgard_x::SIZE block_size = 0; - if (has_arg(argc, argv, "-dd", "--domain-decomposition")) { - domain_decomposition = get_arg( - argc, argv, "Domain decomposition", "-dd", "--domain-decomposition"); - if (domain_decomposition == "block") { - block_size = get_arg(argc, argv, "Block size", "-dd-size", - "--domain-decomposition-size"); - } - } if (dtype == mgard_x::data_type::Double) { launch_refactor(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, domain_decomposition, - block_size, dev_type, verbose, max_memory_footprint, + output_file.c_str(), shape, dev_type, verbose, max_memory_footprint, rank); } else if (dtype == mgard_x::data_type::Float) { launch_refactor(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, domain_decomposition, - block_size, dev_type, verbose, max_memory_footprint, + output_file.c_str(), shape, dev_type, verbose, max_memory_footprint, rank); } return true; diff --git a/src/mgard-x/Executables/pmdr-x-qoi.cpp b/src/mgard-x/Executables/pmdr-x-qoi.cpp index f731a1106..d34c173b0 100644 --- a/src/mgard-x/Executables/pmdr-x-qoi.cpp +++ b/src/mgard-x/Executables/pmdr-x-qoi.cpp @@ -43,9 +43,7 @@ void print_usage_message(std::string error) { \t\t -d / --device : device type\n\ \t\t (optional) -v / --verbose <0|1|2|3> 0: error; 1: error+info; 2: error+timing; 3: all\n\ \t\t (optional) -m / --max-memory \n\ - \t\t (optional) -dd / --domain-decomposition \n\ - \t\t\t (optional) -dd-size / --domain-decomposition-size (for block domain decomposition only) \n\ - \n\ + \n\ \t -x / --reconstruct: reconstruct data\n\ \t\t -i / --input \n\ \t\t -o / --output \n\ @@ -659,7 +657,6 @@ template int launch_refactor(mgard_x::DIM D, enum mgard_x::data_type dtype, std::string input_file, std::string output_file, std::vector shape, - std::string domain_decomposition, mgard_x::SIZE block_size, enum mgard_x::device_type dev_type, int verbose, mgard_x::SIZE max_memory_footprint, int rank) { @@ -667,19 +664,11 @@ int launch_refactor(mgard_x::DIM D, enum mgard_x::data_type dtype, config.normalize_coordinates = false; config.log_level = verbose_to_log_level(verbose); config.decomposition = mgard_x::decomposition_type::MultiDim; - if (domain_decomposition == "max-dim") { - config.domain_decomposition = mgard_x::domain_decomposition_type::MaxDim; - } else if (domain_decomposition == "block") { - config.domain_decomposition = mgard_x::domain_decomposition_type::Block; - config.block_size = block_size; - } else if (domain_decomposition == "variable") { - config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; - } - - config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; - config.domain_decomposition_dim = 0; - config.domain_decomposition_sizes = {shape[0] / 3, shape[0] / 3, - shape[0] / 3}; + // One subdomain per variable: the variables are stacked along dimension 0. + config.domain_decomposition_mode = + mgard_x::domain_decomposition_mode_type::Manual; + config.domain_decomposition_block_size = shape; + config.domain_decomposition_block_size[0] = shape[0] / 3; config.dev_type = dev_type; config.max_memory_footprint = max_memory_footprint; @@ -794,10 +783,6 @@ int launch_reconstruct(std::string input_file, std::string output_file, config.mdr_qoi_mode = true; config.mdr_qoi_num_variables = shape.size(); - config.domain_decomposition = mgard_x::domain_decomposition_type::Variable; - config.domain_decomposition_dim = 0; - config.domain_decomposition_sizes = {shape[0] / 3, shape[0] / 3, - shape[0] / 3}; mgard_x::Byte *original_data; size_t in_size = 0; @@ -1127,25 +1112,13 @@ bool try_refactoring(int argc, char *argv[]) { max_memory_footprint = (mgard_x::SIZE)get_arg( argc, argv, "Max memory", "-m", "--max-memory"); } - std::string domain_decomposition = "max-dim"; - mgard_x::SIZE block_size = 0; - if (has_arg(argc, argv, "-dd", "--domain-decomposition")) { - domain_decomposition = get_arg( - argc, argv, "Domain decomposition", "-dd", "--domain-decomposition"); - if (domain_decomposition == "block") { - block_size = get_arg(argc, argv, "Block size", "-dd-size", - "--domain-decomposition-size"); - } - } if (dtype == mgard_x::data_type::Double) { launch_refactor(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, domain_decomposition, - block_size, dev_type, verbose, max_memory_footprint, + output_file.c_str(), shape, dev_type, verbose, max_memory_footprint, rank); } else if (dtype == mgard_x::data_type::Float) { launch_refactor(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, domain_decomposition, - block_size, dev_type, verbose, max_memory_footprint, + output_file.c_str(), shape, dev_type, verbose, max_memory_footprint, rank); } return true; diff --git a/src/mgard-x/Metadata/Metadata.cpp b/src/mgard-x/Metadata/Metadata.cpp index c9a32fa4c..9add9a2be 100644 --- a/src/mgard-x/Metadata/Metadata.cpp +++ b/src/mgard-x/Metadata/Metadata.cpp @@ -234,8 +234,6 @@ DeserializeROITolerances(const mgard::pb::RegionOfInterest &roi) { namespace mgard_x { void MetadataBase::InitializeConfig(Config &config) { - config.domain_decomposition = ddtype; - config.domain_decomposition_dim = domain_decomposed_dim; config.decomposition = decomposition; config.lossless = ltype; // Only overwrite a backend's parameters when the metadata actually carries @@ -345,15 +343,11 @@ void MetadataBase::PrintSummary() { } std::cout << "Domain Decomposition: "; if (domain_decomposed) { - if (ddtype == domain_decomposition_type::MaxDim) { - std::cout << "MaxDim\n"; - } else if (ddtype == domain_decomposition_type::Variable) { - std::cout << "Variable\n"; - } else { - std::cout << "Block\n"; + std::cout << "blocks of "; + for (size_t d = 0; d < domain_decomposition_block_shape.size(); d++) { + std::cout << (d ? "x" : "") << domain_decomposition_block_shape[d]; } - std::cout << "Decomposed Dim: " << domain_decomposed_dim << "\n"; - std::cout << "Decomposed Size: " << domain_decomposed_size << "\n"; + std::cout << "\n"; } else { std::cout << "No\n"; } @@ -507,21 +501,14 @@ std::vector MetadataBase::Serialize() { mgard::pb::DomainDecomposition &domainDecomposition = *header.mutable_domain_decomposition(); if (domain_decomposed) { - if (ddtype == domain_decomposition_type::MaxDim) { - domainDecomposition.set_method( - mgard::pb::DomainDecomposition::MAX_DIMENSION); - } else if (ddtype == domain_decomposition_type::Block) { - domainDecomposition.set_method(mgard::pb::DomainDecomposition::BLOCK); - } else if (ddtype == domain_decomposition_type::Variable) { - domainDecomposition.set_method( - mgard::pb::DomainDecomposition::VARIABLE); + domainDecomposition.set_method(mgard::pb::DomainDecomposition::BLOCK); + for (uint64_t extent : domain_decomposition_block_shape) { + domainDecomposition.add_block_shape(extent); } } else { domainDecomposition.set_method( mgard::pb::DomainDecomposition::NOOP_METHOD); } - domainDecomposition.set_decomposition_dimension(domain_decomposed_dim); - domainDecomposition.set_decomposition_size(domain_decomposed_size); } { // Function Decomposition @@ -826,24 +813,43 @@ void MetadataBase::Deserialize( { // Domain Decomposition const mgard::pb::DomainDecomposition domainDecomposition = header.domain_decomposition(); - if (domainDecomposition.method() != - mgard::pb::DomainDecomposition::NOOP_METHOD) { - domain_decomposed = true; - if (domainDecomposition.method() == - mgard::pb::DomainDecomposition::MAX_DIMENSION) { - ddtype = domain_decomposition_type::MaxDim; - } else if (domainDecomposition.method() == - mgard::pb::DomainDecomposition::BLOCK) { - ddtype = domain_decomposition_type::Block; - } else if (domainDecomposition.method() == - mgard::pb::DomainDecomposition::VARIABLE) { - ddtype = domain_decomposition_type::Variable; + // Every decomposition is read back as a block shape; the two layouts of + // MGARD-X 1.6 and earlier are blocks of a particular shape. + domain_decomposition_block_shape = shape; + domain_decomposed = domainDecomposition.method() != + mgard::pb::DomainDecomposition::NOOP_METHOD; + if (domainDecomposition.method() == + mgard::pb::DomainDecomposition::BLOCK) { + if (domainDecomposition.block_shape_size() > 0) { + if ((size_t)domainDecomposition.block_shape_size() != shape.size()) { + throw InvalidDataException( + "domain decomposition block shape does not match the data " + "dimensions."); + } + for (size_t d = 0; d < shape.size(); d++) { + domain_decomposition_block_shape[d] = + domainDecomposition.block_shape(d); + } + } else { + for (size_t d = 0; d < shape.size(); d++) { + domain_decomposition_block_shape[d] = + domainDecomposition.decomposition_size(); + } } - - domain_decomposed_dim = domainDecomposition.decomposition_dimension(); - domain_decomposed_size = domainDecomposition.decomposition_size(); - } else { - domain_decomposed = false; + } else if (domainDecomposition.method() == + mgard::pb::DomainDecomposition::MAX_DIMENSION) { + uint64_t dim = domainDecomposition.decomposition_dimension(); + if (dim >= shape.size()) { + throw InvalidDataException( + "domain decomposition dimension out of range."); + } + domain_decomposition_block_shape[dim] = + domainDecomposition.decomposition_size(); + } else if (domainDecomposition.method() != + mgard::pb::DomainDecomposition::NOOP_METHOD) { + throw InvalidDataException( + "unsupported domain decomposition method (variable-size " + "decomposition is no longer supported)."); } } diff --git a/src/mgard.proto b/src/mgard.proto index 07d97e769..05a80b5b3 100644 --- a/src/mgard.proto +++ b/src/mgard.proto @@ -74,8 +74,16 @@ message DomainDecomposition { } Method method = 1; + // MAX_DIMENSION (MGARD-X 1.6 and earlier): slabs of decomposition_size + // nodes along dimension decomposition_dimension. uint64 decomposition_dimension = 2; + // BLOCK without block_shape (MGARD-X 1.6 and earlier): cubic blocks of + // decomposition_size nodes per dimension. uint64 decomposition_size = 3; + // BLOCK: the shape of every block, one extent per dimension. Along each + // dimension block i starts at i * block_shape[d]; a remainder of fewer than + // 3 nodes is merged into the block before it. + repeated uint64 block_shape = 4; } // Per-block error tolerances for the region-of-interest (ROI) mode of the diff --git a/tests/mgard-x/src/test_HybridMetadata.cpp.in b/tests/mgard-x/src/test_HybridMetadata.cpp.in index d051e1def..3075e5364 100644 --- a/tests/mgard-x/src/test_HybridMetadata.cpp.in +++ b/tests/mgard-x/src/test_HybridMetadata.cpp.in @@ -28,8 +28,7 @@ Metadata base_metadata(std::vector shape) { (float)1.0, decomposition_type::Hybrid, lossless_type::Huffman, /*huff_dict_size=*/8192, /*huff_block_size=*/1024, shape, /*domain_decomposed=*/false, - domain_decomposition_type::MaxDim, /*domain_decomposed_dim=*/0, - /*domain_decomposed_size=*/0); + /*domain_decomposition_block_shape=*/shape); return m; } From 6e0c3e77621d280e10bff3a15d4f984067a554f4 Mon Sep 17 00:00:00 2001 From: Jieyang Chen Date: Sun, 11 Oct 2026 11:44:15 -0700 Subject: [PATCH 09/11] docs: update 1.7.0 release notes and MGARD-X guide Release notes: add MGARD 1.6.x data compatibility, the generic block domain decomposition (Auto/Manual), Copy3D, MDR-X ZSTD bitplane compression, the breaking Config changes (domain decomposition settings, num_global_refactoring_level), the memory reductions, optional MPI for MDR, and the recent fixes. MGARD-X guide: document MGARD_ENABLE_DATA_COMPATIBILITY and decompressing data from earlier versions, the domain decomposition mode and block size, and the DataCompatibilityNotEnabledFailure status. Bring the CLI section in line with the current mgard-x options, and fix stale Config entries (projection_mode, huff_block_size default, nonexistent num_dev). Co-Authored-By: Claude Opus 5.5 --- doc/MGARD-X.md | 88 +++++++++++++++++++++----------------- doc/release notes/1.7.0.md | 21 +++++++++ 2 files changed, 70 insertions(+), 39 deletions(-) diff --git a/doc/MGARD-X.md b/doc/MGARD-X.md index ec2dffd4d..d062a76c6 100644 --- a/doc/MGARD-X.md +++ b/doc/MGARD-X.md @@ -84,7 +84,16 @@ MGARD-X is a portable implementation of the MGARD lossy compressor supporting va |MGARD\_ENABLE\_HIP|ON/OFF|OFF|Enable portable GPU compression/decompression with HIP on AMD GPUs| |MGARD\_ENABLE\_SYCL|ON/OFF|OFF|Enable portable GPU compression/decompression with SYCL on Intel GPUs| |MGARD\_ENABLE\_AUTO\_TUNING|ON/OFF|OFF|Enable auto tuning feature| -|MGARD\_ENABLE\_MDR|ON/OFF|OFF|Enable building MDR and MDR-X| +|MGARD\_ENABLE\_MDR|ON/OFF|OFF|Enable building MDR and MDR-X. MPI is optional: the parallel ```pmdr-x-qoi``` and ```pmdr-x-qoi-io``` executables are built only when MPI is found| +|MGARD\_ENABLE\_DATA\_COMPATIBILITY|ON/OFF|OFF|Also build the MGARD 1.6.0-compatible decompression pipeline so that data compressed by MGARD 1.6.x and earlier can be decompressed (see [Decompressing data from earlier versions](#decompressing-data-from-earlier-versions)). Roughly doubles the MGARD-X build time| + +## Decompressing data from earlier versions +Data compressed by MGARD 1.7.0 uses a different format from MGARD 1.6.x (hierarchy geometry, Huffman layout, default number of levels, and domain decomposition header). Decompression reads the software version recorded in the header of the compressed data and handles each version as follows: + +* **Compressed by 1.7.0:** decompressed by the 1.7.0 pipeline. +* **Compressed by 1.6.x or earlier:** decompressed by a separate pipeline that reproduces the MGARD 1.6.0 decompressor, if MGARD-X is built with ```-DMGARD_ENABLE_DATA_COMPATIBILITY=ON```. The result is the same as decompressing with MGARD 1.6.0 (bit-identical on CPUs). Without this option, decompression returns ```mgard_x::compress_status_type::DataCompatibilityNotEnabledFailure```. + +This is transparent to users: the same CLI and high-level APIs decompress both formats, and decompression needs no extra configuration. Compression always produces the 1.7.0 format. Data compressed by MGARD 1.6.x with the variable-size domain decomposition (```domain_decomposition_type::Variable```) is not supported, because its block sizes are not recorded in the header. ## Control Errors MGARD can bound errors in various types of norm: @@ -101,23 +110,19 @@ MGARD can bound error `tol` in two ways (using L∞ norm as an ex ## Using command line interface (CLI) An executable ```mgard-x``` will be built after building the MGARD-X library. To use the ```mgard-x``` CLI, here are the options: -+ ```-z```: enable compression mode - + ```-i ``` path to data file to be compressed - + ```-c ``` path to compressed file - + ```-t ``` data type (s: single | d:double) - + ```-n ``` total number of dimensions - + `````` slowest dimention - + `````` 2nd slowest dimention - + ... - + `````` fastest dimention - + ```-u ``` path to coordinate file (non-uniform only) - + ```-m ``` error bound mode (abs: abolute | rel: relative) - + ```-e ``` error bound - + ```-s ``` smoothness parameter - + ```-r <0|1>``` internal data layout (0: Higher throughput | 1: Higher compression ratio) - + ```-dd ``` domain decomposition mode (auto: decompose only when the data does not fit the memory limit (default) | manual: use the block size given by ```-dd-size```) - + ```-dd-size ... ``` domain decomposition block size, slowest dimension first (manual mode only) - + ```-f ``` maximum memory footprint in bytes (if not specify: no limit) ++ ```-z / --compress```: enable compression mode + + ```-i / --input ``` path to data file to be compressed + + ```-o / --output ``` path to compressed file + + ```-dt / --data-type ``` data type (s: single | d: double) + + ```-dim / --dimension ... ``` total number of dimensions followed by the size of each dimension, from slowest (```n_1```) to fastest (```n_D```) + + ```-em / --error-bound-mode ``` error bound mode (abs: absolute | rel: relative) + + ```-e / --error-bound ``` error bound + + ```-s / --smoothness ``` smoothness parameter + + ```-m / --max-memory ``` maximum memory footprint in bytes (default: no limit) + + ```-dd / --domain-decomposition ``` domain decomposition mode (see ```domain_decomposition_mode``` below): + + ```auto```: decompose only when the data does not fit the memory limit (default) + + ```manual```: decompose into blocks of the size given by ```-dd-size``` + + ```-dd-size / --domain-decomposition-size ... ``` block size for ```-dd manual```: number of dimensions followed by the block size of each dimension, from slowest to fastest. For example, ```-dim 3 400 300 200 -dd manual -dd-size 3 100 300 200``` splits the data into four 100x300x200 blocks along the slowest dimension. + ```-l / --lossless ``` choose lossless compressor: + ```huffman```: Huffman coding + ```huffman-lz4```: Huffman coding followed by LZ4 @@ -130,18 +135,22 @@ An executable ```mgard-x``` will be built after building the MGARD-X library. To + ```symbol-rans```: symbol-alphabet rANS + ```-hh / --hybrid``` use the hybrid hierarchy (1D-3D only) + ```-ll / --local-levels ``` number of hybrid block-local refactoring levels (default: 1) - + ```-gl / --global-levels ``` number of global refactoring levels, multi-dim or the hybrid global stage (default: 3) - + ```-hp / --hybrid-projection ``` select the projection mode used by the hybrid hierarchy (default: orthogonal) - + ```orthogonal```: retain the L2 projection (default) + + ```-gl / --global-levels ``` maximum number of global refactoring levels, of the multi-dimensional hierarchy or of the hybrid global stage (default: 3) + + ```-pm / --projection-mode ``` select the projection mode (default: auto) + + ```auto```: hierarchical under an L∞ error bound, orthogonal otherwise + + ```orthogonal```: retain the L2 projection + ```hierarchical```: remove the L2 projection (L∞ only) + + ```-r / --roi-tolerance-map ``` and ```-roi / --enable-roi``` region-of-interest mode with per-block tolerances (see [MGARD-RoI](MGARD-RoI.md)) + ```-nkf / --no-kernel-fusion``` disable fused local decomposition and quantization kernels for the hybrid hierarchy -+ ```-x```: enable decompression mode - + ```-c ``` path to compressed file - + ```-o ``` path to decompressed file -+ ```-d ``` choose processor (auto: Auto select | serial: CPU | openmp: multi-threaded CPU | cuda: NVIDIA GPU | hip: AMD GPU | sycl: Intel GPU ) -+ ```-v``` enable verbose (0: error only (default) | 1: error + infomation | 2: error + timing | 3: all) -+ ```-g ``` number of devices (GPUs) to use (default: 1) -+ ```-h <0|1>``` enable/disable prefecth pipeline optimization (0: disable | 1: enable (default)) + + ```-w / --warm-up``` run a small throwaway compression first to exclude one-time GPU kernel loading from the timing ++ ```-x / --decompress```: enable decompression mode + + ```-i / --input ``` path to compressed file + + ```-o / --output ``` path to decompressed file + + ```-orig / --original-data ``` (optional) original data, for reporting the decompression error + + ```-em / --error-bound-mode ``` (optional) error mode used for that report (default: rel) + + ```-nkf / --no-kernel-fusion``` disable fused dequantization and recomposition kernels for the hybrid hierarchy ++ ```-d / --device ``` choose processor (auto: Auto select | serial: CPU | openmp: multi-threaded CPU | cuda: NVIDIA GPU | hip: AMD GPU | sycl: Intel GPU ) ++ ```-v / --verbose <0|1|2|3>``` verbosity (0: error only (default) | 1: error + information | 2: error + timing | 3: all) ## For using both the high-level APIs and low-level API @@ -159,34 +168,34 @@ An executable ```mgard-x``` will be built after building the MGARD-X library. To + ```mgard_x::device_type::SYCL ```: Use Intel GPUs + ```dev_id```: sepcifying a specific GPU to use in multi-GPU systems (***Default: 0***) - + ```num_dev```: sepcifying the number of GPU to use in multi-GPU systems (***Default: 1***) + ```lossless```: control the lossless compression used: + ```mgard_x::lossless_type::Huffman```: Huffman compression (***Default***) + ```mgard_x::lossless_type::Huffman_LZ4```: Huffman and LZ4 compression + ```mgard_x::lossless_type::Huffman_Zstd```: Huffman and ZSTD compression + *Note:* there will be no effect configuring the lossless comrpessor for decompression as MGARD has to use the same lossless compressor that was used for compression. + ```huff_dict_size```: Huffman dictionary size (***Default: 8192***) - + ```huff_block_size```: Huffman parallelized block size (***Default: 20480***) + + ```huff_block_size```: Huffman parallelized block size (***Default: 1024***) + ```lz4_block_size```: LZ4 parallelized block size (***Default: 32768***) + ```zstd_compress_level```: Zstd compression level (***Default: 3***) + ```normalize_coordinates```: enable or disable coordinate normalization (***Controlled by CMake option***) - + ```domain_decomposition_mode``` (for high-level APIs only): controls how data is split into blocks (subdomains) that are processed one at a time. Blocks may have any shape, e.g. a 400x300x200 array with 100x300x200 blocks is split along the slowest dimension only. Along each dimension a remainder narrower than 3 nodes is merged into the block before it. The block shape is recorded in the compressed header, so decompression needs no setting. - + ```mgard_x::domain_decomposition_mode_type::Auto```: decompose only when the data does not fit the memory limit (device memory capped by ```max_memory_footprint```); the largest block extent is then halved until the largest block fits (***Default***) - + ```mgard_x::domain_decomposition_mode_type::Manual```: use ```domain_decomposition_block_size```; compression fails if any block extent is smaller than 3 or a block would not fit the memory limit - + ```domain_decomposition_block_size```: block shape for the manual mode, one extent per dimension, slowest first; extents larger than the data are clamped to it + + ```domain_decomposition_mode``` (for high-level APIs and MDR-X): controls how the data is split into blocks (subdomains) that are compressed one at a time, which lets MGARD-X compress data larger than the device memory and overlap data transfer with computation. All blocks share one shape, which may differ per dimension: for a 400x300x200 array, 100x300x200 blocks split it along the slowest dimension only, and 128x128x128 blocks split every dimension. Along each dimension, a leftover piece narrower than 3 nodes is merged into the last block. The block shape is recorded in the compressed header, so decompression needs no setting. + + ```mgard_x::domain_decomposition_mode_type::Auto```: decompose only when the data does not fit the memory limit (the free device memory, capped by ```max_memory_footprint```). The block size is then chosen automatically: the largest block dimension is halved until the largest block fits. Compression fails if even the smallest blocks (3 nodes per dimension) do not fit (***Default***) + + ```mgard_x::domain_decomposition_mode_type::Manual```: use the block shape in ```domain_decomposition_block_size```. Compression fails with an error if the number of block sizes does not match the number of dimensions, any block size is smaller than 3, or the largest block does not fit the memory limit + + ```domain_decomposition_block_size```: block shape for the manual mode, one size per dimension, from slowest to fastest; sizes larger than the data are clamped to the data size + ```decomposition```: controls multi-level decomposition: + ```mgard_x::decomposition_type::MultiDim```: N-D decomposition (***Default***) + ```mgard_x::decomposition_type::SingleDim```: 1D-at-a-time decomposition + ```mgard_x::decomposition_type::Hybrid```: hybrid hierarchy with block-local and global decomposition (1D-3D only; high-level APIs only) + ```num_local_refactoring_level```: controls the number of block-local refactoring levels in the hybrid hierarchy (***Default: 1***) + ```num_global_refactoring_level```: controls the max number of global refactoring levels: of the multi-dimensional hierarchy, and of the global stage of the hybrid hierarchy. Recorded in the compressed header, so decompression needs no setting (***Default: 3***) - + ```hybrid_projection_mode```: controls the projection mode used by the hybrid hierarchy: - + ```mgard_x::hybrid_projection_mode_type::Orthogonal```: retain the L2 projection (***Default***) - + ```mgard_x::hybrid_projection_mode_type::Hierarchical```: remove the L2 projection (L∞ only) + + ```projection_mode```: controls the projection mode: + + ```mgard_x::compression_projection_mode_type::Auto```: hierarchical under an L∞ error bound, orthogonal otherwise (***Default***) + + ```mgard_x::compression_projection_mode_type::Orthogonal```: retain the L2 projection + + ```mgard_x::compression_projection_mode_type::Hierarchical```: remove the L2 projection (L∞ only) + ```fuse_decompose_quantize```: controls fused local decomposition and quantization for the hybrid hierarchy (***Default: true***) + ```fuse_dequantize_recompose```: controls fused local dequantization and recomposition for the hybrid hierarchy (***Default: true***) + ```max_memory_footprint```(for high-level APIs only): controls maximum memory footprint in bytes (***Default: inf (no limit)***) - + ```adjust_shape```(for high-level APIs only): controls whether or not to enable shape adjustment for better decomposition performance (***Default: false***) + + ```adjust_shape```(for high-level APIs only): controls whether or not to enable shape adjustment for better decomposition performance; cannot be combined with ```domain_decomposition_mode_type::Manual``` (***Default: false***) ## Using high-level APIs * **For compression:** ```enum mgard_x::compress_status_type mgard_x::compress(mgard_x::DIM D, mgard_x::data_type dtype, std::vector shape, double tol, double s, enum error_bound_type mode, const void *original_data, void *&compressed_data, size_t &compressed_size, mgard_x::Config config)``` + ```[In] shape:``` Shape of the Dataset to be compressed (from slowest to fastest). @@ -221,6 +230,7 @@ An executable ```mgard-x``` will be built after building the MGARD-X library. To + mgard_x::compress_status_type::NotSupportHigherNumberOfDimensionsFailure + mgard_x::compress_status_type::NotSupportDataTypeFailure + mgard_x::compress_status_type::BackendNotAvailableFailure + + mgard_x::compress_status_type::DataCompatibilityNotEnabledFailure: the data was compressed by MGARD 1.6.x or earlier and MGARD-X was built without ```MGARD_ENABLE_DATA_COMPATIBILITY``` (see [Decompressing data from earlier versions](#decompressing-data-from-earlier-versions)) ## Using low-level APIs * **Step 1: Build Hierarchy** @@ -293,7 +303,7 @@ For achieving the best performance: + **Tune for a specific backend:** run ```mgard-x-autotuner -d ``` + **Tune for a specific shape of data on a specific backend :** run ```mgard-x-autotuner -d -n [dim1] [dim2] ... [dimN]```. + ***Note:*** MGARD-X needs to be recompiled after auto tuning to make it effective. -* **Use large data**: Using larger input data will help MGARD-X achieve more efficient compression and decompression pipeline. The high-level API allows users to provide data larger than the available memory size on the GPU. In this case, MGARD-X can achieve much better performance with pipeline optimization (always enabled). +* **Use large data**: Using larger input data will help MGARD-X achieve more efficient compression and decompression pipeline. The high-level API allows users to provide data larger than the available memory size on the GPU: the data is then split into blocks (see ```domain_decomposition_mode```) whose transfers overlap with computation (always enabled). With the manual mode, blocks that keep the fastest dimensions whole (e.g. 100x300x200 for 400x300x200 data) are copied to and from the device in fewer, larger transfers. * **Use decompsition operation**: MGARD-X can be configured to reornagize the input data to imporve the performance of its multilevel decompsition process. To enable such optimization, set ```Config.adjust_shape = true```. ## Performance diff --git a/doc/release notes/1.7.0.md b/doc/release notes/1.7.0.md index 9f62f6b3a..6cc1cd7f2 100644 --- a/doc/release notes/1.7.0.md +++ b/doc/release notes/1.7.0.md @@ -9,6 +9,10 @@ * A BlockDelta (FLE-based) lossless backend for mgard-x, as a Huffman alternative. * A portable, warp-cooperative LZ4 implementation for mgard-x, removing the nvcomp dependency. * Blackwell (sm_120) GPU build support. +* Decompression of data compressed by MGARD 1.6.x and earlier, through a separate MGARD 1.6.0-compatible decompression pipeline enabled with the new CMake option `MGARD_ENABLE_DATA_COMPATIBILITY` (default OFF). Decompression detects the version from the header and routes 1.6.x data automatically; output matches MGARD 1.6.0 (bit-identical on CPUs). Builds without the option report such data with the new status `DataCompatibilityNotEnabledFailure`. +* Generic block domain decomposition with two modes (`Config::domain_decomposition_mode`): `Auto` (default) decomposes only when the data does not fit the memory limit and picks the block size, and `Manual` uses a user-given block shape (`Config::domain_decomposition_block_size`), which may differ per dimension. Available in the `mgard-x` and `mdr-x` CLIs as `-dd auto|manual` and `-dd-size`. +* `MemoryManager::Copy3D` in RuntimeX (native 3D copies on CUDA and HIP), used for copying blocks between host and device in fewer transfers. +* Opt-in ZSTD compression of MDR-X bitplane groups (`lossless = Huffman_Zstd`; `mdr-x -l huffman-zstd`). * Additional unit tests: bitplane encoders, DirectInterleaver, MGARDDecomposer, HybridLevelCompressor, and ComposedRefactor/Reconstructor; unit tests are now discoverable by CTest. ## Changed @@ -18,6 +22,12 @@ * Optimized the Huffman compress/decompress pipeline (roughly 1.9x faster compression). * Bumped the bundled ZSTD to v1.5.6 and protobuf to v3.21.12; stopped building nvcomp. * Raised the minimum ROCm requirement to 7.0 for HIP builds. +* ***Breaking (API):*** replaced the MaxDim/Block/Variable domain decomposition types with the generic block decomposition. `Config::domain_decomposition`, `block_size`, `domain_decomposition_dim` and `domain_decomposition_sizes` are removed in favor of `domain_decomposition_mode` and `domain_decomposition_block_size`; variable-size decomposition is no longer supported. The CLIs' `-dd max-dim|block` became `-dd auto|manual`. Compressed data with the previous MaxDim and Block decomposition headers still decompresses. +* ***Breaking (API):*** merged `max_larget_level` into `num_global_refactoring_level`, which now caps the levels of every multilevel hierarchy (multi-dimensional, MDR-X, and the hybrid global stage) and is recorded in the compressed header. Defaults are `num_local_refactoring_level = 1` and `num_global_refactoring_level = 3` (previously all levels for multi-dimensional compression). This changes the default compression ratio (e.g. relative 1e-3, L-infinity: 1D 2.89 -> 3.44, 3D 2.57 -> 2.85; L2 in 1D: 4.08 -> 3.65); error bounds are unchanged. +* Renamed `MemoryManager::CopyND` to `Copy2D` in RuntimeX. +* Reduced mgard-x memory usage: hierarchy level volumes are stored per level instead of in a dense array, hierarchies are moved rather than copied into the compressor and refactor caches, and the hybrid hierarchy builds only the global hierarchy it needs (new `HybridHierarchy`). Peak device memory for a 2^24-element 1D float input drops from 7.7 GB to 1.8 GB (multi-dimensional) and 2.1 GB (hybrid). +* MPI is now optional for `MGARD_ENABLE_MDR`; only the `pmdr-x-qoi` and `pmdr-x-qoi-io` executables require it. +* The MDR-X QoI executables always use one subdomain per variable and no longer take `-dd`. * Updated documentation: the MGARD-X guide for the hybrid hierarchy pipeline, a hybrid RoI user guide, the MGARD family tree, performance figures, and the publications list. ## Fixed @@ -30,3 +40,14 @@ * Fixed a floating-point exception in Zstd::Decompress. * Fixed CPUPipelines so OpenMP-only builds compile again. * Fixed clang-format violations flagged by CI. +* Fixed decompression of domain-decomposed data in which a subdomain stored uncompressed has a size that is not a multiple of 8 bytes: the following subdomains were read at the wrong offset. +* Fixed an MDR-X sign-buffer overflow in bitplane decoding during reconstruction (heap corruption on CPUs, out-of-bounds writes on GPUs) when a later subdomain needed a larger buffer. +* Fixed MDR-X reconstruction writing back the last subdomain from the wrong buffer when the number of subdomains is a multiple of 3. +* Fixed MDR-X refactoring/reconstruction sizing reused buffers from the first subdomain instead of the largest. +* Fixed MaxDim domain decomposition along a dimension other than the slowest, which copied past the end of the data. +* Fixed Zstd decompression truncating the stored size to 32 bits and ignoring decompression errors. +* Fixed the compression output buffer overflowing for non-uniform grids, whose coordinates are stored in the header. +* Fixed the `mgard-x` CLI ignoring decompression failures. +* Fixed MDR-X requests fetching a full bitplane group for levels that need none, and stale bitplane counts when a reconstructor is reused. +* Fixed the Huffman compression-ratio estimate pairing frequencies with the wrong code lengths, which made MDR-X store compressible bitplane groups uncompressed. +* Fixed CPU (MGARD) Huffman truncation of wide outliers. From 4db27230884c8d621608367fd338834832c368ae Mon Sep 17 00:00:00 2001 From: Jieyang Chen Date: Sun, 11 Oct 2026 11:53:56 -0700 Subject: [PATCH 10/11] mgard-x: make the CLIs' -d sycl select the SYCL backend get_device_type mapped "sycl" to device_type::HIP (copied from the hip case), so -d sycl failed with a missing-backend error in SYCL-only builds and ran on the AMD GPU in builds with both HIP and SYCL. Affects mgard-x, mdr-x, mdr-x-qoi, pmdr-x-qoi and pmdr-x-qoi-io. Co-Authored-By: Claude Opus 5.5 --- src/mgard-x/Executables/ArgumentParser.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mgard-x/Executables/ArgumentParser.h b/src/mgard-x/Executables/ArgumentParser.h index 194452f1d..ec071f2a9 100644 --- a/src/mgard-x/Executables/ArgumentParser.h +++ b/src/mgard-x/Executables/ArgumentParser.h @@ -156,7 +156,7 @@ enum mgard_x::device_type get_device_type(int argc, char *argv[]) { } else if (dev.compare("hip") == 0) { dev_type = mgard_x::device_type::HIP; } else if (dev.compare("sycl") == 0) { - dev_type = mgard_x::device_type::HIP; + dev_type = mgard_x::device_type::SYCL; } else { throw std::runtime_error("wrong device type."); } From 8d522735163e7b68ce4a8b0d89bdd4587170aed6 Mon Sep 17 00:00:00 2001 From: Jieyang Chen Date: Mon, 12 Oct 2026 20:19:59 -0700 Subject: [PATCH 11/11] Apply clang-format (14, as in CI) to files that failed the format check Formatting only: line wrapping, comment reflow, and one long string literal split into adjacent literals. Covers the mgard-x-160 compatibility tree, the domain decomposer, Metadata, the device adapters, the executables, and Hierarchy/HybridHierarchy. Co-Authored-By: Claude Opus 5.5 --- include/mgard-x-160/Hierarchy/Hierarchy.h | 3 +- include/mgard-x-160/Hierarchy/Hierarchy.hpp | 12 +-- .../mgard-x-160/Hierarchy/HybridHierarchy.h | 4 +- include/mgard-x-160/Lossless/CPU.hpp | 6 +- .../mgard-x-160/Lossless/LZ4/LZ4Kernels.hpp | 17 ++-- include/mgard-x-160/Lossless/Zstd.hpp | 6 +- .../CompressionHighLevel.hpp | 26 +++--- .../CompressionHighLevel/DataCompatibility.h | 4 +- .../DomainDecomposer/DomainDecomposer.hpp | 28 +++--- include/mgard-x/Hierarchy/Hierarchy.h | 3 +- include/mgard-x/Hierarchy/HybridHierarchy.h | 4 +- include/mgard-x/MDRHighLevel/MDRHighLevel.hpp | 11 +-- include/mgard-x/Metadata/Metadata.hpp | 18 ++-- .../DeviceAdapters/DeviceAdapterCuda.h | 10 +- .../DeviceAdapters/DeviceAdapterHip.h | 10 +- .../DeviceAdapters/DeviceAdapterKokkos.h | 7 +- .../DeviceAdapters/DeviceAdapterOpenmp.h | 4 +- .../DeviceAdapters/DeviceAdapterSerial.h | 4 +- .../DeviceAdapters/DeviceAdapterSycl.h | 4 +- src/mgard-x-160/Metadata/Metadata.cpp | 16 ++-- .../CompressionHighLevel/DynamicAPI.cpp | 92 +++++++++++-------- src/mgard-x/Executables/ArgumentParser.h | 9 +- src/mgard-x/Executables/mdr-x-qoi.cpp | 3 +- src/mgard-x/Executables/mdr-x.cpp | 13 +-- src/mgard-x/Executables/mgard-x.cpp | 19 ++-- src/mgard-x/Executables/pmdr-x-qoi-io.cpp | 8 +- src/mgard-x/Executables/pmdr-x-qoi.cpp | 8 +- src/mgard-x/Metadata/Metadata.cpp | 10 +- 28 files changed, 182 insertions(+), 177 deletions(-) diff --git a/include/mgard-x-160/Hierarchy/Hierarchy.h b/include/mgard-x-160/Hierarchy/Hierarchy.h index af8dfea61..4f8c54d06 100644 --- a/include/mgard-x-160/Hierarchy/Hierarchy.h +++ b/include/mgard-x-160/Hierarchy/Hierarchy.h @@ -55,8 +55,7 @@ template struct Hierarchy { // counts. bool matches_config(Config config) { return true; } size_t EstimateMemoryFootprint(std::vector shape); - static size_t EstimateMemoryFootprint(std::vector shape, - Config config); + static size_t EstimateMemoryFootprint(std::vector shape, Config config); ~Hierarchy(); diff --git a/include/mgard-x-160/Hierarchy/Hierarchy.hpp b/include/mgard-x-160/Hierarchy/Hierarchy.hpp index dfbab7b58..1efd5de44 100644 --- a/include/mgard-x-160/Hierarchy/Hierarchy.hpp +++ b/include/mgard-x-160/Hierarchy/Hierarchy.hpp @@ -766,8 +766,8 @@ void Hierarchy::init_shape_only(std::vector shape, data_structure_type dstype) { int ret = check_shape(shape); if (ret == -1) { - throw std::runtime_error( - "Number of dimensions mismatch. mgard_x_160::Hierarchy not initialized!"); + throw std::runtime_error("Number of dimensions mismatch. " + "mgard_x_160::Hierarchy not initialized!"); } if (ret == -2) { throw std::runtime_error("Size of any dimension cannot be smaller than 3. " @@ -793,8 +793,8 @@ template Hierarchy::Hierarchy(std::vector shape, Config config) { int ret = check_shape(shape); if (ret == -1) { - throw std::runtime_error( - "Number of dimensions mismatch. mgard_x_160::Hierarchy not initialized!"); + throw std::runtime_error("Number of dimensions mismatch. " + "mgard_x_160::Hierarchy not initialized!"); } if (ret == -2) { throw std::runtime_error("Size of any dimension cannot be smaller than 3. " @@ -819,8 +819,8 @@ Hierarchy::Hierarchy(std::vector shape, std::vector coords, Config config) { int ret = check_shape(shape); if (ret == -1) { - throw std::runtime_error( - "Number of dimensions mismatch. mgard_x_160::Hierarchy not initialized!"); + throw std::runtime_error("Number of dimensions mismatch. " + "mgard_x_160::Hierarchy not initialized!"); } if (ret == -2) { throw std::runtime_error("Size of any dimension cannot be smaller than 3. " diff --git a/include/mgard-x-160/Hierarchy/HybridHierarchy.h b/include/mgard-x-160/Hierarchy/HybridHierarchy.h index 1495b2f14..a5982b11d 100644 --- a/include/mgard-x-160/Hierarchy/HybridHierarchy.h +++ b/include/mgard-x-160/Hierarchy/HybridHierarchy.h @@ -86,8 +86,8 @@ class HybridHierarchy : public Hierarchy { void check_config(Config config) { if (!matches_config(config)) { throw ProcessingException( - "HybridHierarchy was built for " + std::to_string(L) + - " local / " + std::to_string(M) + " global levels, but config has " + + "HybridHierarchy was built for " + std::to_string(L) + " local / " + + std::to_string(M) + " global levels, but config has " + std::to_string(config.num_local_refactoring_level) + " / " + std::to_string(config.num_global_refactoring_level) + "."); } diff --git a/include/mgard-x-160/Lossless/CPU.hpp b/include/mgard-x-160/Lossless/CPU.hpp index 06696eeee..11385f03d 100644 --- a/include/mgard-x-160/Lossless/CPU.hpp +++ b/include/mgard-x-160/Lossless/CPU.hpp @@ -18,9 +18,9 @@ char mgard_x_check_msg[256]; \ std::snprintf(mgard_x_check_msg, sizeof(mgard_x_check_msg), \ "CHECK(" #cond ") failed: " __VA_ARGS__); \ - throw mgard_x_160::ProcessingException(std::string(__FILE__) + ":" + \ - std::to_string(__LINE__) + " " + \ - mgard_x_check_msg); \ + throw mgard_x_160::ProcessingException(std::string(__FILE__) + ":" + \ + std::to_string(__LINE__) + " " + \ + mgard_x_check_msg); \ } \ } while (0) diff --git a/include/mgard-x-160/Lossless/LZ4/LZ4Kernels.hpp b/include/mgard-x-160/Lossless/LZ4/LZ4Kernels.hpp index db2049782..b33fe6ee8 100644 --- a/include/mgard-x-160/Lossless/LZ4/LZ4Kernels.hpp +++ b/include/mgard-x-160/Lossless/LZ4/LZ4Kernels.hpp @@ -77,14 +77,15 @@ MGARDX_EXEC int write_len(Byte *out, int op, int length) { // --------------------------------------------------------------------------- // ONE cooperative LZ4 codec, written against the portable SubGroup abstraction -// (mgard_x_160::SubGroup / SubGroupScalar, defined in the DeviceAdapter -// headers). A "sub-group" is a set of lanes running in lockstep that can shfl / -// ballot / ffs / sync. At size 1 (CPU) every collective is an identity and this -// exact code degenerates into the greedy serial encoder/decoder. Warp-size is -// handled via SG::size()/full_mask()/the mask typedef, so it is correct for any -// width -- the match scan examines size() positions per step (a wider sub-group -// gives slightly different, still-valid matches => CR can vary marginally with -// width; decode output is width-independent). +// (mgard_x_160::SubGroup / SubGroupScalar, defined in the +// DeviceAdapter headers). A "sub-group" is a set of lanes running in lockstep +// that can shfl / ballot / ffs / sync. At size 1 (CPU) every collective is an +// identity and this exact code degenerates into the greedy serial +// encoder/decoder. Warp-size is handled via SG::size()/full_mask()/the mask +// typedef, so it is correct for any width -- the match scan examines size() +// positions per step (a wider sub-group gives slightly different, still-valid +// matches => CR can vary marginally with width; decode output is +// width-independent). // --------------------------------------------------------------------------- namespace lz4_coop { diff --git a/include/mgard-x-160/Lossless/Zstd.hpp b/include/mgard-x-160/Lossless/Zstd.hpp index f0dd4ec74..7570e48a0 100644 --- a/include/mgard-x-160/Lossless/Zstd.hpp +++ b/include/mgard-x-160/Lossless/Zstd.hpp @@ -15,9 +15,9 @@ namespace mgard_x_160 { char mgard_x_check_msg[256]; \ std::snprintf(mgard_x_check_msg, sizeof(mgard_x_check_msg), \ "CHECK(" #cond ") failed: " __VA_ARGS__); \ - throw mgard_x_160::ProcessingException(std::string(__FILE__) + ":" + \ - std::to_string(__LINE__) + " " + \ - mgard_x_check_msg); \ + throw mgard_x_160::ProcessingException(std::string(__FILE__) + ":" + \ + std::to_string(__LINE__) + " " + \ + mgard_x_check_msg); \ } \ } while (0) diff --git a/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp b/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp index 75c5fd435..06a9247ca 100644 --- a/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp +++ b/include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp @@ -296,14 +296,13 @@ general_compress_pipeline(std::vector shape, T tol, T s, m.FillForCompression( ebtype, tol, s, norm, config.decomposition, config.lossless, config.huff_dict_size, config.huff_block_size, shape, - domain_decomposer.domain_decomposed(), - domain_decomposer.block_shape()); + domain_decomposer.domain_decomposed(), domain_decomposer.block_shape()); } else { - m.FillForCompression( - ebtype, tol, s, norm, config.decomposition, config.lossless, - config.huff_dict_size, config.huff_block_size, shape, - domain_decomposer.domain_decomposed(), - domain_decomposer.block_shape(), coords); + m.FillForCompression(ebtype, tol, s, norm, config.decomposition, + config.lossless, config.huff_dict_size, + config.huff_block_size, shape, + domain_decomposer.domain_decomposed(), + domain_decomposer.block_shape(), coords); } uint32_t metadata_size; @@ -354,14 +353,13 @@ general_compress_pipeline(std::vector shape, T tol, T s, m.FillForCompression( ebtype, tol, s, norm, config.decomposition, config.lossless, config.huff_dict_size, config.huff_block_size, shape, - domain_decomposer.domain_decomposed(), - domain_decomposer.block_shape()); + domain_decomposer.domain_decomposed(), domain_decomposer.block_shape()); } else { - m.FillForCompression( - ebtype, tol, s, norm, config.decomposition, config.lossless, - config.huff_dict_size, config.huff_block_size, shape, - domain_decomposer.domain_decomposed(), - domain_decomposer.block_shape(), coords); + m.FillForCompression(ebtype, tol, s, norm, config.decomposition, + config.lossless, config.huff_dict_size, + config.huff_block_size, shape, + domain_decomposer.domain_decomposed(), + domain_decomposer.block_shape(), coords); } serizalied_meta = m.Serialize(metadata_size); diff --git a/include/mgard-x/CompressionHighLevel/DataCompatibility.h b/include/mgard-x/CompressionHighLevel/DataCompatibility.h index d08d3fac7..bddb2836f 100644 --- a/include/mgard-x/CompressionHighLevel/DataCompatibility.h +++ b/include/mgard-x/CompressionHighLevel/DataCompatibility.h @@ -35,8 +35,8 @@ decompress(const void *compressed_data, size_t compressed_size, template enum compress_status_type decompress(const void *compressed_data, size_t compressed_size, - void *&decompressed_data, data_type &dtype, - std::vector &shape, Config config, bool output_pre_allocated); + void *&decompressed_data, data_type &dtype, std::vector &shape, + Config config, bool output_pre_allocated); template enum compress_status_type release_cache(); diff --git a/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp b/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp index 93ee29903..0a0e28f97 100644 --- a/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp +++ b/include/mgard-x/DomainDecomposer/DomainDecomposer.hpp @@ -402,8 +402,8 @@ class DomainDecomposer { std::vector largest_block_shape(std::vector block) { std::vector largest(D); for (DIM d = 0; d < D; d++) { - largest[d] = block_extent(shape[d], block[d], - num_blocks(shape[d], block[d]) - 1); + largest[d] = + block_extent(shape[d], block[d], num_blocks(shape[d], block[d]) - 1); } return largest; } @@ -457,13 +457,13 @@ class DomainDecomposer { } if (!fits_memory(block)) { std::vector largest = largest_block_shape(block); - throw Exception( - "Manual domain decomposition: blocks of " + shape_string(block) + - " need about " + - std::to_string( - EstimateMemoryFootprint(largest, count_blocks(block) > 1)) + - " bytes (largest block " + shape_string(largest) + "), more than " + - "the " + std::to_string(available_memory()) + " bytes available."); + throw Exception("Manual domain decomposition: blocks of " + + shape_string(block) + " need about " + + std::to_string(EstimateMemoryFootprint( + largest, count_blocks(block) > 1)) + + " bytes (largest block " + shape_string(largest) + + "), more than " + "the " + + std::to_string(available_memory()) + " bytes available."); } return block; } @@ -476,11 +476,11 @@ class DomainDecomposer { _block_shape[d] = std::min(block[d], shape[d]); // A dimension narrower than MIN_BLOCK_EXTENT can only be kept whole. if (_block_shape[d] < std::min(MIN_BLOCK_EXTENT, shape[d])) { - throw InvalidDataException( - "domain decomposition block extent " + - std::to_string(_block_shape[d]) + " along dimension " + - std::to_string(d) + " is smaller than " + - std::to_string(MIN_BLOCK_EXTENT) + "."); + throw InvalidDataException("domain decomposition block extent " + + std::to_string(_block_shape[d]) + + " along dimension " + std::to_string(d) + + " is smaller than " + + std::to_string(MIN_BLOCK_EXTENT) + "."); } _blocks_per_dim[d] = num_blocks(shape[d], _block_shape[d]); _num_subdomains *= _blocks_per_dim[d]; diff --git a/include/mgard-x/Hierarchy/Hierarchy.h b/include/mgard-x/Hierarchy/Hierarchy.h index 6cf480384..a4d2c2109 100644 --- a/include/mgard-x/Hierarchy/Hierarchy.h +++ b/include/mgard-x/Hierarchy/Hierarchy.h @@ -55,8 +55,7 @@ template struct Hierarchy { // counts. bool matches_config(Config config) { return true; } size_t EstimateMemoryFootprint(std::vector shape); - static size_t EstimateMemoryFootprint(std::vector shape, - Config config); + static size_t EstimateMemoryFootprint(std::vector shape, Config config); ~Hierarchy(); diff --git a/include/mgard-x/Hierarchy/HybridHierarchy.h b/include/mgard-x/Hierarchy/HybridHierarchy.h index 7447331d1..d94357039 100644 --- a/include/mgard-x/Hierarchy/HybridHierarchy.h +++ b/include/mgard-x/Hierarchy/HybridHierarchy.h @@ -86,8 +86,8 @@ class HybridHierarchy : public Hierarchy { void check_config(Config config) { if (!matches_config(config)) { throw ProcessingException( - "HybridHierarchy was built for " + std::to_string(L) + - " local / " + std::to_string(M) + " global levels, but config has " + + "HybridHierarchy was built for " + std::to_string(L) + " local / " + + std::to_string(M) + " global levels, but config has " + std::to_string(config.num_local_refactoring_level) + " / " + std::to_string(config.num_global_refactoring_level) + "."); } diff --git a/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp b/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp index c5aa6c31a..4d790fa6d 100644 --- a/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp +++ b/include/mgard-x/MDRHighLevel/MDRHighLevel.hpp @@ -142,16 +142,15 @@ void MDRefactor(std::vector shape, const void *original_data, timer_each.start(); Metadata m; if (uniform) { - m.FillForMDR( - (T)0.0, config.decomposition, config.lossless, config.huff_dict_size, - config.huff_block_size, shape, domain_decomposer.domain_decomposed(), - domain_decomposer.block_shape(), config.total_num_bitplanes); + m.FillForMDR((T)0.0, config.decomposition, config.lossless, + config.huff_dict_size, config.huff_block_size, shape, + domain_decomposer.domain_decomposed(), + domain_decomposer.block_shape(), config.total_num_bitplanes); } else { m.FillForMDR( (T)0.0, config.decomposition, config.lossless, config.huff_dict_size, config.huff_block_size, shape, domain_decomposer.domain_decomposed(), - domain_decomposer.block_shape(), config.total_num_bitplanes, - coords); + domain_decomposer.block_shape(), config.total_num_bitplanes, coords); } // The reconstructor rebuilds the hierarchy, so it needs the level cap. diff --git a/include/mgard-x/Metadata/Metadata.hpp b/include/mgard-x/Metadata/Metadata.hpp index 6941ba426..6b8555b59 100644 --- a/include/mgard-x/Metadata/Metadata.hpp +++ b/include/mgard-x/Metadata/Metadata.hpp @@ -152,14 +152,13 @@ template struct Metadata : MetadataBase { } template - void - FillForCompression(enum error_bound_type ebtype, T tol, T s, T norm, - enum decomposition_type decomposition, - enum lossless_type ltype, uint32_t huff_dict_size, - uint32_t huff_block_size, std::vector shape, - bool domain_decomposed, - std::vector domain_decomposition_block_shape, - std::vector coords) { + void FillForCompression(enum error_bound_type ebtype, T tol, T s, T norm, + enum decomposition_type decomposition, + enum lossless_type ltype, uint32_t huff_dict_size, + uint32_t huff_block_size, std::vector shape, + bool domain_decomposed, + std::vector domain_decomposition_block_shape, + std::vector coords) { FillForCompression(ebtype, tol, s, norm, decomposition, ltype, huff_dict_size, huff_block_size, shape, domain_decomposed, domain_decomposition_block_shape); @@ -222,8 +221,7 @@ template struct Metadata : MetadataBase { uint32_t huff_block_size, std::vector shape, bool domain_decomposed, std::vector domain_decomposition_block_shape, - uint64_t number_bitplanes, - std::vector coords) { + uint64_t number_bitplanes, std::vector coords) { FillForMDR(norm, decomposition, ltype, huff_dict_size, huff_block_size, shape, domain_decomposed, domain_decomposition_block_shape, number_bitplanes); diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h index e054fb9ea..6b1ee64ec 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterCuda.h @@ -1030,10 +1030,12 @@ template <> class MemoryManager { } cudaStream_t stream = DeviceRuntime::GetQueue(queue_idx); cudaMemcpy3DParms p = {}; - p.srcPtr = make_cudaPitchedPtr((void *)src_ptr, src_ld1 * sizeof(converted_T), - n1 * sizeof(converted_T), src_ld2 / src_ld1); - p.dstPtr = make_cudaPitchedPtr((void *)dst_ptr, dst_ld1 * sizeof(converted_T), - n1 * sizeof(converted_T), dst_ld2 / dst_ld1); + p.srcPtr = + make_cudaPitchedPtr((void *)src_ptr, src_ld1 * sizeof(converted_T), + n1 * sizeof(converted_T), src_ld2 / src_ld1); + p.dstPtr = + make_cudaPitchedPtr((void *)dst_ptr, dst_ld1 * sizeof(converted_T), + n1 * sizeof(converted_T), dst_ld2 / dst_ld1); p.extent = make_cudaExtent(n1 * sizeof(converted_T), n2, n3); p.kind = cudaMemcpyDefault; gpuErrchk(cudaMemcpy3DAsync(&p, stream)); diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.h index 90b578976..a343ea8db 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterHip.h @@ -959,10 +959,12 @@ template <> class MemoryManager { } hipStream_t stream = DeviceRuntime::GetQueue(queue_idx); hipMemcpy3DParms p = {}; - p.srcPtr = make_hipPitchedPtr((void *)src_ptr, src_ld1 * sizeof(converted_T), - n1 * sizeof(converted_T), src_ld2 / src_ld1); - p.dstPtr = make_hipPitchedPtr((void *)dst_ptr, dst_ld1 * sizeof(converted_T), - n1 * sizeof(converted_T), dst_ld2 / dst_ld1); + p.srcPtr = + make_hipPitchedPtr((void *)src_ptr, src_ld1 * sizeof(converted_T), + n1 * sizeof(converted_T), src_ld2 / src_ld1); + p.dstPtr = + make_hipPitchedPtr((void *)dst_ptr, dst_ld1 * sizeof(converted_T), + n1 * sizeof(converted_T), dst_ld2 / dst_ld1); p.extent = make_hipExtent(n1 * sizeof(converted_T), n2, n3); p.kind = hipMemcpyDefault; gpuErrchk(hipMemcpy3DAsync(&p, stream)); diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h index 805f5f29b..818e99369 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterKokkos.h @@ -357,12 +357,11 @@ template <> class MemoryManager { template MGARDX_CONT static void Copy3D(T *dst_ptr, SIZE dst_ld1, SIZE dst_ld2, const T *src_ptr, SIZE src_ld1, SIZE src_ld2, - SIZE n1, SIZE n2, SIZE n3, - int queue_idx) { + SIZE n1, SIZE n2, SIZE n3, int queue_idx) { log::dbg("Calling MemoryManager::Copy3D"); for (SIZE k = 0; k < n3; k++) { - Copy2D(dst_ptr + k * dst_ld2, dst_ld1, src_ptr + k * src_ld2, src_ld1, - n1, n2, queue_idx); + Copy2D(dst_ptr + k * dst_ld2, dst_ld1, src_ptr + k * src_ld2, src_ld1, n1, + n2, queue_idx); } } diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h index b31958c14..dade194aa 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterOpenmp.h @@ -430,8 +430,8 @@ template <> class MemoryManager { int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { log::dbg("Calling MemoryManager::Copy3D"); for (SIZE k = 0; k < n3; k++) { - Copy2D(dst_ptr + k * dst_ld2, dst_ld1, src_ptr + k * src_ld2, src_ld1, - n1, n2, queue_idx); + Copy2D(dst_ptr + k * dst_ld2, dst_ld1, src_ptr + k * src_ld2, src_ld1, n1, + n2, queue_idx); } } diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h index bf0a1b47c..0c68adb28 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSerial.h @@ -918,8 +918,8 @@ template <> class MemoryManager { int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { log::dbg("Calling MemoryManager::Copy3D"); for (SIZE k = 0; k < n3; k++) { - Copy2D(dst_ptr + k * dst_ld2, dst_ld1, src_ptr + k * src_ld2, src_ld1, - n1, n2, queue_idx); + Copy2D(dst_ptr + k * dst_ld2, dst_ld1, src_ptr + k * src_ld2, src_ld1, n1, + n2, queue_idx); } } diff --git a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h index 534e55f4d..e1b04ef69 100644 --- a/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h +++ b/include/mgard-x/RuntimeX/DeviceAdapters/DeviceAdapterSycl.h @@ -666,8 +666,8 @@ template <> class MemoryManager { int queue_idx = MGARDX_SYNCHRONIZED_QUEUE) { log::dbg("Calling MemoryManager::Copy3D"); for (SIZE k = 0; k < n3; k++) { - Copy2D(dst_ptr + k * dst_ld2, dst_ld1, src_ptr + k * src_ld2, src_ld1, - n1, n2, queue_idx); + Copy2D(dst_ptr + k * dst_ld2, dst_ld1, src_ptr + k * src_ld2, src_ld1, n1, + n2, queue_idx); } } diff --git a/src/mgard-x-160/Metadata/Metadata.cpp b/src/mgard-x-160/Metadata/Metadata.cpp index 25cea2abc..21ed80e4d 100644 --- a/src/mgard-x-160/Metadata/Metadata.cpp +++ b/src/mgard-x-160/Metadata/Metadata.cpp @@ -43,7 +43,8 @@ mgard::pb::Quantization::Type QuantizationTypeForWidth(std::size_t width) { case 8: return mgard::pb::Quantization::INT64_T; default: - throw mgard_x_160::InvalidDataException("unsupported quantized integer width."); + throw mgard_x_160::InvalidDataException( + "unsupported quantized integer width."); } } @@ -256,10 +257,9 @@ void MetadataBase::InitializeConfig(Config &config) { } // The decomposition has to be rebuilt with the levels it was made with. // Streams from before the level cap was recorded used all levels. - config.num_global_refactoring_level = - has_num_global_refactoring_level - ? (SIZE)num_global_refactoring_level - : std::numeric_limits::max(); + config.num_global_refactoring_level = has_num_global_refactoring_level + ? (SIZE)num_global_refactoring_level + : std::numeric_limits::max(); // The hybrid parameters are only meaningful for a hybrid file, and // Deserialize refuses to produce a hybrid file without them, so a Hybrid // decomposition here always carries a full set. For non-hybrid files leave @@ -934,7 +934,8 @@ void MetadataBase::Deserialize( "-bit integers, but this build of MGARD-X uses " + std::to_string(sizeof(mgard_x_160::QUANTIZED_INT) * 8) + "-bit."); } - if (quantization.big_endian() != big_endian()) { + if (quantization.big_endian() != + big_endian()) { throw InvalidDataException( "this file was written with the opposite endianness."); } @@ -983,8 +984,7 @@ void MetadataBase::Deserialize( { // Encoding const mgard::pb::Encoding encoding = header.encoding(); - reorder = - encoding.preprocessor() == mgard::pb::Encoding::SHUFFLE ? 1 : 0; + reorder = encoding.preprocessor() == mgard::pb::Encoding::SHUFFLE ? 1 : 0; if (encoding.compressor() == mgard::pb::Encoding::X_HUFFMAN) { ltype = mgard_x_160::lossless_type::Huffman; huff_dict_size = encoding.huffman_dictionary_size(); diff --git a/src/mgard-x/CompressionHighLevel/DynamicAPI.cpp b/src/mgard-x/CompressionHighLevel/DynamicAPI.cpp index 4e7e6c050..c59d27b24 100644 --- a/src/mgard-x/CompressionHighLevel/DynamicAPI.cpp +++ b/src/mgard-x/CompressionHighLevel/DynamicAPI.cpp @@ -57,8 +57,9 @@ enum device_type auto_detect_device() { namespace { void report_v160_disabled() { - log::err("this data was compressed by MGARD 1.6.x or earlier; decompressing " - "it requires MGARD built with -DMGARD_ENABLE_DATA_COMPATIBILITY=ON."); + log::err( + "this data was compressed by MGARD 1.6.x or earlier; decompressing " + "it requires MGARD built with -DMGARD_ENABLE_DATA_COMPATIBILITY=ON."); } template @@ -99,8 +100,8 @@ decompress_routed(const void *compressed_data, size_t compressed_size, if (compressed_by_v160(compressed_data)) { #if MGARD_ENABLE_DATA_COMPATIBILITY return v160::decompress(compressed_data, compressed_size, - decompressed_data, dtype, shape, - config, output_pre_allocated); + decompressed_data, dtype, shape, config, + output_pre_allocated); #else report_v160_disabled(); return compress_status_type::DataCompatibilityNotEnabledFailure; @@ -363,35 +364,40 @@ enum compress_status_type decompress(const void *compressed_data, if (dev_type == device_type::SERIAL) { #if MGARD_ENABLE_SERIAL return decompress_routed(compressed_data, compressed_size, - decompressed_data, config, output_pre_allocated); + decompressed_data, config, + output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP return decompress_routed(compressed_data, compressed_size, - decompressed_data, config, output_pre_allocated); + decompressed_data, config, + output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA - return decompress_routed(compressed_data, compressed_size, decompressed_data, - config, output_pre_allocated); + return decompress_routed(compressed_data, compressed_size, + decompressed_data, config, + output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP - return decompress_routed(compressed_data, compressed_size, decompressed_data, - config, output_pre_allocated); + return decompress_routed(compressed_data, compressed_size, + decompressed_data, config, + output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL - return decompress_routed(compressed_data, compressed_size, decompressed_data, - config, output_pre_allocated); + return decompress_routed(compressed_data, compressed_size, + decompressed_data, config, + output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif @@ -410,35 +416,35 @@ enum compress_status_type decompress(const void *compressed_data, if (dev_type == device_type::SERIAL) { #if MGARD_ENABLE_SERIAL return decompress_routed(compressed_data, compressed_size, - decompressed_data, output_pre_allocated); + decompressed_data, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP return decompress_routed(compressed_data, compressed_size, - decompressed_data, output_pre_allocated); + decompressed_data, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA - return decompress_routed(compressed_data, compressed_size, decompressed_data, - output_pre_allocated); + return decompress_routed(compressed_data, compressed_size, + decompressed_data, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP - return decompress_routed(compressed_data, compressed_size, decompressed_data, - output_pre_allocated); + return decompress_routed(compressed_data, compressed_size, + decompressed_data, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL - return decompress_routed(compressed_data, compressed_size, decompressed_data, - output_pre_allocated); + return decompress_routed(compressed_data, compressed_size, + decompressed_data, output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif @@ -460,37 +466,40 @@ decompress(const void *compressed_data, size_t compressed_size, if (dev_type == device_type::SERIAL) { #if MGARD_ENABLE_SERIAL return decompress_routed(compressed_data, compressed_size, - decompressed_data, dtype, shape, config, - output_pre_allocated); + decompressed_data, dtype, shape, config, + output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP return decompress_routed(compressed_data, compressed_size, - decompressed_data, dtype, shape, config, - output_pre_allocated); + decompressed_data, dtype, shape, config, + output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA - return decompress_routed(compressed_data, compressed_size, decompressed_data, - dtype, shape, config, output_pre_allocated); + return decompress_routed(compressed_data, compressed_size, + decompressed_data, dtype, shape, config, + output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP - return decompress_routed(compressed_data, compressed_size, decompressed_data, - dtype, shape, config, output_pre_allocated); + return decompress_routed(compressed_data, compressed_size, + decompressed_data, dtype, shape, config, + output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL - return decompress_routed(compressed_data, compressed_size, decompressed_data, - dtype, shape, config, output_pre_allocated); + return decompress_routed(compressed_data, compressed_size, + decompressed_data, dtype, shape, config, + output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif @@ -509,8 +518,8 @@ decompress(const void *compressed_data, size_t compressed_size, if (dev_type == device_type::SERIAL) { #if MGARD_ENABLE_SERIAL return decompress_routed(compressed_data, compressed_size, - decompressed_data, dtype, shape, - output_pre_allocated); + decompressed_data, dtype, shape, + output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif @@ -518,29 +527,32 @@ decompress(const void *compressed_data, size_t compressed_size, if (dev_type == device_type::OPENMP) { #if MGARD_ENABLE_OPENMP return decompress_routed(compressed_data, compressed_size, - decompressed_data, dtype, shape, - output_pre_allocated); + decompressed_data, dtype, shape, + output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::CUDA) { #if MGARD_ENABLE_CUDA - return decompress_routed(compressed_data, compressed_size, decompressed_data, - dtype, shape, output_pre_allocated); + return decompress_routed(compressed_data, compressed_size, + decompressed_data, dtype, shape, + output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::HIP) { #if MGARD_ENABLE_HIP - return decompress_routed(compressed_data, compressed_size, decompressed_data, - dtype, shape, output_pre_allocated); + return decompress_routed(compressed_data, compressed_size, + decompressed_data, dtype, shape, + output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif } else if (dev_type == device_type::SYCL) { #if MGARD_ENABLE_SYCL - return decompress_routed(compressed_data, compressed_size, decompressed_data, - dtype, shape, output_pre_allocated); + return decompress_routed(compressed_data, compressed_size, + decompressed_data, dtype, shape, + output_pre_allocated); #else return compress_status_type::BackendNotAvailableFailure; #endif diff --git a/src/mgard-x/Executables/ArgumentParser.h b/src/mgard-x/Executables/ArgumentParser.h index ec071f2a9..caaab5d67 100644 --- a/src/mgard-x/Executables/ArgumentParser.h +++ b/src/mgard-x/Executables/ArgumentParser.h @@ -129,13 +129,14 @@ void get_domain_decomposition(int argc, char *argv[], mgard_x::Config &config) { "--domain-decomposition"); } if (dd == "auto") { - config.domain_decomposition_mode = mgard_x::domain_decomposition_mode_type::Auto; + config.domain_decomposition_mode = + mgard_x::domain_decomposition_mode_type::Auto; } else if (dd == "manual") { config.domain_decomposition_mode = mgard_x::domain_decomposition_mode_type::Manual; - config.domain_decomposition_block_size = get_args( - argc, argv, "Domain decomposition block size", "-dd-size", - "--domain-decomposition-size"); + config.domain_decomposition_block_size = + get_args(argc, argv, "Domain decomposition block size", + "-dd-size", "--domain-decomposition-size"); } else { throw std::runtime_error("Wrong domain decomposition mode."); } diff --git a/src/mgard-x/Executables/mdr-x-qoi.cpp b/src/mgard-x/Executables/mdr-x-qoi.cpp index 4395b8691..b0d04d460 100644 --- a/src/mgard-x/Executables/mdr-x-qoi.cpp +++ b/src/mgard-x/Executables/mdr-x-qoi.cpp @@ -732,7 +732,8 @@ bool try_refactoring(int argc, char *argv[]) { max_memory_footprint); } else if (dtype == mgard_x::data_type::Float) { launch_refactor(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, dev_type, verbose, max_memory_footprint); + output_file.c_str(), shape, dev_type, verbose, + max_memory_footprint); } return true; } diff --git a/src/mgard-x/Executables/mdr-x.cpp b/src/mgard-x/Executables/mdr-x.cpp index 691e68681..a5ed1fc6d 100644 --- a/src/mgard-x/Executables/mdr-x.cpp +++ b/src/mgard-x/Executables/mdr-x.cpp @@ -306,8 +306,7 @@ int verbose_to_log_level(int verbose) { template int launch_refactor(mgard_x::DIM D, enum mgard_x::data_type dtype, std::string input_file, std::string output_file, - std::vector shape, - mgard_x::Config dd_config, + std::vector shape, mgard_x::Config dd_config, enum mgard_x::device_type dev_type, int verbose, mgard_x::SIZE max_memory_footprint, enum mgard_x::lossless_type lossless) { @@ -505,14 +504,12 @@ bool try_refactoring(int argc, char *argv[]) { get_domain_decomposition(argc, argv, dd_config); if (dtype == mgard_x::data_type::Double) { launch_refactor(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, dd_config, - dev_type, verbose, max_memory_footprint, - lossless); + output_file.c_str(), shape, dd_config, dev_type, + verbose, max_memory_footprint, lossless); } else if (dtype == mgard_x::data_type::Float) { launch_refactor(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, dd_config, - dev_type, verbose, max_memory_footprint, - lossless); + output_file.c_str(), shape, dd_config, dev_type, + verbose, max_memory_footprint, lossless); } return true; } diff --git a/src/mgard-x/Executables/mgard-x.cpp b/src/mgard-x/Executables/mgard-x.cpp index 5f7054ea0..cae616d53 100644 --- a/src/mgard-x/Executables/mgard-x.cpp +++ b/src/mgard-x/Executables/mgard-x.cpp @@ -568,8 +568,7 @@ int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, mgard_x::SIZE max_memory_footprint, mgard_x::SIZE num_local_levels, mgard_x::SIZE num_global_levels, bool use_hybrid, - bool warm_up, - bool kernel_fusion, + bool warm_up, bool kernel_fusion, mgard_x::compression_projection_mode_type projection_mode) { mgard_x::Config config; config.log_level = verbose_to_log_level(verbose); @@ -725,8 +724,8 @@ int launch_compress(mgard_x::DIM D, enum mgard_x::data_type dtype, void *decompressed_data = malloc(original_size * sizeof(T)); mgard_x::pin_memory(decompressed_data, original_size * sizeof(T), config); - ret = mgard_x::decompress(compressed_data, compressed_size, - decompressed_data, config, true); + ret = mgard_x::decompress(compressed_data, compressed_size, decompressed_data, + config, true); if (ret != mgard_x::compress_status_type::Success) { throw std::runtime_error("Decompression failed"); } @@ -962,15 +961,15 @@ bool try_compression(int argc, char *argv[]) { if (dtype == mgard_x::data_type::Double) { launch_compress( shape.size(), dtype, input_file.c_str(), output_file.c_str(), shape, - tol, tol_map, enable_roi, s, mode, lossless, dd_config, - dev_type, verbose, max_memory_footprint, num_local_levels, - num_global_levels, use_hybrid, warm_up, kernel_fusion, projection_mode); + tol, tol_map, enable_roi, s, mode, lossless, dd_config, dev_type, + verbose, max_memory_footprint, num_local_levels, num_global_levels, + use_hybrid, warm_up, kernel_fusion, projection_mode); } else if (dtype == mgard_x::data_type::Float) { launch_compress( shape.size(), dtype, input_file.c_str(), output_file.c_str(), shape, - tol, tol_map, enable_roi, s, mode, lossless, dd_config, - dev_type, verbose, max_memory_footprint, num_local_levels, - num_global_levels, use_hybrid, warm_up, kernel_fusion, projection_mode); + tol, tol_map, enable_roi, s, mode, lossless, dd_config, dev_type, + verbose, max_memory_footprint, num_local_levels, num_global_levels, + use_hybrid, warm_up, kernel_fusion, projection_mode); } mgard_x::release_cache(mgard_x::Config()); return true; diff --git a/src/mgard-x/Executables/pmdr-x-qoi-io.cpp b/src/mgard-x/Executables/pmdr-x-qoi-io.cpp index d0854e796..82a1eaaa7 100644 --- a/src/mgard-x/Executables/pmdr-x-qoi-io.cpp +++ b/src/mgard-x/Executables/pmdr-x-qoi-io.cpp @@ -1163,12 +1163,12 @@ bool try_refactoring(int argc, char *argv[]) { } if (dtype == mgard_x::data_type::Double) { launch_refactor(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, dev_type, verbose, max_memory_footprint, - rank); + output_file.c_str(), shape, dev_type, verbose, + max_memory_footprint, rank); } else if (dtype == mgard_x::data_type::Float) { launch_refactor(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, dev_type, verbose, max_memory_footprint, - rank); + output_file.c_str(), shape, dev_type, verbose, + max_memory_footprint, rank); } return true; } diff --git a/src/mgard-x/Executables/pmdr-x-qoi.cpp b/src/mgard-x/Executables/pmdr-x-qoi.cpp index d34c173b0..5d758ddd6 100644 --- a/src/mgard-x/Executables/pmdr-x-qoi.cpp +++ b/src/mgard-x/Executables/pmdr-x-qoi.cpp @@ -1114,12 +1114,12 @@ bool try_refactoring(int argc, char *argv[]) { } if (dtype == mgard_x::data_type::Double) { launch_refactor(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, dev_type, verbose, max_memory_footprint, - rank); + output_file.c_str(), shape, dev_type, verbose, + max_memory_footprint, rank); } else if (dtype == mgard_x::data_type::Float) { launch_refactor(shape.size(), dtype, input_file.c_str(), - output_file.c_str(), shape, dev_type, verbose, max_memory_footprint, - rank); + output_file.c_str(), shape, dev_type, verbose, + max_memory_footprint, rank); } return true; } diff --git a/src/mgard-x/Metadata/Metadata.cpp b/src/mgard-x/Metadata/Metadata.cpp index 9add9a2be..846c1cc1a 100644 --- a/src/mgard-x/Metadata/Metadata.cpp +++ b/src/mgard-x/Metadata/Metadata.cpp @@ -253,10 +253,9 @@ void MetadataBase::InitializeConfig(Config &config) { } // The decomposition has to be rebuilt with the levels it was made with. // Streams from before the level cap was recorded used all levels. - config.num_global_refactoring_level = - has_num_global_refactoring_level - ? (SIZE)num_global_refactoring_level - : std::numeric_limits::max(); + config.num_global_refactoring_level = has_num_global_refactoring_level + ? (SIZE)num_global_refactoring_level + : std::numeric_limits::max(); // The hybrid parameters are only meaningful for a hybrid file, and // Deserialize refuses to produce a hybrid file without them, so a Hybrid // decomposition here always carries a full set. For non-hybrid files leave @@ -818,8 +817,7 @@ void MetadataBase::Deserialize( domain_decomposition_block_shape = shape; domain_decomposed = domainDecomposition.method() != mgard::pb::DomainDecomposition::NOOP_METHOD; - if (domainDecomposition.method() == - mgard::pb::DomainDecomposition::BLOCK) { + if (domainDecomposition.method() == mgard::pb::DomainDecomposition::BLOCK) { if (domainDecomposition.block_shape_size() > 0) { if ((size_t)domainDecomposition.block_shape_size() != shape.size()) { throw InvalidDataException(