Skip to content

MGARD-X: record the decomposition depth cap in the header and restore it on decompression #263

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
gqian-coder wants to merge 2 commits into from
Closed

MGARD-X: record the decomposition depth cap in the header and restore it on decompression #263

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c2e1dfe
mgard-x: record the decomposition depth cap in the header and restore…
gqian-coder Oct 14, 2026
e08798c
mgard-x: key the hierarchy caches on the depth cap as well as the shape
gqian-coder Oct 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/MGARD-X.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ An executable ```mgard-x``` will be built after building the MGARD-X library. To
+ ```mgard_x::hybrid_projection_mode_type::Hierarchical```: remove the L<sup>2</sup> projection (L<sup>&infin;</sup> only)
+ ```fuse_decompose_quantize```: controls fused local decomposition and quantization for the hybrid hierarchy (***Default: true***)
+ ```fuse_dequantize_recompose```: controls fused local dequantization and recomposition for the hybrid hierarchy (***Default: true***)
+ ```max_larget_level```: controls max level of multi-level decomposition (***Default: 0 (no limit)***)
+ ```max_larget_level```: caps the multi-level decomposition at levels 0 to ```max_larget_level```; the cap is recorded in the compressed stream and restored on decompression (***Default: no limit***)
+ ```max_memory_footprint```(for high-level APIs only): controls maximum memory footprint in bytes (***Default: inf (no limit)***)
+ ```adjust_shape```(for high-level APIs only): controls whether or not to enable shape adjustment for better decomposition performance (***Default: false***)
## Using high-level APIs
Expand Down
24 changes: 14 additions & 10 deletions include/mgard-x/CompressionHighLevel/CPUPipelines.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,17 @@ enum compress_status_type compress_pipeline_cpu(
Cache::cache.device_compressed_buffer;

if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0),
domain_decomposer.uniform)) {
domain_decomposer.uniform,
config.max_larget_level)) {
Cache::cache.ClearHierarchyCache();
}

for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) {
if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id),
domain_decomposer.uniform)) {
domain_decomposer.uniform,
config.max_larget_level)) {
Cache::cache.InsertHierarchyCache(
domain_decomposer.subdomain_hierarchy(id));
domain_decomposer.subdomain_hierarchy(id), config.max_larget_level);
}
}
log::info("Adjust device buffers and hierarchy");
Expand All @@ -37,8 +39,8 @@ enum compress_status_type compress_pipeline_cpu(
{domain_decomposer.subdomain_compressed_buffer_size(
domain_decomposer.largest_subdomain())});

HierarchyType &hierarchy =
Cache::cache.GetHierarchyCache(domain_decomposer.subdomain_shape(i));
HierarchyType &hierarchy = Cache::cache.GetHierarchyCache(
domain_decomposer.subdomain_shape(i), config.max_larget_level);
compressor[i].Adapt(hierarchy, config, 0);
}

Expand Down Expand Up @@ -290,15 +292,17 @@ enum compress_status_type decompress_pipeline_cpu(
Cache::cache.device_compressed_buffer;

if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0),
domain_decomposer.uniform)) {
domain_decomposer.uniform,
config.max_larget_level)) {
Cache::cache.ClearHierarchyCache();
}

for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) {
if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id),
domain_decomposer.uniform)) {
domain_decomposer.uniform,
config.max_larget_level)) {
Cache::cache.InsertHierarchyCache(
domain_decomposer.subdomain_hierarchy(id));
domain_decomposer.subdomain_hierarchy(id), config.max_larget_level);
}
}

Expand All @@ -310,8 +314,8 @@ enum compress_status_type decompress_pipeline_cpu(
{domain_decomposer.subdomain_compressed_buffer_size(
domain_decomposer.largest_subdomain())});

HierarchyType &hierarchy =
Cache::cache.GetHierarchyCache(domain_decomposer.subdomain_shape(i));
HierarchyType &hierarchy = Cache::cache.GetHierarchyCache(
domain_decomposer.subdomain_shape(i), config.max_larget_level);
compressor[i].Adapt(hierarchy, config, 0);
}
std::vector<SIZE> compressed_size(domain_decomposer.num_subdomains(), 0);
Expand Down
6 changes: 6 additions & 0 deletions include/mgard-x/CompressionHighLevel/CompressionHighLevel.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,12 @@ general_compress_pipeline(std::vector<SIZE> shape, T tol, T s,
// FillForCompression does not carry the BlockDelta block size; set it here so
// it is persisted in (and restored from) the metadata header.
m.block_delta_block_size = config.block_delta_block_size;
// Nor the cap on the decomposition depth. The shape alone does not determine
// the number of levels once a cap is set, and the decompressor has to rebuild
// the exact hierarchy, so record it -- "unlimited" included, so a stale cap
// in the decompressor's Config cannot leak in either.
m.max_larget_level = config.max_larget_level;
m.max_larget_level_recorded = true;
// Same for the hybrid (BlockMGARD) parameters: the decompressor cannot
// reconstruct the buffer layout or the per-block quantization steps without
// them.
Expand Down
28 changes: 18 additions & 10 deletions include/mgard-x/CompressionHighLevel/GPUPipelines.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,17 @@ enum compress_status_type compress_pipeline_gpu(
Cache::cache.device_compressed_buffer;

if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0),
domain_decomposer.uniform)) {
domain_decomposer.uniform,
config.max_larget_level)) {
Cache::cache.ClearHierarchyCache();
}

for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) {
if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id),
domain_decomposer.uniform)) {
domain_decomposer.uniform,
config.max_larget_level)) {
Cache::cache.InsertHierarchyCache(
domain_decomposer.subdomain_hierarchy(id));
domain_decomposer.subdomain_hierarchy(id), config.max_larget_level);
}
}
log::info("Adjust device buffers");
Expand All @@ -45,7 +47,8 @@ enum compress_status_type compress_pipeline_gpu(
domain_decomposer.largest_subdomain())});

HierarchyType &hierarchy = Cache::cache.GetHierarchyCache(
domain_decomposer.subdomain_shape(domain_decomposer.largest_subdomain()));
domain_decomposer.subdomain_shape(domain_decomposer.largest_subdomain()),
config.max_larget_level);
log::info("Adapt Compressor to hierarchy");
compressor.Adapt(hierarchy, config, 0);

Expand Down Expand Up @@ -93,7 +96,8 @@ enum compress_status_type compress_pipeline_gpu(
int next_buffer = (current_buffer + 1) % 2;
int next_queue = (current_queue + 1) % 3;
HierarchyType &hierarchy = Cache::cache.GetHierarchyCache(
domain_decomposer.subdomain_shape(curr_subdomain_id));
domain_decomposer.subdomain_shape(curr_subdomain_id),
config.max_larget_level);
log::info("Adapt Compressor to hierarchy");
compressor.Adapt(hierarchy, config, current_queue);

Expand Down Expand Up @@ -277,15 +281,17 @@ enum compress_status_type decompress_pipeline_gpu(
Cache::cache.device_compressed_buffer;

if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(0),
domain_decomposer.uniform)) {
domain_decomposer.uniform,
config.max_larget_level)) {
Cache::cache.ClearHierarchyCache();
}

for (SIZE id = 0; id < domain_decomposer.num_subdomains(); id++) {
if (!Cache::cache.InHierarchyCache(domain_decomposer.subdomain_shape(id),
domain_decomposer.uniform)) {
domain_decomposer.uniform,
config.max_larget_level)) {
Cache::cache.InsertHierarchyCache(
domain_decomposer.subdomain_hierarchy(id));
domain_decomposer.subdomain_hierarchy(id), config.max_larget_level);
}
}

Expand All @@ -305,7 +311,8 @@ enum compress_status_type decompress_pipeline_gpu(
domain_decomposer.largest_subdomain())});

HierarchyType &hierarchy = Cache::cache.GetHierarchyCache(
domain_decomposer.subdomain_shape(domain_decomposer.largest_subdomain()));
domain_decomposer.subdomain_shape(domain_decomposer.largest_subdomain()),
config.max_larget_level);
log::info("Adapt Compressor to hierarchy");
compressor.Adapt(hierarchy, config, 0);

Expand Down Expand Up @@ -365,7 +372,8 @@ enum compress_status_type decompress_pipeline_gpu(
int next_buffer = (current_buffer + 1) % 2;
int next_queue = (current_queue + 1) % 3;
HierarchyType &hierarchy = Cache::cache.GetHierarchyCache(
domain_decomposer.subdomain_shape(curr_subdomain_id));
domain_decomposer.subdomain_shape(curr_subdomain_id),
config.max_larget_level);
log::info("Adapt Compressor to hierarchy");
compressor.Adapt(hierarchy, config, current_queue);

Expand Down
33 changes: 22 additions & 11 deletions include/mgard-x/CompressionLowLevel/CompressorCache.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,47 +34,58 @@ class CompressorBundle {

bool initialized = false;

std::string GenHierarchyKey(std::vector<SIZE> shape) {
// A hierarchy is determined by its shape together with the cap on its depth
// (Config::max_larget_level): the same shape under a different cap has a
// different number of levels. Both are part of the key; keying on the shape
// alone let a decompress (or a later compress) silently reuse a hierarchy
// that was built for another cap.
std::string GenHierarchyKey(std::vector<SIZE> shape, SIZE max_larget_level) {
std::stringstream ss;
for (int i = 0; i < shape.size(); i++) {
ss << std::setfill('0') << std::setw(20) << shape[i] << std::fixed;
}
ss << "|" << max_larget_level;
return ss.str();
}

bool InHierarchyCache(std::vector<SIZE> shape, bool uniform) {
bool InHierarchyCache(std::vector<SIZE> shape, bool uniform,
SIZE max_larget_level) {
if (!initialized) {
return false;
} else if (!uniform) {
return false;
} else if (hierarchy_cache->find(GenHierarchyKey(shape)) ==
hierarchy_cache->end()) {
} else if (hierarchy_cache->find(GenHierarchyKey(
shape, max_larget_level)) == hierarchy_cache->end()) {
return false;
} else if ((*hierarchy_cache)[GenHierarchyKey(shape)].data_structure() ==
} else if ((*hierarchy_cache)[GenHierarchyKey(shape, max_larget_level)]
.data_structure() ==
data_structure_type::Cartesian_Grid_Non_Uniform) {
return false;
} else {
return true;
}
}

void InsertHierarchyCache(HierarchyType hierarchy) {
void InsertHierarchyCache(HierarchyType hierarchy, SIZE max_larget_level) {
std::vector<SIZE> shape = hierarchy.level_shape(hierarchy.l_target());
(*hierarchy_cache)[GenHierarchyKey(shape)] = hierarchy;
(*hierarchy_cache)[GenHierarchyKey(shape, max_larget_level)] = hierarchy;
std::stringstream ss;
for (int i = 0; i < shape.size(); i++) {
ss << shape[i] << " ";
}
log::info("Add hierarchy with shape ( " + ss.str() + ") into cache");
log::info("Add hierarchy with shape ( " + ss.str() + ") and max level " +
std::to_string(max_larget_level) + " into cache");
}

HierarchyType &GetHierarchyCache(std::vector<SIZE> shape) {
HierarchyType &GetHierarchyCache(std::vector<SIZE> shape,
SIZE max_larget_level) {
std::stringstream ss;
for (int i = 0; i < shape.size(); i++) {
ss << shape[i] << " ";
}
log::info("Get hierarchy with shape ( " + ss.str() + ") from cache");
return (*hierarchy_cache)[GenHierarchyKey(shape)];
log::info("Get hierarchy with shape ( " + ss.str() + ") and max level " +
std::to_string(max_larget_level) + " from cache");
return (*hierarchy_cache)[GenHierarchyKey(shape, max_larget_level)];
}

void Release() {
Expand Down
33 changes: 22 additions & 11 deletions include/mgard-x/MDR-X/Reconstructor/ReconstructorCache.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,47 +28,58 @@ class ReconstructorBundle {

bool initialized = false;

std::string GenHierarchyKey(std::vector<SIZE> shape) {
// A hierarchy is determined by its shape together with the cap on its depth
// (Config::max_larget_level): the same shape under a different cap has a
// different number of levels. Both are part of the key; keying on the shape
// alone let a decompress (or a later compress) silently reuse a hierarchy
// that was built for another cap.
std::string GenHierarchyKey(std::vector<SIZE> shape, SIZE max_larget_level) {
std::stringstream ss;
for (int i = 0; i < shape.size(); i++) {
ss << std::setfill('0') << std::setw(20) << shape[i] << std::fixed;
}
ss << "|" << max_larget_level;
return ss.str();
}

bool InHierarchyCache(std::vector<SIZE> shape, bool uniform) {
bool InHierarchyCache(std::vector<SIZE> shape, bool uniform,
SIZE max_larget_level) {
if (!initialized) {
return false;
} else if (!uniform) {
return false;
} else if (hierarchy_cache->find(GenHierarchyKey(shape)) ==
hierarchy_cache->end()) {
} else if (hierarchy_cache->find(GenHierarchyKey(
shape, max_larget_level)) == hierarchy_cache->end()) {
return false;
} else if ((*hierarchy_cache)[GenHierarchyKey(shape)].data_structure() ==
} else if ((*hierarchy_cache)[GenHierarchyKey(shape, max_larget_level)]
.data_structure() ==
data_structure_type::Cartesian_Grid_Non_Uniform) {
return false;
} else {
return true;
}
}

void InsertHierarchyCache(HierarchyType hierarchy) {
void InsertHierarchyCache(HierarchyType hierarchy, SIZE max_larget_level) {
std::vector<SIZE> shape = hierarchy.level_shape(hierarchy.l_target());
(*hierarchy_cache)[GenHierarchyKey(shape)] = hierarchy;
(*hierarchy_cache)[GenHierarchyKey(shape, max_larget_level)] = hierarchy;
std::stringstream ss;
for (int i = 0; i < shape.size(); i++) {
ss << shape[i] << " ";
}
log::info("Add hierarchy with shape ( " + ss.str() + ") into cache");
log::info("Add hierarchy with shape ( " + ss.str() + ") and max level " +
std::to_string(max_larget_level) + " into cache");
}

HierarchyType &GetHierarchyCache(std::vector<SIZE> shape) {
HierarchyType &GetHierarchyCache(std::vector<SIZE> shape,
SIZE max_larget_level) {
std::stringstream ss;
for (int i = 0; i < shape.size(); i++) {
ss << shape[i] << " ";
}
log::info("Get hierarchy with shape ( " + ss.str() + ") from cache");
return (*hierarchy_cache)[GenHierarchyKey(shape)];
log::info("Get hierarchy with shape ( " + ss.str() + ") and max level " +
std::to_string(max_larget_level) + " from cache");
return (*hierarchy_cache)[GenHierarchyKey(shape, max_larget_level)];
}

void Release() {
Expand Down
33 changes: 22 additions & 11 deletions include/mgard-x/MDR-X/Refactor/RefactorCache.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,47 +28,58 @@ class RefactorBundle {

bool initialized = false;

std::string GenHierarchyKey(std::vector<SIZE> shape) {
// A hierarchy is determined by its shape together with the cap on its depth
// (Config::max_larget_level): the same shape under a different cap has a
// different number of levels. Both are part of the key; keying on the shape
// alone let a decompress (or a later compress) silently reuse a hierarchy
// that was built for another cap.
std::string GenHierarchyKey(std::vector<SIZE> shape, SIZE max_larget_level) {
std::stringstream ss;
for (int i = 0; i < shape.size(); i++) {
ss << std::setfill('0') << std::setw(20) << shape[i] << std::fixed;
}
ss << "|" << max_larget_level;
return ss.str();
}

bool InHierarchyCache(std::vector<SIZE> shape, bool uniform) {
bool InHierarchyCache(std::vector<SIZE> shape, bool uniform,
SIZE max_larget_level) {
if (!initialized) {
return false;
} else if (!uniform) {
return false;
} else if (hierarchy_cache->find(GenHierarchyKey(shape)) ==
hierarchy_cache->end()) {
} else if (hierarchy_cache->find(GenHierarchyKey(
shape, max_larget_level)) == hierarchy_cache->end()) {
return false;
} else if ((*hierarchy_cache)[GenHierarchyKey(shape)].data_structure() ==
} else if ((*hierarchy_cache)[GenHierarchyKey(shape, max_larget_level)]
.data_structure() ==
data_structure_type::Cartesian_Grid_Non_Uniform) {
return false;
} else {
return true;
}
}

void InsertHierarchyCache(HierarchyType hierarchy) {
void InsertHierarchyCache(HierarchyType hierarchy, SIZE max_larget_level) {
std::vector<SIZE> shape = hierarchy.level_shape(hierarchy.l_target());
(*hierarchy_cache)[GenHierarchyKey(shape)] = hierarchy;
(*hierarchy_cache)[GenHierarchyKey(shape, max_larget_level)] = hierarchy;
std::stringstream ss;
for (int i = 0; i < shape.size(); i++) {
ss << shape[i] << " ";
}
log::info("Add hierarchy with shape ( " + ss.str() + ") into cache");
log::info("Add hierarchy with shape ( " + ss.str() + ") and max level " +
std::to_string(max_larget_level) + " into cache");
}

HierarchyType &GetHierarchyCache(std::vector<SIZE> shape) {
HierarchyType &GetHierarchyCache(std::vector<SIZE> shape,
SIZE max_larget_level) {
std::stringstream ss;
for (int i = 0; i < shape.size(); i++) {
ss << shape[i] << " ";
}
log::info("Get hierarchy with shape ( " + ss.str() + ") from cache");
return (*hierarchy_cache)[GenHierarchyKey(shape)];
log::info("Get hierarchy with shape ( " + ss.str() + ") and max level " +
std::to_string(max_larget_level) + " from cache");
return (*hierarchy_cache)[GenHierarchyKey(shape, max_larget_level)];
}

void Release() {
Expand Down
Loading
Loading