From 5a4e54999c16b445b66e03c1d988622b8003d9e1 Mon Sep 17 00:00:00 2001 From: nhz2 Date: Mon, 24 Aug 2026 21:22:45 -0400 Subject: [PATCH 1/3] Use ChunkCodecLibLzma for XZ compression --- Project.toml | 4 ++-- src/ASDF.jl | 16 +++++++--------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/Project.toml b/Project.toml index efe8092..7ae2024 100644 --- a/Project.toml +++ b/Project.toml @@ -11,9 +11,9 @@ AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" ChunkCodecLibBlosc = "c6a955be-ab7f-4fbb-b38f-caf93db6b928" ChunkCodecLibBzip2 = "2b723af9-f480-4e8d-a1e4-4a9f5a906122" ChunkCodecLibLz4 = "7e9cc85e-5614-42a3-ad86-b78f920b38a5" +ChunkCodecLibLzma = "e95d29e5-19c5-4afd-ae0f-beb790efacdf" ChunkCodecLibZlib = "4c0bbee4-addc-4d73-81a0-b6caacae83c8" ChunkCodecLibZstd = "55437552-ac27-4d47-9aa3-63184e8fd398" -CodecXz = "ba30903b-d9e8-5048-a5ec-d1f5b0d4b47b" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" MD5 = "6ac74813-4b46-53a4-afec-0b5dc9d7885c" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" @@ -26,9 +26,9 @@ AbstractTrees = "0.4.5" ChunkCodecLibBlosc = "0.2.0, 0.3" ChunkCodecLibBzip2 = "0.2.0, 1.0" ChunkCodecLibLz4 = "0.2.1, 1.0" +ChunkCodecLibLzma = "1.0" ChunkCodecLibZlib = "0.2.0, 1.0" ChunkCodecLibZstd = "0.2.0, 1.0" -CodecXz = "0.7.4" FileIO = "1" MD5 = "0.2.1" OrderedCollections = "1.8.1, 2" diff --git a/src/ASDF.jl b/src/ASDF.jl index ce7d528..2de14f6 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -3,9 +3,9 @@ module ASDF using ChunkCodecLibBlosc: BloscCodec, BloscEncodeOptions using ChunkCodecLibBzip2: BZ2Codec, BZ2EncodeOptions using ChunkCodecLibLz4: LZ4BlockCodec, LZ4FrameCodec, LZ4BlockEncodeOptions, LZ4FrameEncodeOptions +using ChunkCodecLibLzma: XZCodec, XZEncodeOptions using ChunkCodecLibZlib: ZlibCodec, ZlibEncodeOptions using ChunkCodecLibZstd: ZstdCodec, ZstdEncodeOptions, decode, encode -using CodecXz: XzCompressor, XzDecompressor using MD5: md5 using PkgVersion: PkgVersion using StridedViews: StridedView @@ -282,8 +282,6 @@ function read_block(header::BlockHeader) compression = compression_enums[header.compression] if compression == C_None # do nothing, the block is uncompressed - elseif compression == C_Xz - data = transcode(XzDecompressor, data) elseif compression == C_Lz4 data = decode_Lz4(data) else @@ -291,6 +289,8 @@ function read_block(header::BlockHeader) codec = BloscCodec() elseif compression == C_Bzip2 codec = BZ2Codec() + elseif compression == C_Xz + codec = XZCodec() elseif compression == C_Zlib codec = ZlibCodec() elseif compression == C_Zstd @@ -298,7 +298,8 @@ function read_block(header::BlockHeader) else error("Invalid compression format found: $compression") end - data = decode(codec, data) + d_size = Int64(header.data_size) + data = decode(codec, data; max_size=d_size, size_hint=d_size) end data::AbstractVector{UInt8} @@ -1835,11 +1836,6 @@ function write_file(filename::AbstractString, document::AbstractDict) # TODO: Write directly to file if array.compression == C_None data = input - elseif array.compression == C_Xz - # Copy - # TODO: Don't copy input - input = input isa Vector ? input : Vector(input) - data = transcode(XzCompressor, input) elseif array.compression == C_Lz4 && array.lz4_layout == :block data = encode_Lz4_block(input) #data = encode(LZ4BlockEncodeOptions(), input) # Not compatible with Python asdf @@ -1850,6 +1846,8 @@ function write_file(filename::AbstractString, document::AbstractDict) encode_options = BZ2EncodeOptions(; blockSize100k = 9) elseif array.compression == C_Lz4 && array.lz4_layout == :frame encode_options = LZ4FrameEncodeOptions(; compressionLevel = 12, blockSizeID = 7) + elseif array.compression == C_Xz + encode_options = XZEncodeOptions(; preset = UInt32(6)) elseif array.compression == C_Zlib encode_options = ZlibEncodeOptions(; level = 9) elseif array.compression == C_Zstd From 22d2b54d14e7882ee69238c3576426e39aa873ce Mon Sep 17 00:00:00 2001 From: nhz2 Date: Mon, 24 Aug 2026 21:34:19 -0400 Subject: [PATCH 2/3] docs fix --- src/ASDF.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index 2de14f6..6a8bacf 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -36,7 +36,7 @@ Identifies the compression algorithm used for a data block. Available variants: | `C_Bzip2` | `bzp2` | ChunkCodecLibBzip2.jl | Good ratio, moderate speed (default) | | `C_Lz4` (:block) | `lz4\\0` | ChunkCodecLibLz4.jl | Fastest decompression, Python-compatible | | `C_Lz4` (:frame) | `lz4\\0` | ChunkCodecLibLz4.jl | LZ4 frame format for non-Python consumers | -| `C_Xz` | `xz\\0\\0` | CodecXz.jl | Highest compression ratio, slowest | +| `C_Xz` | `xz\\0\\0` | ChunkCodecLibLzma.jl | Highest compression ratio, slowest | | `C_Zlib` | `zlib` | ChunkCodecLibZlib.jl | Broad compatibility | | `C_Zstd` | `zstd` | ChunkCodecLibZstd.jl | Best ratio/speed trade-off | """ From e862e3a9b0115340d816b7fec68665fa49732f6f Mon Sep 17 00:00:00 2001 From: nhz2 Date: Mon, 24 Aug 2026 21:38:27 -0400 Subject: [PATCH 3/3] fix formatting --- src/ASDF.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ASDF.jl b/src/ASDF.jl index 6a8bacf..8b68824 100644 --- a/src/ASDF.jl +++ b/src/ASDF.jl @@ -299,7 +299,7 @@ function read_block(header::BlockHeader) error("Invalid compression format found: $compression") end d_size = Int64(header.data_size) - data = decode(codec, data; max_size=d_size, size_hint=d_size) + data = decode(codec, data; max_size = d_size, size_hint = d_size) end data::AbstractVector{UInt8}