From e5278409a134a337e63d6f8998c4485507581abc Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 10 Sep 2026 16:29:54 -0700 Subject: [PATCH 1/6] buck: add missing comma in the :errors target lib/BUCK does not parse : `exported_headers=[...]` is not followed by a comma, so the whole file is a syntax error, for buck1 and buck2 alike. --- lib/BUCK | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/BUCK b/lib/BUCK index 8c555a8be18..70414b1d379 100644 --- a/lib/BUCK +++ b/lib/BUCK @@ -132,7 +132,7 @@ cxx_library( exported_headers=[ 'zstd_errors.h', 'common/error_private.h', - ] + ], srcs=['common/error_private.c'], ) From cbaf011e6b8187c5ec77bdc18a9df2b1d3145daf Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 10 Sep 2026 16:20:28 -0700 Subject: [PATCH 2/6] build: align remaining legacy defaults with ZSTD_LEGACY_SUPPORT=0 .buckconfig and the VS2005 projects were left at 4 when the other build systems moved to 0, and programs/README.md still documented v0.4+ as the default. In tests/fuzz/Makefile, emit the define from $(ZSTD_LEGACY_SUPPORT) instead of hardcoding 1, so the local ?= 1 actually controls the fuzzers, which keep legacy enabled. It has to stay in FUZZ_CPPFLAGS : fuzz.py passes CPPFLAGS as a make command line variable, which overrides the `CPPFLAGS +=` from libzstd.mk. Remove the cmake include of programs/legacy, a directory that no longer exists; no program source includes a legacy header. --- .buckconfig | 4 ++-- build/cmake/programs/CMakeLists.txt | 5 ----- contrib/VS2005/zstd/zstd.vcproj | 8 ++++---- contrib/VS2005/zstdlib/zstdlib.vcproj | 8 ++++---- programs/README.md | 7 ++++--- tests/fuzz/Makefile | 7 ++++++- 6 files changed, 20 insertions(+), 19 deletions(-) diff --git a/.buckconfig b/.buckconfig index 483f6053b26..1d0d1153008 100644 --- a/.buckconfig +++ b/.buckconfig @@ -1,7 +1,7 @@ [cxx] - cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4 + cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=0 cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith - cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4 + cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=0 cxxflags = -std=c++11 -Wno-deprecated-declarations gtest_dep = //contrib/pzstd:gtest diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt index 5e239e32a3d..ec23787bb6e 100644 --- a/build/cmake/programs/CMakeLists.txt +++ b/build/cmake/programs/CMakeLists.txt @@ -16,11 +16,6 @@ set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib) set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs) include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/compress ${LIBRARY_DIR}/dictBuilder) -if (ZSTD_LEGACY_SUPPORT) - set(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy) - include_directories(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy) -endif () - if (ZSTD_PROGRAMS_LINK_SHARED) set(PROGRAMS_ZSTD_LINK_TARGET libzstd_shared) else () diff --git a/contrib/VS2005/zstd/zstd.vcproj b/contrib/VS2005/zstd/zstd.vcproj index e37ebee3911..979a7f70e4e 100644 --- a/contrib/VS2005/zstd/zstd.vcproj +++ b/contrib/VS2005/zstd/zstd.vcproj @@ -44,7 +44,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress" - PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -121,7 +121,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress" - PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" @@ -196,7 +196,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress" - PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -274,7 +274,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress" - PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" diff --git a/contrib/VS2005/zstdlib/zstdlib.vcproj b/contrib/VS2005/zstdlib/zstdlib.vcproj index 67ddd2dba3a..e5e98d14f0e 100644 --- a/contrib/VS2005/zstdlib/zstdlib.vcproj +++ b/contrib/VS2005/zstdlib/zstdlib.vcproj @@ -44,7 +44,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -120,7 +120,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" @@ -194,7 +194,7 @@ Name="VCCLCompilerTool" Optimization="0" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE" MinimalRebuild="true" BasicRuntimeChecks="3" RuntimeLibrary="3" @@ -271,7 +271,7 @@ EnableIntrinsicFunctions="true" OmitFramePointers="true" AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder" - PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE" + PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_MULTITHREAD=1;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE" RuntimeLibrary="0" EnableFunctionLevelLinking="true" UsePrecompiledHeader="0" diff --git a/programs/README.md b/programs/README.md index 6522b647dd3..268bdc52357 100644 --- a/programs/README.md +++ b/programs/README.md @@ -3,8 +3,8 @@ Command Line Interface for Zstandard library Command Line Interface (CLI) can be created using the `make` command without any additional parameters. There are however other Makefile targets that create different variations of CLI: -- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder, benchmark, and supports decompression of legacy zstd formats -- `zstd_nolegacy` : Same as `zstd` but without support for legacy zstd formats +- `zstd` : default CLI supporting gzip-like arguments; includes dictionary builder and benchmark +- `zstd_nolegacy` : Same as `zstd` but with support for legacy zstd formats explicitly disabled - `zstd-small` : CLI optimized for minimal size; no dictionary builder, no benchmark, and no support for legacy zstd formats - `zstd-compress` : version of CLI which can only compress into zstd format - `zstd-decompress` : version of CLI which can only decompress zstd format @@ -23,11 +23,12 @@ There are however other Makefile targets that create different variations of CLI - __ZSTD_LEGACY_SUPPORT__ : `zstd` can decompress files compressed by older versions of `zstd`. Starting v0.8.0, all versions of `zstd` produce frames compliant with the [specification](../doc/zstd_compression_format.md), and are therefore compatible. But older versions (< v0.8.0) produced different, incompatible, frames. - By default, `zstd` supports decoding legacy formats >= v0.4.0 (`ZSTD_LEGACY_SUPPORT=4`). + Since v1.6.0, legacy format support is disabled by default (`ZSTD_LEGACY_SUPPORT=0`). This can be altered by modifying this compilation variable. `ZSTD_LEGACY_SUPPORT=1` means "support all formats >= v0.1.0". `ZSTD_LEGACY_SUPPORT=2` means "support all formats >= v0.2.0", and so on. `ZSTD_LEGACY_SUPPORT=0` means _DO NOT_ support any legacy format. + Example : `make zstd ZSTD_LEGACY_SUPPORT=4` if `ZSTD_LEGACY_SUPPORT >= 8`, it's the same as `0`, since there is no legacy format after `7`. Note : `zstd` only supports decoding older formats, and cannot generate any legacy format. diff --git a/tests/fuzz/Makefile b/tests/fuzz/Makefile index 13e29bb7a96..1141c51265c 100644 --- a/tests/fuzz/Makefile +++ b/tests/fuzz/Makefile @@ -26,6 +26,7 @@ CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corp LIBZSTD_MK_DIR = ../../lib DEBUGLEVEL ?= 2 +# fuzzers keep legacy decoders enabled, they are still built on demand ZSTD_LEGACY_SUPPORT ?= 1 .PHONY: default @@ -40,10 +41,14 @@ DEFAULT_SEQ_PROD_DIR = $(CONTRIBDIR)/externalSequenceProducer DEFAULT_SEQ_PROD_SRC = $(DEFAULT_SEQ_PROD_DIR)/sequence_producer.c THIRD_PARTY_SEQ_PROD_OBJ ?= +# Note : ZSTD_LEGACY_SUPPORT below looks redundant with libzstd.mk, which adds +# the same define to CPPFLAGS. It is not : fuzz.py passes CPPFLAGS on the make +# command line, and that overrides the `CPPFLAGS +=` there. Dropping it silently +# disables the legacy decoders in every OSS-Fuzz and regressiontest build. FUZZ_CPPFLAGS := -I$(LIB_SRCDIR) -I$(LIB_SRCDIR)/common -I$(LIB_SRCDIR)/compress \ -I$(LIB_SRCDIR)/dictBuilder -I$(LIB_SRCDIR)/deprecated -I$(LIB_SRCDIR)/legacy \ -I$(CONTRIBDIR)/seekable_format -I$(PRGDIR) -I$(DEFAULT_SEQ_PROD_DIR) \ - -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS) + -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $(CPPFLAGS) FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ -Wstrict-prototypes -Wundef \ From 6fe3047920e5c8691fee74cb22bc87aa530b33b2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 10 Sep 2026 16:20:45 -0700 Subject: [PATCH 3/6] tests: select legacy decoders with the same filter as the library tests/Makefile had its own unfiltered wildcard, so the test binaries linked all 7 decoders at every level : 7 instead of 3 at level 5, and 7 instead of none at level 0 or 8, which the documented semantics treat as disabled. fuzzer, zstreamtest, fullbench and decodecorpus all carried decoders ZSTD_isLegacy() can never reach. Reuse ZSTD_LEGACY_FILES, already filtered by libzstd.mk, so tests compile exactly what the library compiles. The `legacy` target now names the full list explicitly, since it overrides the level through CPPFLAGS only. While there, test-variants.sh checked the default CLI for ZSTDv01..v04 only, which the old default of 5 already guaranteed. Check for ZSTDv0 instead, so it asserts the CLI ships no legacy decoder at all. --- tests/Makefile | 10 +++++++--- tests/test-variants.sh | 6 ++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index bfa796ba05b..c4b516e1acd 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -44,8 +44,12 @@ CPPFLAGS += -I$(LIB_SRCDIR) -I$(LIB_SRCDIR)/common -I$(LIB_SRCDIR)/compress -I ZSTDCOMMON_FILES := $(sort $(ZSTD_COMMON_FILES)) ZSTDCOMP_FILES := $(sort $(ZSTD_COMPRESS_FILES)) ZSTDDECOMP_FILES := $(sort $(ZSTD_DECOMPRESS_FILES)) -ZSTDLEGACY_FILES := $(sort $(wildcard $(LIB_SRCDIR)/legacy/*.c)) -ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) $(ZSTDLEGACY_FILES) +# every decoder, whatever ZSTD_LEGACY_SUPPORT : only the `legacy` target wants these +ALL_LEGACY_FILES := $(sort $(wildcard $(LIB_SRCDIR)/legacy/*.c)) +# ZSTD_LEGACY_FILES (from libzstd.mk) is the subset selected for the requested +# level. It's empty when legacy support is disabled, hence unreachable decoders +# are not compiled into the test binaries. +ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) $(sort $(ZSTD_LEGACY_FILES)) ZDICT_FILES := $(sort $(ZSTD_DICTBUILDER_FILES)) ZSTD_F1 := $(sort $(wildcard $(ZSTD_FILES))) @@ -234,7 +238,7 @@ invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c CLEAN += legacy legacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=4 -legacy : $(ZSTD_FILES) legacy.c +legacy : $(ZSTD_FILES) $(ALL_LEGACY_FILES) legacy.c CLEAN += decodecorpus decodecorpus : LDLIBS += -lm diff --git a/tests/test-variants.sh b/tests/test-variants.sh index f3a9e065bde..36803e8ff8a 100755 --- a/tests/test-variants.sh +++ b/tests/test-variants.sh @@ -97,10 +97,8 @@ cliextra_not_present "$ZSTD_DICTBUILDER" legacy_not_present "$ZSTD_DICTBUILDER" legacy_not_present "$ZSTD_NOLEGACY" -symbol_not_present "$ZSTD" ZSTDv01 -symbol_not_present "$ZSTD" ZSTDv02 -symbol_not_present "$ZSTD" ZSTDv03 -symbol_not_present "$ZSTD" ZSTDv04 +# since v1.6.0, the default CLI is built without any legacy decoder +legacy_not_present "$ZSTD" test_compress "$ZSTD_COMPRESS" test_decompress "$ZSTD_DECOMPRESS" From a31fac707b03967834e5010bde5b3bb4109e9df0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 10 Sep 2026 16:20:52 -0700 Subject: [PATCH 4/6] tests: cover legacy decoders v0.1, v0.2 and v0.3 legacy.c only carried frames from v0.4.3 onward, so the 3 oldest decoders were never exercised by any in-tree test. Add one frame per format version, produced by the last release able to generate it (v0.1.1, v0.2.2 and v0.3.6), and make the test select at compile time the frames its ZSTD_LEGACY_SUPPORT level can decode. It therefore works at any level, instead of requiring exactly <= 4. `legacy` is now built at level 1 by default, tunable via LEGACY_TEST_LEVEL. Formats older than v0.4 have no streaming decoder, so the streaming leg skips them, mirroring ZSTD_initLegacyStream(). It gets its own frame buffer, so that each buffer keeps matching the size recorded next to it. Both APIs now also verify the decoded size, instead of reporting the mismatch and carrying on over a buffer that was never fully written. --- build/meson/tests/meson.build | 7 +- tests/Makefile | 5 +- tests/legacy.c | 334 +++++++++++++++++++++++++--------- 3 files changed, 257 insertions(+), 89 deletions(-) diff --git a/build/meson/tests/meson.build b/build/meson/tests/meson.build index 71ffc50693d..dcfc07c67a9 100644 --- a/build/meson/tests/meson.build +++ b/build/meson/tests/meson.build @@ -102,12 +102,11 @@ invalidDictionaries = executable('invalidDictionaries', dependencies: [ libzstd_dep ], install: false) -if 0 < legacy_level and legacy_level <= 4 +if legacy_level > 0 legacy_sources = [join_paths(zstd_rootdir, 'tests/legacy.c')] + # the test adapts to ZSTD_LEGACY_SUPPORT, set project wide from legacy_level legacy = executable('legacy', legacy_sources, - # Use -Dlegacy_level build option to control it - #c_args: '-DZSTD_LEGACY_SUPPORT=4', dependencies: [ libzstd_dep ], install: false) endif @@ -199,7 +198,7 @@ test('test-zstream-3', timeout: 120) test('test-longmatch', longmatch, timeout: 36) test('test-invalidDictionaries', invalidDictionaries) # should be fast -if 0 < legacy_level and legacy_level <= 4 +if legacy_level > 0 test('test-legacy', legacy) # should be fast endif test('test-decodecorpus', diff --git a/tests/Makefile b/tests/Makefile index c4b516e1acd..e268ddeb9e4 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -237,7 +237,10 @@ CLEAN += invalidDictionaries invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c CLEAN += legacy -legacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=4 +# legacy.c adapts to the level it is compiled with : it only tests the frames +# this level can decode. Default to the oldest format, to cover all decoders. +LEGACY_TEST_LEVEL ?= 1 +legacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=$(LEGACY_TEST_LEVEL) legacy : $(ZSTD_FILES) $(ALL_LEGACY_FILES) legacy.c CLEAN += decodecorpus diff --git a/tests/legacy.c b/tests/legacy.c index 8fe291c5a0d..18a590f1d51 100644 --- a/tests/legacy.c +++ b/tests/legacy.c @@ -19,11 +19,18 @@ #include /* size_t */ #include /* malloc, free */ #include /* fprintf */ -#include /* strlen */ +#include /* strlen, memcpy, memcmp */ #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_decompressBound */ #include "zstd.h" #include "zstd_errors.h" +/* Same normalization as lib/legacy/zstd_legacy.h : + * no legacy support means only the current format can be decoded */ +#if !defined(ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0) +# undef ZSTD_LEGACY_SUPPORT +# define ZSTD_LEGACY_SUPPORT 8 +#endif + /*=========================================== * Macros *==========================================*/ @@ -32,15 +39,87 @@ /*=========================================== * Precompressed frames *==========================================*/ -const char* const COMPRESSED; /* content is at end of file */ -size_t const COMPRESSED_SIZE = 917; -const char* const EXPECTED; /* content is at end of file */ +/* Each frame below is the "BLOCK" text, compressed with default settings by + * the last release able to produce that format version. + * A frame produced by v0.N is only decodable when ZSTD_LEGACY_SUPPORT <= N, + * hence the test adapts itself to the level it has been compiled with. + */ +#define FRAME_V01_SIZE 189 +#define FRAME_V02_SIZE 187 +#define FRAME_V03_SIZE 187 +#define FRAME_V04_SIZE 198 +#define FRAME_V05_SIZE 184 +#define FRAME_V06_SIZE 178 +#define FRAME_V07_SIZE 178 +#define FRAME_V08_SIZE 179 +#define ALL_FRAMES_SIZE (FRAME_V01_SIZE + FRAME_V02_SIZE + FRAME_V03_SIZE + FRAME_V04_SIZE + FRAME_V05_SIZE + FRAME_V06_SIZE + FRAME_V07_SIZE + FRAME_V08_SIZE) + +const char* const FRAME_V01; /* content is at end of file, produced by v0.1.1 */ +const char* const FRAME_V02; /* content is at end of file, produced by v0.2.2 */ +const char* const FRAME_V03; /* content is at end of file, produced by v0.3.6 */ +const char* const FRAME_V04; /* content is at end of file, produced by v0.4.3 */ +const char* const FRAME_V05; /* content is at end of file, produced by v0.5.0 */ +const char* const FRAME_V06; /* content is at end of file, produced by v0.6.0 */ +const char* const FRAME_V07; /* content is at end of file, produced by v0.7.0 */ +const char* const FRAME_V08; /* content is at end of file, produced by v0.8.0 */ +const char* const BLOCK; /* content is at end of file */ + +/* legacy formats older than v0.4 only offer a single-shot decoder : + * ZSTD_decompressStream() answers version_unsupported on such frames + * (see ZSTD_initLegacyStream() in lib/legacy/zstd_legacy.h) */ +#define OLDEST_STREAMABLE_VERSION 4 -static int testSimpleAPI(void) +/* buildTestFrames() : + * concatenates, oldest first, the frames this build is able to decode, + * skipping those older than @fromVersion. + * @dst must be at least ALL_FRAMES_SIZE bytes long. + * @return : nb of frames written; *cSize receives their total size */ +static unsigned buildTestFrames(char* dst, size_t* cSize, int fromVersion) { - size_t const size = strlen(EXPECTED); - char* const output = malloc(size); + unsigned nbFrames = 0; + size_t pos = 0; + +#define ADD_FRAME(n, v) \ + if (fromVersion <= (n)) { \ + memcpy(dst + pos, FRAME_V##v, FRAME_V##v##_SIZE); \ + pos += FRAME_V##v##_SIZE; \ + nbFrames++; \ + } + +#if (ZSTD_LEGACY_SUPPORT <= 1) + ADD_FRAME(1, 01) +#endif +#if (ZSTD_LEGACY_SUPPORT <= 2) + ADD_FRAME(2, 02) +#endif +#if (ZSTD_LEGACY_SUPPORT <= 3) + ADD_FRAME(3, 03) +#endif +#if (ZSTD_LEGACY_SUPPORT <= 4) + ADD_FRAME(4, 04) +#endif +#if (ZSTD_LEGACY_SUPPORT <= 5) + ADD_FRAME(5, 05) +#endif +#if (ZSTD_LEGACY_SUPPORT <= 6) + ADD_FRAME(6, 06) +#endif +#if (ZSTD_LEGACY_SUPPORT <= 7) + ADD_FRAME(7, 07) +#endif + ADD_FRAME(8, 08) +#undef ADD_FRAME + + *cSize = pos; + return nbFrames; +} + + +static int testSimpleAPI(const char* compressed, size_t compressedSize, + const char* expected, size_t expectedSize) +{ + char* const output = malloc(expectedSize); if (!output) { DISPLAY("ERROR: Not enough memory!\n"); @@ -48,7 +127,7 @@ static int testSimpleAPI(void) } { - size_t const ret = ZSTD_decompress(output, size, COMPRESSED, COMPRESSED_SIZE); + size_t const ret = ZSTD_decompress(output, expectedSize, compressed, compressedSize); if (ZSTD_isError(ret)) { if (ret == ZSTD_error_prefix_unknown) { DISPLAY("ERROR: Invalid frame magic number, was this compiled " @@ -59,11 +138,13 @@ static int testSimpleAPI(void) free(output); return 1; } - if (ret != size) { + if (ret != expectedSize) { DISPLAY("ERROR: Wrong decoded size\n"); + free(output); + return 1; } } - if (memcmp(EXPECTED, output, size) != 0) { + if (memcmp(expected, output, expectedSize) != 0) { DISPLAY("ERROR: Wrong decoded output produced\n"); free(output); return 1; @@ -75,16 +156,20 @@ static int testSimpleAPI(void) } -static int testStreamingAPI(void) +static int testStreamingAPI(const char* compressed, size_t compressedSize, + const char* expected, size_t expectedSize) { int error_code = 0; size_t const outBuffSize = ZSTD_DStreamOutSize(); char* const outBuff = malloc(outBuffSize); ZSTD_DStream* const stream = ZSTD_createDStream(); - ZSTD_inBuffer input = { COMPRESSED, COMPRESSED_SIZE, 0 }; + ZSTD_inBuffer input = { NULL, 0, 0 }; size_t outputPos = 0; int needsInit = 1; + input.src = compressed; + input.size = compressedSize; + if (outBuff == NULL) { DISPLAY("ERROR: Could not allocate memory\n"); return 1; @@ -116,7 +201,12 @@ static int testStreamingAPI(void) needsInit = 1; } } - if (memcmp(outBuff, EXPECTED + outputPos, output.pos) != 0) { + if (outputPos + output.pos > expectedSize) { + DISPLAY("ERROR: Decoded more than expected\n"); + error_code = 1; + break; + } + if (memcmp(outBuff, expected + outputPos, output.pos) != 0) { DISPLAY("ERROR: Wrong decoded output produced\n"); error_code = 1; break; @@ -127,20 +217,26 @@ static int testStreamingAPI(void) } } + if (error_code == 0 && outputPos != expectedSize) { + DISPLAY("ERROR: Decoded less than expected\n"); + error_code = 1; + } + free(outBuff); ZSTD_freeDStream(stream); if (error_code == 0) DISPLAY("Streaming API OK\n"); return error_code; } -static int testFrameDecoding(void) +static int testFrameDecoding(const char* compressed, size_t compressedSize, + size_t expectedSize) { - if (strlen(EXPECTED) > ZSTD_decompressBound(COMPRESSED, COMPRESSED_SIZE)) { + if (expectedSize > ZSTD_decompressBound(compressed, compressedSize)) { DISPLAY("ERROR: ZSTD_decompressBound: decompressed bound too small\n"); return 1; } - { const char* ip = COMPRESSED; - size_t remainingSize = COMPRESSED_SIZE; + { const char* ip = compressed; + size_t remainingSize = compressedSize; while (1) { size_t frameSize = ZSTD_findFrameCompressedSize(ip, remainingSize); if (ZSTD_isError(frameSize)) { @@ -162,25 +258,99 @@ static int testFrameDecoding(void) int main(void) { - { int const ret = testSimpleAPI(); - if (ret) return ret; } - { int const ret = testStreamingAPI(); - if (ret) return ret; } - { int const ret = testFrameDecoding(); - if (ret) return ret; } - - DISPLAY("OK\n"); - return 0; + size_t const blockSize = strlen(BLOCK); + char* const compressed = malloc(ALL_FRAMES_SIZE); + /* the streaming API can only replay the subset of frames for which a + * legacy streaming decoder exists : it gets a buffer of its own, so that + * each buffer keeps matching the size recorded next to it */ + char* const streamable = malloc(ALL_FRAMES_SIZE); + char* expected = NULL; + size_t compressedSize = 0; + size_t streamableSize = 0; + size_t expectedSize = 0; + unsigned nbFrames = 0; + unsigned nbStreamFrames = 0; + int result = 1; + + do { + if (compressed == NULL || streamable == NULL) { + DISPLAY("ERROR: Not enough memory!\n"); + break; + } + nbFrames = buildTestFrames(compressed, &compressedSize, 1); + nbStreamFrames = buildTestFrames(streamable, &streamableSize, OLDEST_STREAMABLE_VERSION); + DISPLAY("Testing %u frames, from format v0.%i upward\n", nbFrames, ZSTD_LEGACY_SUPPORT); + + expectedSize = blockSize * nbFrames; + expected = malloc(expectedSize); + if (expected == NULL) { + DISPLAY("ERROR: Not enough memory!\n"); + break; + } + { unsigned u; + for (u = 0; u < nbFrames; u++) + memcpy(expected + (u * blockSize), BLOCK, blockSize); + } + + if (testSimpleAPI(compressed, compressedSize, expected, expectedSize)) break; + if (testFrameDecoding(compressed, compressedSize, expectedSize)) break; + /* all blocks being identical, the first @nbStreamFrames ones + * are what the streaming subset must decode to */ + if (testStreamingAPI(streamable, streamableSize, + expected, blockSize * nbStreamFrames)) break; + result = 0; + } while (0); + + free(expected); + free(streamable); + free(compressed); + if (result == 0) DISPLAY("OK\n"); + return result; } -/* Consists of the "EXPECTED" string compressed with default settings on - - v0.4.3 - - v0.5.0 - - v0.6.0 - - v0.7.0 - - v0.8.0 -*/ -const char* const COMPRESSED = +const char* const FRAME_V01 = + "\xFD\x2F\xB5\x1E\x00\x00\xB3\x00\x00\x9D\x00\xD8\x1F\xB0\x01\x10" + "\x00\x00\x00\xA0\xCA\x4C\xCF\x43\xDF\xF9\xF8\x7B\x7F\xFE\x3F\xDF" + "\xDD\xDF\xC3\x86\x55\x63\x3A\x68\x70\x44\x44\x08\x1F\x00\x1D\x00" + "\x1D\x00\x44\x21\x80\x32\xAD\x75\x6F\xB1\x6B\x3B\x7A\x3F\x77\xAE" + "\x8F\x95\x4A\x73\xEC\xCF\x73\x0C\x07\x9A\x9E\xF0\x5E\xB4\xCB\x94" + "\x06\x99\xE6\x3E\x28\xF0\x80\x42\xA9\x1A\x2B\x0E\xDF\xC9\x06\x82" + "\xB6\x56\x72\x98\x49\xB2\xBE\xEE\xCD\x25\xA1\xAC\x86\x01\x99\xD4" + "\x28\x05\x9F\xC8\xF9\x1D\x90\x30\x6B\x18\x08\x9B\xA0\xF5\x12\xC9" + "\x5F\x4B\x80\x37\xAD\x14\x5A\xD2\xCD\x98\x01\xE8\x77\x03\x63\x08" + "\xB4\xA4\x81\xD8\xE9\x24\xB2\x73\x9F\x9C\x03\x07\x52\x6A\x2F\xC2" + "\xCE\x70\x1B\xFF\xD0\x4E\x01\x04\x00\x54\x01\x43\x80\x07\x00\xA1" + "\x28\x3F\x20\x9C\x6A\x30\xA3\x01\xC8\x12\xC0\x00\x00"; + +const char* const FRAME_V02 = + "\x22\xB5\x2F\xFD\x00\x00\xB1\x60\x03\x20\x13\x00\x1F\xB0\x01\x10" + "\x00\x00\x00\xA0\xCA\x4C\xCF\x43\xDF\xF9\xF8\x7B\x7F\xFE\x3F\xDF" + "\xDD\xDF\xC3\x86\x55\x63\x3A\x68\x70\x44\x44\x08\x19\x00\x1D\x00" + "\x1E\x00\xAB\x65\x8D\x76\xE9\x32\x5A\x7D\xA3\x19\xED\xF6\x94\xB8" + "\x12\xEC\x2E\xA3\xB9\x6C\x35\xA3\x55\x4A\xD4\xC9\x68\x56\x3B\x79" + "\xDB\x40\xEA\x07\x52\x23\x99\x5C\x3E\xB2\xB6\xB3\x6B\xCE\x96\x18" + "\x2A\xF3\x20\x00\x0A\xED\xED\x36\xB5\xB1\x2E\x72\xBA\x1D\xCC\x19" + "\xED\x12\x51\x9C\x63\x0E\xE7\x5E\xC8\xB8\xCD\xF0\x39\x84\xC4\x6E" + "\x2F\x27\x46\x8D\x90\x0C\x44\x21\x80\x32\xAD\x28\xB3\x4E\x79\xAB" + "\x79\x8D\xDB\xCB\x85\x41\xC1\xC1\x9A\x8B\x35\xA3\x11\xE4\xAB\x48" + "\xC0\xE5\x02\xCA\x0C\x04\x00\x54\x01\x43\x80\x07\x00\xA1\x28\x3F" + "\x20\x9C\x6A\x30\xA3\x01\xC8\x12\xC0\x00\x00"; + +const char* const FRAME_V03 = + "\x23\xB5\x2F\xFD\x00\x00\xB1\x50\x03\xE0\x12\x00\x1F\xB0\x01\x10" + "\x00\x00\x00\xA0\xCA\x4C\xCF\x43\xDF\xF9\xF8\x7B\x7F\xFE\x3F\xDF" + "\xDD\xDF\xC3\x86\x55\x63\x3A\x68\x70\x44\x44\x08\x19\x00\x1D\x00" + "\x1D\x00\x6A\x59\xA3\x5D\xBA\x8C\x56\xDF\x68\x46\xBB\x3D\x25\xAE" + "\x04\xBB\xCB\x68\x2E\x5B\xCD\x68\x95\x12\x35\x48\x26\xA3\x59\xED" + "\xE4\x6D\x03\xA9\x1F\x48\x8D\x64\x72\xF9\xC8\xDA\xCE\xAE\x39\x5B" + "\x62\xA8\xCC\x83\x00\x28\xB4\x0F\x59\x1B\xEB\x22\xA7\xDB\xC1\x9C" + "\xD1\x2E\x11\xC5\x39\xE6\x70\xEE\x85\x8C\xDB\x0C\x9F\x43\x48\xEC" + "\xF6\x72\x62\xD4\x28\x44\x21\x80\x32\xAD\x28\xB3\x4E\x79\xAB\x79" + "\x8D\xDB\xCB\x85\x41\xC1\xC1\x9A\x8B\x35\xA3\x11\xE4\xAB\x48\xC0" + "\xE5\x02\xCA\x05\x00\x54\x01\x43\x80\x07\x00\xA1\x28\x3F\xE0\x04" + "\x80\x00\xA0\xC1\x8C\x06\x20\x4B\xC0\x00\x00"; + +const char* const FRAME_V04 = "\x24\xB5\x2F\xFD\x00\x00\x00\xBB\xB0\x02\xC0\x10\x00\x1E\xB0\x01" "\x02\x00\x00\x80\x00\xE8\x92\x34\x12\x97\xC8\xDF\xE9\xF3\xEF\x53" "\xEA\x1D\x27\x4F\x0C\x44\x90\x0C\x8D\xF1\xB4\x89\x17\x00\x18\x00" @@ -193,29 +363,37 @@ const char* const COMPRESSED = "\xC6\xBA\x01\x0E\x00\x54\x00\x00\x19\x00\x00\x54\x14\x00\x24\x24" "\x04\xFE\x04\x84\x4E\x41\x00\x27\xE2\x02\xC4\xB1\x00\xD2\x51\x00" "\x79\x58\x41\x28\x00\xE0\x0C\x01\x68\x65\x00\x04\x13\x0C\xDA\x0C" - "\x80\x22\x06\xC0\x00\x00\x25\xB5\x2F\xFD\x00\x00\x00\xAD\x12\xB0" - "\x7D\x1E\xB0\x01\x02\x00\x00\x80\x00\xE8\x92\x34\x12\x97\xC8\xDF" - "\xE9\xF3\xEF\x53\xEA\x1D\x27\x4F\x0C\x44\x90\x0C\x8D\xF1\xB4\x89" - "\x03\x01\x50\x67\x56\xF5\x9F\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC" - "\xAB\xAB\xE0\xE2\x81\xFA\xCF\xC6\xBA\xEB\xA8\xD1\x40\x39\x90\x4C" - "\x64\xF8\xEB\x53\xE6\x18\x0B\x67\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A" - "\xF9\x63\x0C\xB8\xFA\x58\xE7\xF5\xE6\xE2\xE3\x67\xCC\x5D\x94\xC6" - "\x56\xDC\x7F\x0C\x84\x4B\xA8\xF8\x63\x2E\x3E\x4E\x67\xCD\x9E\xAC" - "\x04\x1E\x17\x27\xE4\x43\xF6\xA4\x56\xE4\x84\xC7\x9E\x34\x0E\x00" - "\x00\x32\x40\x80\xA8\x00\x01\x49\x81\xE0\x3C\x01\x29\x1D\x00\x87" - "\xCE\x80\x75\x08\x80\x72\x24\x00\x7B\x52\x00\x94\x00\x20\xCC\x01" - "\x86\xD2\x00\x81\x09\x83\xC1\x34\xA0\x88\x01\xC0\x00\x00\x26\xB5" - "\x2F\xFD\x42\xEF\x00\x00\xA6\x12\xB0\x7D\x1E\xB0\x01\x02\x00\x00" - "\x54\xA0\xBA\x24\x8D\xC4\x25\xF2\x77\xFA\xFC\xFB\x94\x7A\xC7\xC9" - "\x13\x03\x11\x24\x43\x63\x3C\x6D\x22\x03\x01\x50\x67\x56\xF5\x9F" - "\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC\xAB\xAB\xE0\xE2\x81\xFA\xCF" - "\xC6\xBA\xEB\xA8\xD1\x40\x39\x90\x4C\x64\xF8\xEB\x53\xE6\x18\x0B" - "\x67\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A\xF9\x63\x0C\xB8\xFA\x58\xE7" - "\xF5\xE6\xE2\xE3\x67\xCC\x5D\x94\xC6\x56\xDC\x7F\x0C\x84\x4B\xA8" - "\xF8\x63\x2E\x3E\x4E\x67\xCD\x9E\xAC\x04\x1E\x17\x27\xE4\x43\xF6" - "\xA4\x56\xE4\x84\xC7\x9E\x34\x0E\x00\x35\x0B\x71\xB5\xC0\x2A\x5C" - "\x26\x94\x22\x20\x8B\x4C\x8D\x13\x47\x58\x67\x15\x6C\xF1\x1C\x4B" - "\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D\x01\xC0\x00\x00" + "\x80\x22\x06\xC0\x00\x00"; + +const char* const FRAME_V05 = + "\x25\xB5\x2F\xFD\x00\x00\x00\xAD\x12\xB0\x7D\x1E\xB0\x01\x02\x00" + "\x00\x80\x00\xE8\x92\x34\x12\x97\xC8\xDF\xE9\xF3\xEF\x53\xEA\x1D" + "\x27\x4F\x0C\x44\x90\x0C\x8D\xF1\xB4\x89\x03\x01\x50\x67\x56\xF5" + "\x9F\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC\xAB\xAB\xE0\xE2\x81\xFA" + "\xCF\xC6\xBA\xEB\xA8\xD1\x40\x39\x90\x4C\x64\xF8\xEB\x53\xE6\x18" + "\x0B\x67\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A\xF9\x63\x0C\xB8\xFA\x58" + "\xE7\xF5\xE6\xE2\xE3\x67\xCC\x5D\x94\xC6\x56\xDC\x7F\x0C\x84\x4B" + "\xA8\xF8\x63\x2E\x3E\x4E\x67\xCD\x9E\xAC\x04\x1E\x17\x27\xE4\x43" + "\xF6\xA4\x56\xE4\x84\xC7\x9E\x34\x0E\x00\x00\x32\x40\x80\xA8\x00" + "\x01\x49\x81\xE0\x3C\x01\x29\x1D\x00\x87\xCE\x80\x75\x08\x80\x72" + "\x24\x00\x7B\x52\x00\x94\x00\x20\xCC\x01\x86\xD2\x00\x81\x09\x83" + "\xC1\x34\xA0\x88\x01\xC0\x00\x00"; + +const char* const FRAME_V06 = + "\x26\xB5\x2F\xFD\x42\xEF\x00\x00\xA6\x12\xB0\x7D\x1E\xB0\x01\x02" + "\x00\x00\x54\xA0\xBA\x24\x8D\xC4\x25\xF2\x77\xFA\xFC\xFB\x94\x7A" + "\xC7\xC9\x13\x03\x11\x24\x43\x63\x3C\x6D\x22\x03\x01\x50\x67\x56" + "\xF5\x9F\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC\xAB\xAB\xE0\xE2\x81" + "\xFA\xCF\xC6\xBA\xEB\xA8\xD1\x40\x39\x90\x4C\x64\xF8\xEB\x53\xE6" + "\x18\x0B\x67\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A\xF9\x63\x0C\xB8\xFA" + "\x58\xE7\xF5\xE6\xE2\xE3\x67\xCC\x5D\x94\xC6\x56\xDC\x7F\x0C\x84" + "\x4B\xA8\xF8\x63\x2E\x3E\x4E\x67\xCD\x9E\xAC\x04\x1E\x17\x27\xE4" + "\x43\xF6\xA4\x56\xE4\x84\xC7\x9E\x34\x0E\x00\x35\x0B\x71\xB5\xC0" + "\x2A\x5C\x26\x94\x22\x20\x8B\x4C\x8D\x13\x47\x58\x67\x15\x6C\xF1" + "\x1C\x4B\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D\x01\xC0" + "\x00\x00"; + +const char* const FRAME_V07 = "\x27\xB5\x2F\xFD\x20\xEF\x00\x00\xA6\x12\xE4\x84\x1F\xB0\x01\x10" "\x00\x00\x00\x35\x59\xA6\xE7\xA1\xEF\x7C\xFC\xBD\x3F\xFF\x9F\xEF" "\xEE\xEF\x61\xC3\xAA\x31\x1D\x34\x38\x22\x22\x04\x44\x21\x80\x32" @@ -227,36 +405,24 @@ const char* const COMPRESSED = "\x7B\x4A\x5C\x09\x76\x97\xD1\x5C\x96\x12\x75\x35\xA3\x55\x4A\xD4" "\x0B\x00\x35\x0B\x71\xB5\xC0\x2A\x5C\xE6\x08\x45\xF1\x39\x43\xF1" "\x1C\x4B\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D\x01\xC0" - "\x00\x00\x28\xB5\x2F\xFD\x24\xEF\x35\x05\x00\x92\x0B\x21\x1F\xB0" - "\x01\x10\x00\x00\x00\x35\x59\xA6\xE7\xA1\xEF\x7C\xFC\xBD\x3F\xFF" - "\x9F\xEF\xEE\xEF\x61\xC3\xAA\x31\x1D\x34\x38\x22\x22\x04\x44\x21" - "\x80\x32\xAD\x28\xF3\xD6\x28\x0C\x0A\x0E\xD6\x5C\xAC\x19\x8D\x20" - "\x5F\x45\x02\x2E\x17\x50\x66\x6D\xAC\x8B\x9C\x6E\x07\x73\x46\xBB" - "\x44\x14\xE7\x98\xC3\xB9\x17\x32\x6E\x33\x7C\x0E\x21\xB1\xDB\xCB" - "\x89\x51\x23\x34\xAB\x9D\xBC\x6D\x20\xF5\x03\xA9\x91\x4C\x2E\x1F" - "\x59\xDB\xD9\x35\x67\x4B\x0C\x95\x79\x10\x00\x85\xA6\x96\x95\x2E" - "\xDF\x78\x7B\x4A\x5C\x09\x76\x97\xD1\x5C\x96\x12\x75\x35\xA3\x55" - "\x4A\xD4\x0B\x00\x35\x0B\x71\xB5\xC0\x2A\x5C\xE6\x08\x45\xF1\x39" - "\x43\xF1\x1C\x4B\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D" - "\x01\xD2\x2F\x21\x80"; - -const char* const EXPECTED = - "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" - "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" - "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" - - "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" - "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" - "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" + "\x00\x00"; - "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" - "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" - "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" +const char* const FRAME_V08 = + "\x28\xB5\x2F\xFD\x24\xEF\x35\x05\x00\x92\x0B\x21\x1F\xB0\x01\x10" + "\x00\x00\x00\x35\x59\xA6\xE7\xA1\xEF\x7C\xFC\xBD\x3F\xFF\x9F\xEF" + "\xEE\xEF\x61\xC3\xAA\x31\x1D\x34\x38\x22\x22\x04\x44\x21\x80\x32" + "\xAD\x28\xF3\xD6\x28\x0C\x0A\x0E\xD6\x5C\xAC\x19\x8D\x20\x5F\x45" + "\x02\x2E\x17\x50\x66\x6D\xAC\x8B\x9C\x6E\x07\x73\x46\xBB\x44\x14" + "\xE7\x98\xC3\xB9\x17\x32\x6E\x33\x7C\x0E\x21\xB1\xDB\xCB\x89\x51" + "\x23\x34\xAB\x9D\xBC\x6D\x20\xF5\x03\xA9\x91\x4C\x2E\x1F\x59\xDB" + "\xD9\x35\x67\x4B\x0C\x95\x79\x10\x00\x85\xA6\x96\x95\x2E\xDF\x78" + "\x7B\x4A\x5C\x09\x76\x97\xD1\x5C\x96\x12\x75\x35\xA3\x55\x4A\xD4" + "\x0B\x00\x35\x0B\x71\xB5\xC0\x2A\x5C\xE6\x08\x45\xF1\x39\x43\xF1" + "\x1C\x4B\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D\x01\xD2" + "\x2F\x21\x80"; - "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" - "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" - "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" +const char* const BLOCK = "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n"; From fd27dfb97b0d63e7d381d7980f1c64c2fd6fd60c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 10 Sep 2026 16:20:58 -0700 Subject: [PATCH 5/6] ci: run the legacy decoders under ASan + UBSan Nothing in CI was running lib/legacy under a sanitizer since the default moved to ZSTD_LEGACY_SUPPORT=0. Add a job for it, next to the other sanitizer jobs. Also pass -Dlegacy_level=1 to the meson job : with the new default of 0, meson was silently dropping test-legacy instead of reporting anything. --- .github/workflows/dev-long-tests.yml | 10 ++++++++++ .github/workflows/dev-short-tests.yml | 1 + 2 files changed, 11 insertions(+) diff --git a/.github/workflows/dev-long-tests.yml b/.github/workflows/dev-long-tests.yml index 55eb0c5d7a9..4b0f6c2acee 100644 --- a/.github/workflows/dev-long-tests.yml +++ b/.github/workflows/dev-long-tests.yml @@ -160,6 +160,16 @@ jobs: - name: clang + ASan + UBSan + Fuzz Test run: CC=clang FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest + # the only job decoding real legacy frames under sanitizers. + # fuzzers do build these decoders, but only reach them if an input + # happens to carry a legacy magic number. + clang-asan-ubsan-legacy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # tag=v7.0.1 + - name: clang + ASan + UBSan + legacy decoders + run: CC=clang make uasan-test-legacy + gcc-asan-ubsan-fuzz32: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/dev-short-tests.yml b/.github/workflows/dev-short-tests.yml index acd32363992..45c7a702be7 100644 --- a/.github/workflows/dev-short-tests.yml +++ b/.github/workflows/dev-short-tests.yml @@ -237,6 +237,7 @@ jobs: -Dbin_tests=true \ -Dbin_contrib=true \ -Ddefault_library=both \ + -Dlegacy_level=1 \ build/meson mesonBuild ninja -C mesonBuild/ meson test -C mesonBuild/ --print-errorlogs From 96c4d2529b10f6b22423250e46fcbfcff930a485 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Thu, 10 Sep 2026 16:30:15 -0700 Subject: [PATCH 6/6] buck: stop compiling unreachable legacy decoders :decompress pulled in :legacy, which globs all 7 decoders, while ZSTD_LEGACY_SUPPORT is 0. Both includes of zstd_legacy.h, in zstd_decompress.c and zstd_ddict.c, are guarded on that macro, so :decompress needs neither the sources nor the headers at this level. Drop the dependency, mirroring what tests/Makefile now does. Also drop the -DZSTD_LEGACY_SUPPORT=0 exported by :legacy. It is redundant with .buckconfig, and it would fight it : anyone raising the level there would have :legacy silently force 0 back onto its dependents, which is the opposite of what depending on that target is for. --- lib/BUCK | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/BUCK b/lib/BUCK index 70414b1d379..5d10cfa10db 100644 --- a/lib/BUCK +++ b/lib/BUCK @@ -32,7 +32,6 @@ cxx_library( srcs=glob(['decompress/zstd*.c']), deps=[ ':common', - ':legacy', ], ) @@ -47,6 +46,14 @@ cxx_library( deps=[':common'], ) +# Decoders for formats older than v0.8. Not a dependency of :decompress, +# because ZSTD_LEGACY_SUPPORT is 0 (see .buckconfig), which makes them +# unreachable. +# Enabling legacy formats means two edits, which only make sense together : +# raise the level in .buckconfig, AND add ':legacy' to :decompress deps. +# Raising the level alone fails to link, which is easy to notice. Adding the +# dep alone is silent : these decoders get compiled back in, while +# ZSTD_isLegacy() remains compiled out, so legacy frames are still rejected. cxx_library( name='legacy', header_namespace='', @@ -56,9 +63,6 @@ cxx_library( ]), srcs=glob(['legacy/*.c']), deps=[':common'], - exported_preprocessor_flags=[ - '-DZSTD_LEGACY_SUPPORT=0', - ], ) cxx_library(