Fix unintended high memory usage

lib/common/allocations.h

@@ -33,9 +33,13 @@ MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
 MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
 {
     if (customMem.customAlloc) {
-        /* calloc implemented as malloc+memset;
-         * not as efficient as calloc, but next best guess for custom malloc */
+        /* calloc implemented as malloc+memset */
         void* const ptr = customMem.customAlloc(customMem.opaque, size);
+
+        if (ptr == NULL) {
+            return NULL;
+        }
+
         ZSTD_memset(ptr, 0, size);
         return ptr;
     }

lib/compress/zstd_ldm.c

@@ -152,7 +152,11 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params,
         }
     }
     if (params->hashLog == 0) {
-        params->hashLog = BOUNDED(ZSTD_HASHLOG_MIN, params->windowLog - params->hashRateLog, ZSTD_HASHLOG_MAX);
+        if (params->windowLog > params->hashLog) {
+            params->hashLog = ZSTD_HASHLOG_MIN;
+        } else {
+            params->hashLog = BOUNDED(ZSTD_HASHLOG_MIN, params->windowLog - params->hashRateLog, ZSTD_HASHLOG_MAX);
+        }
     }
     if (params->minMatchLength == 0) {
         params->minMatchLength = LDM_MIN_MATCH_LENGTH;

tests/fuzzer.c

@@ -1414,6 +1414,37 @@ static int basicUnitTests(U32 const seed, double compressibility)
     }
     DISPLAYLEVEL(3, "OK \n");
 
+    DISPLAYLEVEL(3, "test%3i : ldm hashRateLog > windowLog underflow check : ", testNb++);
+    {
+        /* Test that when windowLog < hashRateLog, we don't get excessive memory usage
+         * due to underflow in hashLog calculation (windowLog - hashRateLog). */
+        ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+
+        size_t const size = (1U << 10); // 1 KB
+        size_t const dstCapacity = ZSTD_compressBound(size);
+        void* src = (void*)malloc(size);
+        void* dst = (void*)malloc(dstCapacity);
+
+        RDG_genBuffer(src, size, 0.5, 0.5, seed);
+
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 0)); 
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 12));
+        CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 13));
+
+        CHECK_Z(ZSTD_compress2(cctx, dst, dstCapacity, src, size));
+
+        {   size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
+            DISPLAYLEVEL(3, "CCtx size: %u bytes ", (unsigned)cctxSize);
+            CHECK_LT(cctxSize, 50 MB);
+        }
+
+        ZSTD_freeCCtx(cctx);
+        free(src);
+        free(dst);
+    }
+    DISPLAYLEVEL(3, "OK \n");
+
     DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++);
     {
         ZSTD_CCtx* const cctx = ZSTD_createCCtx();