Merge pull request #1564 from lz4/filecompress_fix

fixed minor static analyzer warning in fileCompress example

examples/fileCompress.c

@@ -28,7 +28,7 @@ static size_t get_file_size(char *filename)
         return 0;
     }
 
-    return statbuf.st_size;
+    return (size_t)statbuf.st_size;
 }
 
 static int compress_file(FILE* f_in, FILE* f_out)
@@ -36,7 +36,6 @@ static int compress_file(FILE* f_in, FILE* f_out)
     assert(f_in != NULL); assert(f_out != NULL);
 
     LZ4F_errorCode_t ret = LZ4F_OK_NoError;
-    size_t len;
     LZ4_writeFile_t* lz4fWrite;
     void* const buf = malloc(CHUNK_SIZE);
     if (!buf) {
@@ -56,7 +55,7 @@ static int compress_file(FILE* f_in, FILE* f_out)
     }
 
     while (1) {
-        len = fread(buf, 1, CHUNK_SIZE, f_in);
+        size_t len = fread(buf, 1, CHUNK_SIZE, f_in);
 
         if (ferror(f_in)) {
             printf("fread error\n");
@@ -94,6 +93,7 @@ static int decompress_file(FILE* f_in, FILE* f_out)
     void* const buf= malloc(CHUNK_SIZE);
     if (!buf) {
         printf("error: memory allocation failed \n");
+        return 1;
     }
 
     ret = LZ4F_readOpen(&lz4fRead, f_in);
@@ -135,9 +135,11 @@ out:
     return 0;
 }
 
+/* @return 0 if both FILE* have identical content */
 int compareFiles(FILE* fp0, FILE* fp1)
 {
     int result = 0;
+    assert(fp0 != NULL); assert(fp1 != NULL);
 
     while (result==0) {
         char b0[1024];
@@ -147,7 +149,7 @@ int compareFiles(FILE* fp0, FILE* fp1)
 
         result = (r0 != r1);
         if (!r0 || !r1) break;
-        if (!result) result = memcmp(b0, b1, r0);
+        if (r0 == r1) result = memcmp(b0, b1, r0);
     }
 
     return result;
@@ -175,7 +177,7 @@ int main(int argc, const char **argv) {
     {   FILE* const inpFp = fopen(inpFilename, "rb");
         FILE* const outFp = fopen(lz4Filename, "wb");
         printf("compress : %s -> %s\n", inpFilename, lz4Filename);
-        LZ4F_errorCode_t ret = compress_file(inpFp, outFp);
+        int ret = compress_file(inpFp, outFp);
         fclose(inpFp);
         fclose(outFp);
 
@@ -188,7 +190,7 @@ int main(int argc, const char **argv) {
             inpFilename,
             get_file_size(inpFilename),
             get_file_size(lz4Filename), /* might overflow is size_t is 32 bits and size_{in,out} > 4 GB */
-            (double)get_file_size(lz4Filename) / get_file_size(inpFilename) * 100);
+            (double)get_file_size(lz4Filename) / (double)get_file_size(inpFilename) * 100);
 
         printf("compress : done\n");
     }
@@ -198,8 +200,11 @@ int main(int argc, const char **argv) {
         FILE* const inpFp = fopen(lz4Filename, "rb");
         FILE* const outFp = fopen(decFilename, "wb");
 
+        if (inpFp == NULL) exit(1);
+        if (outFp == NULL) exit(1);
+
         printf("decompress : %s -> %s\n", lz4Filename, decFilename);
-        LZ4F_errorCode_t ret = decompress_file(inpFp, outFp);
+        int ret = decompress_file(inpFp, outFp);
 
         fclose(outFp);
         fclose(inpFp);
@@ -216,6 +221,9 @@ int main(int argc, const char **argv) {
     {   FILE* const inpFp = fopen(inpFilename, "rb");
         FILE* const decFp = fopen(decFilename, "rb");
 
+        if (inpFp == NULL) exit(1);
+        if (decFp == NULL) exit(1);
+
         printf("verify : %s <-> %s\n", inpFilename, decFilename);
         int const cmp = compareFiles(inpFp, decFp);
 

lib/lz4.h

@@ -258,10 +258,12 @@ LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* d
  * @return : Nb bytes written into 'dst' (necessarily <= dstCapacity)
  *           or 0 if compression fails.
  *
- * Note : from v1.8.2 to v1.9.1, this function had a bug (fixed in v1.9.2+):
- *        the produced compressed content could, in specific circumstances,
- *        require to be decompressed into a destination buffer larger
- *        by at least 1 byte than the content to decompress.
+ * Note : 'targetDstSize' must be >= 1, because it's the smallest valid lz4 payload.
+ *
+ * Note 2:from v1.8.2 to v1.9.1, this function had a bug (fixed in v1.9.2+):
+ *        the produced compressed content could, in rare circumstances,
+ *        require to be decompressed into a destination buffer
+ *        larger by at least 1 byte than decompressesSize.
  *        If an application uses `LZ4_compress_destSize()`,
  *        it's highly recommended to update liblz4 to v1.9.2 or better.
  *        If this can't be done or ensured,

lib/lz4frame.c

@@ -155,7 +155,7 @@ static void LZ4F_free(void* p, LZ4F_CustomMem cmem)
 static int g_debuglog_enable = 1;
 #  define DEBUGLOG(l, ...) {                                  \
                 if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) {  \
-                    fprintf(stderr, __FILE__ " (%i): ", __LINE__ );  \
+                    fprintf(stderr, __FILE__ " %i: ", __LINE__ );  \
                     fprintf(stderr, __VA_ARGS__);             \
                     fprintf(stderr, " \n");                   \
             }   }
@@ -700,6 +700,7 @@ size_t LZ4F_compressBegin_internal(LZ4F_cctx* cctx,
     RETURN_ERROR_IF(dstCapacity < maxFHSize, dstMaxSize_tooSmall);
     if (preferencesPtr == NULL) preferencesPtr = &prefNull;
     cctx->prefs = *preferencesPtr;
+    DEBUGLOG(5, "LZ4F_compressBegin_internal: Independent_blocks=%u", cctx->prefs.frameInfo.blockMode);
 
     /* cctx Management */
     {   U16 const ctxTypeID = (cctx->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;
@@ -887,10 +888,11 @@ static size_t LZ4F_makeBlock(void* dst,
                              LZ4F_blockChecksum_t crcFlag)
 {
     BYTE* const cSizePtr = (BYTE*)dst;
+    int dstCapacity = (srcSize > 1) ? (int)srcSize - 1 : 1;
     U32 cSize;
     assert(compress != NULL);
     cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
-                          (int)(srcSize), (int)(srcSize-1),
+                          (int)srcSize, dstCapacity,
                           level, cdict);
 
     if (cSize == 0 || cSize >= srcSize) {
@@ -961,12 +963,13 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev
     return LZ4F_compressBlockHC_continue;
 }
 
-/* Save history (up to 64KB) into @tmpBuff */
-static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
+/* Save or shorten history (up to 64KB) into @tmpBuff */
+static void LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
 {
-    if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
-        return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
-    return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
+    int const dictSize = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ?
+                    LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB) :
+                    LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
+    cctxPtr->tmpIn = cctxPtr->tmpBuff + dictSize;
 }
 
 typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
@@ -974,7 +977,7 @@ typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
 static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } };
 
 
- /*! LZ4F_compressUpdateImpl() :
+/*! LZ4F_compressUpdateImpl() :
  *  LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
  *  When successful, the function always entirely consumes @srcBuffer.
  *  src data is either buffered or compressed into @dstBuffer.
@@ -1072,9 +1075,7 @@ static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
         if (compressOptionsPtr->stableSrc) {
             cctxPtr->tmpIn = cctxPtr->tmpBuff;  /* src is stable : dictionary remains in src across invocations */
         } else {
-            int const realDictSize = LZ4F_localSaveDict(cctxPtr);
-            assert(0 <= realDictSize && realDictSize <= 64 KB);
-            cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
+            LZ4F_localSaveDict(cctxPtr);
         }
     }
 
@@ -1084,8 +1085,7 @@ static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
     {
         /* only preserve 64KB within internal buffer. Ensures there is enough room for next block.
          * note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */
-        int const realDictSize = LZ4F_localSaveDict(cctxPtr);
-        cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
+        LZ4F_localSaveDict(cctxPtr);
         assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize));
     }
 
@@ -1150,7 +1150,7 @@ size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
 
 /*! LZ4F_flush() :
  *  When compressed data must be sent immediately, without waiting for a block to be filled,
- *  invoke LZ4_flush(), which will immediately compress any remaining data stored within LZ4F_cctx.
+ *  invoke LZ4F_flush(), which will immediately compress any remaining data stored within LZ4F_cctx.
  *  The result of the function is the number of bytes written into dstBuffer.
  *  It can be zero, this means there was no data left within LZ4F_cctx.
  *  The function outputs an error code if it fails (can be tested using LZ4F_isError())
@@ -1164,6 +1164,8 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
     BYTE* dstPtr = dstStart;
     compressFunc_t compress;
 
+    DEBUGLOG(5, "LZ4F_flush: %zu buffered bytes (saved dict size = %i) (dstCapacity=%u)",
+            cctxPtr->tmpInSize, (int)(cctxPtr->tmpIn - cctxPtr->tmpBuff), (unsigned)dstCapacity);
     if (cctxPtr->tmpInSize == 0) return 0;   /* nothing to flush */
     RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized);
     RETURN_ERROR_IF(dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize), dstMaxSize_tooSmall);
@@ -1185,9 +1187,9 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
     cctxPtr->tmpInSize = 0;
 
     /* keep tmpIn within limits */
-    if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) {  /* necessarily LZ4F_blockLinked */
-        int const realDictSize = LZ4F_localSaveDict(cctxPtr);
-        cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
+    if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) {
+        assert(cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked);
+        LZ4F_localSaveDict(cctxPtr);
     }
 
     return (size_t)(dstPtr - dstStart);

lib/lz4hc.c

@@ -262,6 +262,30 @@ static void LZ4HC_init_internal (LZ4HC_CCtx_internal* hc4, const BYTE* start)
 /**************************************
 *  Encode
 **************************************/
+#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 2)
+# define RAWLOG(...) fprintf(stderr, __VA_ARGS__)
+void LZ4HC_hexOut(const void* src, size_t len)
+{
+    const BYTE* p = (const BYTE*)src;
+    size_t n;
+    for (n=0; n<len; n++) {
+        RAWLOG("%02X ", p[n]);
+    }
+    RAWLOG(" \n");
+}
+
+# define HEX_CMP(_lev, _ptr, _ref, _len) \
+    if (LZ4_DEBUG >= _lev) {            \
+        RAWLOG("match bytes: ");        \
+        LZ4HC_hexOut(_ptr, _len);       \
+        RAWLOG("ref bytes: ");          \
+        LZ4HC_hexOut(_ref, _len);       \
+    }
+
+#else
+# define HEX_CMP(l,p,r,_l)
+#endif
+
 /* LZ4HC_encodeSequence() :
  * @return : 0 if ok,
  *           1 if buffer issue detected */
@@ -278,47 +302,49 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
 #define op      (*_op)
 #define anchor  (*_anchor)
 
-    size_t length;
     BYTE* const token = op++;
 
 #if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 6)
     static const BYTE* start = NULL;
     static U32 totalCost = 0;
-    U32 const pos = (start==NULL) ? 0 : (U32)(anchor - start);
+    U32 const pos = (start==NULL) ? 0 : (U32)(anchor - start); /* only works for single segment */
     U32 const ll = (U32)(ip - anchor);
     U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0;
     U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0;
     U32 const cost = 1 + llAdd + ll + 2 + mlAdd;
     if (start==NULL) start = anchor;  /* only works for single segment */
-    /* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */
     DEBUGLOG(6, "pos:%7u -- literals:%4u, match:%4i, offset:%5i, cost:%4u + %5u",
                 pos,
                 (U32)(ip - anchor), matchLength, offset,
                 cost, totalCost);
+# if 1 /* only works on single segment data */
+    HEX_CMP(7, ip, ip-offset, matchLength);
+# endif
     totalCost += cost;
 #endif
 
     /* Encode Literal length */
-    length = (size_t)(ip - anchor);
-    LZ4_STATIC_ASSERT(notLimited == 0);
-    /* Check output limit */
-    if (limit && ((op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) {
-        DEBUGLOG(6, "Not enough room to write %i literals (%i bytes remaining)",
-                (int)length, (int)(oend - op));
-        return 1;
-    }
-    if (length >= RUN_MASK) {
-        size_t len = length - RUN_MASK;
-        *token = (RUN_MASK << ML_BITS);
-        for(; len >= 255 ; len -= 255) *op++ = 255;
-        *op++ = (BYTE)len;
-    } else {
-        *token = (BYTE)(length << ML_BITS);
-    }
+    {   size_t litLen = (size_t)(ip - anchor);
+        LZ4_STATIC_ASSERT(notLimited == 0);
+        /* Check output limit */
+        if (limit && ((op + (litLen / 255) + litLen + (2 + 1 + LASTLITERALS)) > oend)) {
+            DEBUGLOG(6, "Not enough room to write %i literals (%i bytes remaining)",
+                    (int)litLen, (int)(oend - op));
+            return 1;
+        }
+        if (litLen >= RUN_MASK) {
+            size_t len = litLen - RUN_MASK;
+            *token = (RUN_MASK << ML_BITS);
+            for(; len >= 255 ; len -= 255) *op++ = 255;
+            *op++ = (BYTE)len;
+        } else {
+            *token = (BYTE)(litLen << ML_BITS);
+        }
 
-    /* Copy Literals */
-    LZ4_wildCopy8(op, anchor, op + length);
-    op += length;
+        /* Copy Literals */
+        LZ4_wildCopy8(op, anchor, op + litLen);
+        op += litLen;
+    }
 
     /* Encode Offset */
     assert(offset <= LZ4_DISTANCE_MAX );
@@ -327,20 +353,20 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
 
     /* Encode MatchLength */
     assert(matchLength >= MINMATCH);
-    length = (size_t)matchLength - MINMATCH;
-    if (limit && (op + (length / 255) + (1 + LASTLITERALS) > oend)) {
-        DEBUGLOG(6, "Not enough room to write match length");
-        return 1;   /* Check output limit */
-    }
-    if (length >= ML_MASK) {
-        *token += ML_MASK;
-        length -= ML_MASK;
-        for(; length >= 510 ; length -= 510) { *op++ = 255; *op++ = 255; }
-        if (length >= 255) { length -= 255; *op++ = 255; }
-        *op++ = (BYTE)length;
-    } else {
-        *token += (BYTE)(length);
-    }
+    {   size_t mlCode = (size_t)matchLength - MINMATCH;
+        if (limit && (op + (mlCode / 255) + (1 + LASTLITERALS) > oend)) {
+            DEBUGLOG(6, "Not enough room to write match length");
+            return 1;   /* Check output limit */
+        }
+        if (mlCode >= ML_MASK) {
+            *token += ML_MASK;
+            mlCode -= ML_MASK;
+            for(; mlCode >= 510 ; mlCode -= 510) { *op++ = 255; *op++ = 255; }
+            if (mlCode >= 255) { mlCode -= 255; *op++ = 255; }
+            *op++ = (BYTE)mlCode;
+        } else {
+            *token += (BYTE)(mlCode);
+    }   }
 
     /* Prepare next loop */
     ip += matchLength;
@@ -944,6 +970,7 @@ LZ4HC_InsertAndGetWiderMatch (
                         offset = (int)(ipIndex - matchIndex);
                         sBack = back;
                         DEBUGLOG(7, "Found match of len=%i within prefix, offset=%i, back=%i", longest, offset, -back);
+                        HEX_CMP(7, ip + back, ip + back - offset, (size_t)matchLength);
             }   }   }
         } else {   /* lowestMatchIndex <= matchIndex < dictLimit : within Ext Dict */
             const BYTE* const matchPtr = dictStart + (matchIndex - dictIdx);
@@ -963,6 +990,7 @@ LZ4HC_InsertAndGetWiderMatch (
                     offset = (int)(ipIndex - matchIndex);
                     sBack = back;
                     DEBUGLOG(7, "Found match of len=%i within dict, offset=%i, back=%i", longest, offset, -back);
+                    HEX_CMP(7, ip + back, matchPtr + back, (size_t)matchLength);
         }   }   }
 
         if (chainSwap && matchLength==longest) {   /* better match => select a better chain */
@@ -1399,8 +1427,8 @@ LZ4HC_compress_generic_internal (
             const dictCtx_directive dict
             )
 {
-    DEBUGLOG(5, "LZ4HC_compress_generic_internal(src=%p, srcSize=%d)",
-                src, *srcSizePtr);
+    DEBUGLOG(5, "LZ4HC_compress_generic_internal(src=%p, srcSize=%d, dstCapacity=%d)",
+                src, *srcSizePtr, dstCapacity);
 
     /* input sanitization */
     if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0;  /* Unsupported input size (too large or negative) */

tests/frametest.c

@@ -75,6 +75,7 @@ static const U32 nbTestsDefault = 256 KB;
 static const U32 prime1 = 2654435761U;
 static const U32 prime2 = 2246822519U;
 
+static const U64 extCrcSeed = 1;
 
 /*-************************************
 *  Macros
@@ -298,7 +299,7 @@ static int unitTests(U32 seed, double compressibility)
         goto _output_error;
     }
     FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, randState);
-    crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
+    crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, extCrcSeed);
 
     /* LZ4F_compressBound() : special case : srcSize == 0 */
     DISPLAYLEVEL(3, "LZ4F_compressBound(0) = ");
@@ -371,7 +372,7 @@ static int unitTests(U32 seed, double compressibility)
 
         DISPLAYLEVEL(3, "Single Pass decompression : ");
         CHECK( LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL) );
-        { U64 const crcDest = XXH64(decodedBuffer, decodedBufferSize, 1);
+        { U64 const crcDest = XXH64(decodedBuffer, decodedBufferSize, extCrcSeed);
           if (crcDest != crcOrig) goto _output_error; }
         DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize);
 
@@ -398,7 +399,7 @@ static int unitTests(U32 seed, double compressibility)
             if (decResult != 0) goto _output_error;   /* should finish now */
             op += oSize;
             if (op>oend) { DISPLAY("decompression write overflow \n"); goto _output_error; }
-            {   U64 const crcDest = XXH64(decodedBuffer, (size_t)(op-ostart), 1);
+            {   U64 const crcDest = XXH64(decodedBuffer, (size_t)(op-ostart), extCrcSeed);
                 if (crcDest != crcOrig) goto _output_error;
         }   }
 
@@ -471,7 +472,7 @@ static int unitTests(U32 seed, double compressibility)
                 op += oSize;
                 ip += iSize;
             }
-            {   U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
+            {   U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, extCrcSeed);
                 if (crcDest != crcOrig) goto _output_error;
             }
             DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), (unsigned)COMPRESSIBLE_NOISE_LENGTH);
@@ -515,7 +516,7 @@ static int unitTests(U32 seed, double compressibility)
             ip += iSize;
         }
         {   size_t const decodedSize = (size_t)(op - ostart);
-            U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1);
+            U64 const crcDest = XXH64(decodedBuffer, decodedSize, extCrcSeed);
             if (crcDest != crcOrig) goto _output_error;
             DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
          }
@@ -571,8 +572,8 @@ static int unitTests(U32 seed, double compressibility)
         CHECK( LZ4F_decompress(dctx, decodedBuffer, &decodedSize, compressedBuffer, &iSize, NULL) );
         if (decodedSize != testSize) goto _output_error;
         if (iSize != cSize) goto _output_error;
-        {   U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1);
-            U64 const crcSrc = XXH64(CNBuffer, testSize, 1);
+        {   U64 const crcDest = XXH64(decodedBuffer, decodedSize, extCrcSeed);
+            U64 const crcSrc = XXH64(CNBuffer, testSize, extCrcSeed);
             if (crcDest != crcSrc) goto _output_error;
         }
         DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
@@ -994,15 +995,28 @@ static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, o_
 #undef CHECK
 #define CHECK(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__); } }
 
+static LZ4F_errorCode_t LZ4F_returnErrorCode(LZ4F_errorCodes code)
+{
+    return (LZ4F_errorCode_t)-(ptrdiff_t)code;
+}
+
+#define RETURN_ERROR(e) return LZ4F_returnErrorCode(LZ4F_ERROR_ ## e)
+
+
+/* @o_scenario: output behavior: contiguous, non-contiguous, or overwrite
+ * @crcOrig: xxh64 of original data using extCrcSeed
+ * @return : 0 on success, or LZ4F_ErrorCode
+ * */
 size_t test_lz4f_decompression_wBuffers(
           const void* cSrc, size_t cSize,
-                void* dst, size_t dstCapacity, o_scenario_e o_scenario,
+                void* dst, size_t dstCapacity,
+                o_scenario_e o_scenario,
           const void* srcRef, size_t decompressedSize,
                 U64 crcOrig,
                 U32* const randState,
                 LZ4F_dctx* const dCtx,
                 U32 seed, U32 testNb,
-                int findErrorPos)
+                int shouldSucceed)
 {
     const BYTE* ip = (const BYTE*)cSrc;
     const BYTE* const iend = ip + cSize;
@@ -1015,7 +1029,8 @@ size_t test_lz4f_decompression_wBuffers(
     size_t totalOut = 0;
     size_t moreToFlush = 0;
     XXH64_state_t xxh64;
-    XXH64_reset(&xxh64, 1);
+    XXH64_reset(&xxh64, extCrcSeed);
+    o_scenario = o_contiguous;
     assert(ip < iend);
     while (ip < iend) {
         unsigned const nbBitsI = (FUZ_rand(randState) % (maxBits-1)) + 1;
@@ -1047,7 +1062,7 @@ size_t test_lz4f_decompression_wBuffers(
             moreToFlush = LZ4F_decompress(dCtx, tmpop, &oSize, tmpip, &iSize, &dOptions);
             free(iBuffer);
         }
-        DISPLAYLEVEL(7, "oSize=%u,  readSize=%u \n", (unsigned)oSize, (unsigned)iSize);
+        DISPLAYLEVEL(7, "oSize=%u, readSize=%u \n", (unsigned)oSize, (unsigned)iSize);
 
         if (sentinelTest) {
             CHECK(op[oSizeMax] != mark, "op[oSizeMax] = %02X != %02X : "
@@ -1055,17 +1070,21 @@ size_t test_lz4f_decompression_wBuffers(
                     op[oSizeMax], mark);
         }
         if (LZ4F_getErrorCode(moreToFlush) == LZ4F_ERROR_contentChecksum_invalid) {
-            if (findErrorPos) DISPLAYLEVEL(2, "checksum error detected \n");
-            if (findErrorPos) locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
+            if (shouldSucceed) DISPLAYLEVEL(1, "checksum error reported after decompressing segment with LZ4F_decompress() \n");
+            if (shouldSucceed) locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
+            RETURN_ERROR(contentChecksum_invalid);
+        }
+        if (LZ4F_isError(moreToFlush)) {
+            if (shouldSucceed) DISPLAYLEVEL(1, "error reported after decompressing segment with LZ4F_decompress() \n");
+            return moreToFlush;
         }
-        if (LZ4F_isError(moreToFlush)) return moreToFlush;
 
         XXH64_update(&xxh64, op, oSize);
         totalOut += oSize;
         op += oSize;
         ip += iSize;
         if (o_scenario == o_noncontiguous) {
-            if (op == oend) return LZ4F_ERROR_GENERIC;  /* can theoretically happen with bogus data */
+            if (op == oend) RETURN_ERROR(GENERIC);  /* can theoretically happen with bogus data */
             op++; /* create a gap between consecutive output */
         }
         if (o_scenario==o_overwrite) op = (BYTE*)dst;   /* overwrite destination */
@@ -1073,17 +1092,19 @@ size_t test_lz4f_decompression_wBuffers(
           && (iSize == 0)) /* no input consumed */
             break;
     }
-    if (moreToFlush != 0) return LZ4F_ERROR_decompressionFailed;
+    if (moreToFlush != 0) RETURN_ERROR(decompressionFailed);
     if (totalOut) {  /* otherwise, it's a skippable frame */
         U64 const crcDecoded = XXH64_digest(&xxh64);
         if (crcDecoded != crcOrig) {
-            if (findErrorPos) locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
-            return LZ4F_ERROR_contentChecksum_invalid;
+            if (shouldSucceed) DISPLAYLEVEL(1, "checksum error detected at end of decompression \n");
+            if (shouldSucceed) locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
+            RETURN_ERROR(contentChecksum_invalid);
     }   }
     return 0;
 }
 
 
+/* @return : 0 on success, or LZ4F_ErrorCode */
 size_t test_lz4f_decompression(const void* cSrc, size_t cSize,
                                const void* srcRef, size_t decompressedSize,
                                U64 crcOrig,
@@ -1095,7 +1116,7 @@ size_t test_lz4f_decompression(const void* cSrc, size_t cSize,
     o_scenario_e const o_scenario = (o_scenario_e)(FUZ_rand(randState) % 3);   /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */
     /* tighten dst buffer conditions */
     size_t const dstCapacity = (o_scenario == o_noncontiguous) ?
-                               (decompressedSize * 2) + 128 :
+                               (decompressedSize * 2) + 128 : /* should provide enough room to insert spaces beween blocks */
                                decompressedSize;
     size_t result;
     void* const dstBuffer = malloc(dstCapacity);
@@ -1152,7 +1173,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
         size_t const srcStartId = FUZ_rand(&randState) % (CNBufferLength - srcSize);
         const BYTE* const srcStart = (const BYTE*)CNBuffer + srcStartId;
         unsigned const neverFlush = (FUZ_rand(&randState) & 15) == 1;
-        U64 const crcOrig = XXH64(srcStart, srcSize, 1);
+        U64 const crcOrig = XXH64(srcStart, srcSize, extCrcSeed);
         LZ4F_preferences_t prefs;
         const LZ4F_preferences_t* prefsPtr = &prefs;
         size_t cSize;
@@ -1200,11 +1221,9 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                 size_t iSize = MIN(sampleMax, (size_t)(iend-ip));
                 size_t const oSize = LZ4F_compressBound(iSize, prefsPtr);
                 cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);
-                DISPLAYLEVEL(6, "Sending %u bytes to compress (stableSrc:%u) \n",
-                                (unsigned)iSize, cOptions.stableSrc);
 
 #if 1
-                /* insert uncompressed segment */
+                /* test inserting an uncompressed block */
                 if ( (iSize>0)
                   && !neverFlush   /* do not mess with compressBound when neverFlush is set */
                   && prefsPtr != NULL   /* prefs are set */
@@ -1212,6 +1231,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                   && (FUZ_rand(&randState) & 15) == 1 ) {
                     size_t const uSize = FUZ_rand(&randState) % iSize;
                     size_t const flushedSize = LZ4F_uncompressedUpdate(cCtx, op, (size_t)(oend-op), ip, uSize, &cOptions);
+                    DISPLAYLEVEL(6, "Actually sending %u bytes as uncompressed \n", (unsigned)uSize);
                     CHECK(LZ4F_isError(flushedSize), "Insert uncompressed data failed (error %i : %s)",
                             (int)flushedSize, LZ4F_getErrorName(flushedSize));
                     op += flushedSize;
@@ -1220,6 +1240,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                 }
 #endif
 
+                DISPLAYLEVEL(6, "Sending %u bytes to compress or buffer (stableSrc:%u) \n",
+                                (unsigned)iSize, cOptions.stableSrc);
                 {   size_t const flushedSize = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
                     CHECK(LZ4F_isError(flushedSize), "Compression failed (error %i : %s)",
                             (int)flushedSize, LZ4F_getErrorName(flushedSize));
@@ -1230,7 +1252,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                 {   unsigned const forceFlush = neverFlush ? 0 : ((FUZ_rand(&randState) & 3) == 1);
                     if (forceFlush) {
                         size_t const flushSize = LZ4F_flush(cCtx, op, (size_t)(oend-op), &cOptions);
-                        DISPLAYLEVEL(6,"flushing %u bytes \n", (unsigned)flushSize);
+                        DISPLAYLEVEL(6, "flushing %u bytes \n", (unsigned)flushSize);
                         CHECK(LZ4F_isError(flushSize), "Compression failed (error %i)", (int)flushSize);
                         op += flushSize;
                         if ((FUZ_rand(&randState) % 1024) == 3) {
@@ -1244,7 +1266,10 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
                                 op += 4;
                 }   }   }   }
             }  /* while (ip<iend) */
-            CHECK(op>=oend, "LZ4F_compressFrameBound overflow");
+
+            /* check compressBound guarantees */
+            if (neverFlush) CHECK(op>=oend, "LZ4F_compressFrameBound overflow");
+
             {   size_t const dstEndSafeSize = LZ4F_compressBound(0, prefsPtr);
                 int const tooSmallDstEnd = ((FUZ_rand(&randState) & 31) == 3);
                 size_t const dstEndTooSmallSize = (FUZ_rand(&randState) % dstEndSafeSize) + 1;

tests/fuzzer.c

@@ -491,16 +491,20 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
             FUZ_DISPLAYTEST("test LZ4_compress_destSize_extState() with too small dest buffer (must succeed, compress less than full input)");
             {   int inputSize = blockSize;
-                int const r3 = LZ4_compress_destSize_extState(stateLZ4, block, compressedBuffer, &inputSize, r-1, 8);
+                /* note: LZ4_compress_destSize() still requires a targetDstSize > 0, since it's the minimum valid output size produced by LZ4 */
+                int targetDstSize = r-1 < 1 ? 1 : r-1;
+                int const r3 = LZ4_compress_destSize_extState(stateLZ4, block, compressedBuffer, &inputSize, targetDstSize, 8);
                 FUZ_CHECKTEST(r3==0, "LZ4_compress_destSize_extState() failed");
-                FUZ_CHECKTEST(inputSize>=blockSize, "LZ4_compress_destSize_extState() should consume less than full input");
+                if (targetDstSize < r)
+                    FUZ_CHECKTEST(inputSize>=blockSize, "LZ4_compress_destSize_extState() should consume less than full input");
             }
         }
 
         /* Test compression using fast reset external state*/
         FUZ_DISPLAYTEST("test LZ4_compress_fast_extState_fastReset()");
         {   int const r = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
-            FUZ_CHECKTEST(r==0, "LZ4_compress_fast_extState_fastReset() failed"); }
+            FUZ_CHECKTEST(r==0, "LZ4_compress_fast_extState_fastReset() failed");
+        }
 
         /* Test compression */
         FUZ_DISPLAYTEST("test LZ4_compress_default()");
@@ -533,12 +537,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             }
 
             /* Test decoding with one byte missing => must fail */
-            FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short");
-            decodedBuffer[blockSize-1] = 0;
-            {   int const r = LZ4_decompress_fast(cBuffer_exact, decodedBuffer, blockSize-1);
-                FUZ_CHECKTEST(r>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
+            if (blockSize >= 1) {
+                FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short");
+                decodedBuffer[blockSize-1] = 0;
+                {   int const r = LZ4_decompress_fast(cBuffer_exact, decodedBuffer, blockSize-1);
+                    FUZ_CHECKTEST(r>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
+                }
+                FUZ_CHECKTEST(decodedBuffer[blockSize-1]!=0, "LZ4_decompress_fast overrun specified output buffer");
             }
-            FUZ_CHECKTEST(decodedBuffer[blockSize-1]!=0, "LZ4_decompress_fast overrun specified output buffer");
 
             /* Test decoding with one byte too much => must fail */
             FUZ_DISPLAYTEST();
@@ -572,12 +578,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             }
 
             /* Test decoding with output size being one byte too short => must fail */
-            FUZ_DISPLAYTEST();
-            decodedBuffer[blockSize-1] = 0;
-            {   int const r = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize-1);
-                FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
+            if (blockSize >= 1) {
+                FUZ_DISPLAYTEST();
+                decodedBuffer[blockSize-1] = 0;
+                {   int const r = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize-1);
+                    FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
+                }
+                FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
             }
-            FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
 
             /* Test decoding with output size being 10 bytes too short => must fail */
             FUZ_DISPLAYTEST();
@@ -786,6 +794,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
         FUZ_DISPLAYTEST("LZ4_compress_fast_continue() with dictionary and output buffer too short by one byte");
         LZ4_loadDict(&LZ4dictBody, dict, dictSize);
+        assert(blockContinueCompressedSize >= 1);
         ret = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
         FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
 
@@ -820,17 +829,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
         }
 
-        FUZ_DISPLAYTEST();
-        decodedBuffer[blockSize-1] = 0;
-        ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
-        FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
-        FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
+        if (blockSize >= 1) {
+            FUZ_DISPLAYTEST();
+            decodedBuffer[blockSize-1] = 0;
+            ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
+            FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
+            FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
 
-        FUZ_DISPLAYTEST();
-        decodedBuffer[blockSize-1] = 0;
-        ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
-        FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
-        FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+            FUZ_DISPLAYTEST();
+            decodedBuffer[blockSize-1] = 0;
+            ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
+            FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
+            FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+        }
 
         FUZ_DISPLAYTEST();
         {   int const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
@@ -870,6 +881,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short");
             LZ4_resetStream_fast(&LZ4_stream);
             LZ4_attach_dictionary(&LZ4_stream, &LZ4dictBody);
+            assert(blockContinueCompressedSize >= 1);
             ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
             FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
             /* note : context is no longer dirty after a failed compressed block */
@@ -914,17 +926,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
         }
 
-        FUZ_DISPLAYTEST();
-        decodedBuffer[blockSize-1] = 0;
-        ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
-        FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
-        FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
+        if (blockSize >= 1) {
+            FUZ_DISPLAYTEST();
+            decodedBuffer[blockSize-1] = 0;
+            ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
+            FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
+            FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
 
-        FUZ_DISPLAYTEST();
-        decodedBuffer[blockSize-1] = 0;
-        ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
-        FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
-        FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+            FUZ_DISPLAYTEST();
+            decodedBuffer[blockSize-1] = 0;
+            ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
+            FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
+            FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
+        }
 
         FUZ_DISPLAYTEST("LZ4_decompress_safe_usingDict with a too small output buffer");
         {   int const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
@@ -947,6 +961,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
 
         FUZ_DISPLAYTEST("LZ4_compress_HC_continue with same external dictionary, but output buffer 1 byte too short");
         LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
+        assert(blockContinueCompressedSize>=1);
         ret = LZ4_compress_HC_continue(LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
         FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (expected %i, but result=%i)", blockContinueCompressedSize, ret);
         /* note : context is no longer dirty after a failed compressed block */
@@ -983,6 +998,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
             FUZ_DISPLAYTEST();
             LZ4_resetStreamHC_fast (LZ4_streamHC, compressionLevel);
             LZ4_attach_HC_dictionary(LZ4_streamHC, LZ4dictHC);
+            assert(blockContinueCompressedSize>=1);
             ret = LZ4_compress_HC_continue(LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
             FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDictCtx should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
             /* note : context is no longer dirty after a failed compressed block */