Compare commits
23 Commits
gh_x32_dis
...
dev
Author | SHA1 | Date | |
---|---|---|---|
|
07c526c0af | ||
|
b8af3e2f74 | ||
|
555d294b9b | ||
|
d5362329cd | ||
|
b35b24fb78 | ||
|
923839bbc9 | ||
|
205fd851dd | ||
|
527431e832 | ||
|
de629c38e9 | ||
|
eba7d05b8e | ||
|
9b9e03ff09 | ||
|
710ed4f0d7 | ||
|
5c2068c606 | ||
|
6869ab67d7 | ||
|
7e41965fbb | ||
|
6cf42afbea | ||
|
17a8cf523c | ||
|
3f7a031e62 | ||
|
8da5d43433 | ||
|
0c841277a4 | ||
|
17c36d9c9e | ||
|
855a097886 | ||
|
184ffbd686 |
4
.github/workflows/ci.yml
vendored
4
.github/workflows/ci.yml
vendored
@@ -499,7 +499,7 @@ jobs:
|
||||
dry-run: false
|
||||
sanitizer: ${{ matrix.sanitizer }}
|
||||
- name: Upload Crash
|
||||
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # https://github.com/actions/upload-artifact v4.4.3
|
||||
uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # https://github.com/actions/upload-artifact v4.5.0
|
||||
if: failure() && steps.build.outcome == 'success'
|
||||
with:
|
||||
name: ${{ matrix.sanitizer }}-artifacts
|
||||
@@ -690,7 +690,7 @@ jobs:
|
||||
popd
|
||||
|
||||
- name: Upload Generated VS2022 Directory
|
||||
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # https://github.com/actions/upload-artifact v4.4.3
|
||||
uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # https://github.com/actions/upload-artifact v4.5.0
|
||||
with:
|
||||
name: VS2022-Build-Dir
|
||||
path: "build/visual/Visual Studio 17 2022"
|
||||
|
4
.github/workflows/scorecard.yml
vendored
4
.github/workflows/scorecard.yml
vendored
@@ -52,7 +52,7 @@ jobs:
|
||||
# Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
|
||||
# format to the repository Actions tab.
|
||||
- name: "Upload artifact"
|
||||
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
|
||||
uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
|
||||
with:
|
||||
name: SARIF file
|
||||
path: results.sarif
|
||||
@@ -60,6 +60,6 @@ jobs:
|
||||
|
||||
# Upload the results to GitHub's code scanning dashboard.
|
||||
- name: "Upload to code-scanning"
|
||||
uses: github/codeql-action/upload-sarif@429e1977040da7a23b6822b13c129cd1ba93dbb2 # v3.26.2
|
||||
uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
|
@@ -123,7 +123,7 @@ Most distributions are bundled with a package manager
|
||||
which allows easy installation of both the `liblz4` library
|
||||
and the `lz4` command line interface.
|
||||
|
||||
[](https://repology.org/project/lz4/versions)
|
||||
[](https://repology.org/project/lz4/versions)
|
||||
|
||||
|
||||
### Special Thanks
|
||||
|
@@ -35,6 +35,7 @@ WFLAGS = -std=gnu99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-pro
|
||||
CFLAGS = $(WFLAGS) -O2 $(USERCFLAGS)
|
||||
|
||||
TESTFILE = Makefile
|
||||
TESTFILE_SMALL= .gitignore
|
||||
SLIBLZ4 := $(LIBDIR)/liblz4.a
|
||||
LZ4DIR = ../programs
|
||||
LZ4 = $(LZ4DIR)/lz4
|
||||
@@ -80,6 +81,7 @@ test : all $(LZ4)
|
||||
./blockStreaming_lineByLine$(EXT) $(TESTFILE)
|
||||
@echo "\n=== Dictionary Random Access ==="
|
||||
./dictionaryRandomAccess$(EXT) $(TESTFILE) $(TESTFILE) 1100 1400
|
||||
./dictionaryRandomAccess$(EXT) $(TESTFILE_SMALL) $(TESTFILE_SMALL) 0 32
|
||||
@echo "\n=== Frame compression ==="
|
||||
./frameCompress$(EXT) $(TESTFILE)
|
||||
$(LZ4) -vt $(TESTFILE).lz4
|
||||
|
@@ -251,7 +251,7 @@ int main(int argc, char* argv[])
|
||||
FILE* outFp = fopen(decFilename, "wb");
|
||||
|
||||
printf("decompress : %s -> %s\n", lz4Filename, decFilename);
|
||||
test_decompress(outFp, inpFp, dict, DICTIONARY_BYTES, offset, length);
|
||||
test_decompress(outFp, inpFp, dict, dictSize, offset, length);
|
||||
printf("decompress : done\n");
|
||||
|
||||
fclose(outFp);
|
||||
|
@@ -28,7 +28,7 @@ static size_t get_file_size(char *filename)
|
||||
return 0;
|
||||
}
|
||||
|
||||
return statbuf.st_size;
|
||||
return (size_t)statbuf.st_size;
|
||||
}
|
||||
|
||||
static int compress_file(FILE* f_in, FILE* f_out)
|
||||
@@ -36,7 +36,6 @@ static int compress_file(FILE* f_in, FILE* f_out)
|
||||
assert(f_in != NULL); assert(f_out != NULL);
|
||||
|
||||
LZ4F_errorCode_t ret = LZ4F_OK_NoError;
|
||||
size_t len;
|
||||
LZ4_writeFile_t* lz4fWrite;
|
||||
void* const buf = malloc(CHUNK_SIZE);
|
||||
if (!buf) {
|
||||
@@ -56,7 +55,7 @@ static int compress_file(FILE* f_in, FILE* f_out)
|
||||
}
|
||||
|
||||
while (1) {
|
||||
len = fread(buf, 1, CHUNK_SIZE, f_in);
|
||||
size_t len = fread(buf, 1, CHUNK_SIZE, f_in);
|
||||
|
||||
if (ferror(f_in)) {
|
||||
printf("fread error\n");
|
||||
@@ -94,6 +93,7 @@ static int decompress_file(FILE* f_in, FILE* f_out)
|
||||
void* const buf= malloc(CHUNK_SIZE);
|
||||
if (!buf) {
|
||||
printf("error: memory allocation failed \n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
ret = LZ4F_readOpen(&lz4fRead, f_in);
|
||||
@@ -135,9 +135,11 @@ out:
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* @return 0 if both FILE* have identical content */
|
||||
int compareFiles(FILE* fp0, FILE* fp1)
|
||||
{
|
||||
int result = 0;
|
||||
assert(fp0 != NULL); assert(fp1 != NULL);
|
||||
|
||||
while (result==0) {
|
||||
char b0[1024];
|
||||
@@ -147,7 +149,7 @@ int compareFiles(FILE* fp0, FILE* fp1)
|
||||
|
||||
result = (r0 != r1);
|
||||
if (!r0 || !r1) break;
|
||||
if (!result) result = memcmp(b0, b1, r0);
|
||||
if (r0 == r1) result = memcmp(b0, b1, r0);
|
||||
}
|
||||
|
||||
return result;
|
||||
@@ -175,7 +177,7 @@ int main(int argc, const char **argv) {
|
||||
{ FILE* const inpFp = fopen(inpFilename, "rb");
|
||||
FILE* const outFp = fopen(lz4Filename, "wb");
|
||||
printf("compress : %s -> %s\n", inpFilename, lz4Filename);
|
||||
LZ4F_errorCode_t ret = compress_file(inpFp, outFp);
|
||||
int ret = compress_file(inpFp, outFp);
|
||||
fclose(inpFp);
|
||||
fclose(outFp);
|
||||
|
||||
@@ -188,7 +190,7 @@ int main(int argc, const char **argv) {
|
||||
inpFilename,
|
||||
get_file_size(inpFilename),
|
||||
get_file_size(lz4Filename), /* might overflow is size_t is 32 bits and size_{in,out} > 4 GB */
|
||||
(double)get_file_size(lz4Filename) / get_file_size(inpFilename) * 100);
|
||||
(double)get_file_size(lz4Filename) / (double)get_file_size(inpFilename) * 100);
|
||||
|
||||
printf("compress : done\n");
|
||||
}
|
||||
@@ -198,8 +200,11 @@ int main(int argc, const char **argv) {
|
||||
FILE* const inpFp = fopen(lz4Filename, "rb");
|
||||
FILE* const outFp = fopen(decFilename, "wb");
|
||||
|
||||
if (inpFp == NULL) exit(1);
|
||||
if (outFp == NULL) exit(1);
|
||||
|
||||
printf("decompress : %s -> %s\n", lz4Filename, decFilename);
|
||||
LZ4F_errorCode_t ret = decompress_file(inpFp, outFp);
|
||||
int ret = decompress_file(inpFp, outFp);
|
||||
|
||||
fclose(outFp);
|
||||
fclose(inpFp);
|
||||
@@ -216,6 +221,9 @@ int main(int argc, const char **argv) {
|
||||
{ FILE* const inpFp = fopen(inpFilename, "rb");
|
||||
FILE* const decFp = fopen(decFilename, "rb");
|
||||
|
||||
if (inpFp == NULL) exit(1);
|
||||
if (decFp == NULL) exit(1);
|
||||
|
||||
printf("verify : %s <-> %s\n", inpFilename, decFilename);
|
||||
int const cmp = compareFiles(inpFp, decFp);
|
||||
|
||||
|
10
lib/lz4.h
10
lib/lz4.h
@@ -258,10 +258,12 @@ LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* d
|
||||
* @return : Nb bytes written into 'dst' (necessarily <= dstCapacity)
|
||||
* or 0 if compression fails.
|
||||
*
|
||||
* Note : from v1.8.2 to v1.9.1, this function had a bug (fixed in v1.9.2+):
|
||||
* the produced compressed content could, in specific circumstances,
|
||||
* require to be decompressed into a destination buffer larger
|
||||
* by at least 1 byte than the content to decompress.
|
||||
* Note : 'targetDstSize' must be >= 1, because it's the smallest valid lz4 payload.
|
||||
*
|
||||
* Note 2:from v1.8.2 to v1.9.1, this function had a bug (fixed in v1.9.2+):
|
||||
* the produced compressed content could, in rare circumstances,
|
||||
* require to be decompressed into a destination buffer
|
||||
* larger by at least 1 byte than decompressesSize.
|
||||
* If an application uses `LZ4_compress_destSize()`,
|
||||
* it's highly recommended to update liblz4 to v1.9.2 or better.
|
||||
* If this can't be done or ensured,
|
||||
|
@@ -155,7 +155,7 @@ static void LZ4F_free(void* p, LZ4F_CustomMem cmem)
|
||||
static int g_debuglog_enable = 1;
|
||||
# define DEBUGLOG(l, ...) { \
|
||||
if ((g_debuglog_enable) && (l<=LZ4_DEBUG)) { \
|
||||
fprintf(stderr, __FILE__ " (%i): ", __LINE__ ); \
|
||||
fprintf(stderr, __FILE__ " %i: ", __LINE__ ); \
|
||||
fprintf(stderr, __VA_ARGS__); \
|
||||
fprintf(stderr, " \n"); \
|
||||
} }
|
||||
@@ -700,6 +700,7 @@ size_t LZ4F_compressBegin_internal(LZ4F_cctx* cctx,
|
||||
RETURN_ERROR_IF(dstCapacity < maxFHSize, dstMaxSize_tooSmall);
|
||||
if (preferencesPtr == NULL) preferencesPtr = &prefNull;
|
||||
cctx->prefs = *preferencesPtr;
|
||||
DEBUGLOG(5, "LZ4F_compressBegin_internal: Independent_blocks=%u", cctx->prefs.frameInfo.blockMode);
|
||||
|
||||
/* cctx Management */
|
||||
{ U16 const ctxTypeID = (cctx->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2;
|
||||
@@ -887,10 +888,11 @@ static size_t LZ4F_makeBlock(void* dst,
|
||||
LZ4F_blockChecksum_t crcFlag)
|
||||
{
|
||||
BYTE* const cSizePtr = (BYTE*)dst;
|
||||
int dstCapacity = (srcSize > 1) ? (int)srcSize - 1 : 1;
|
||||
U32 cSize;
|
||||
assert(compress != NULL);
|
||||
cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+BHSize),
|
||||
(int)(srcSize), (int)(srcSize-1),
|
||||
(int)srcSize, dstCapacity,
|
||||
level, cdict);
|
||||
|
||||
if (cSize == 0 || cSize >= srcSize) {
|
||||
@@ -961,12 +963,13 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev
|
||||
return LZ4F_compressBlockHC_continue;
|
||||
}
|
||||
|
||||
/* Save history (up to 64KB) into @tmpBuff */
|
||||
static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
|
||||
/* Save or shorten history (up to 64KB) into @tmpBuff */
|
||||
static void LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
|
||||
{
|
||||
if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
|
||||
return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
|
||||
return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
|
||||
int const dictSize = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ?
|
||||
LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB) :
|
||||
LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
|
||||
cctxPtr->tmpIn = cctxPtr->tmpBuff + dictSize;
|
||||
}
|
||||
|
||||
typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
|
||||
@@ -974,7 +977,7 @@ typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus;
|
||||
static const LZ4F_compressOptions_t k_cOptionsNull = { 0, { 0, 0, 0 } };
|
||||
|
||||
|
||||
/*! LZ4F_compressUpdateImpl() :
|
||||
/*! LZ4F_compressUpdateImpl() :
|
||||
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
|
||||
* When successful, the function always entirely consumes @srcBuffer.
|
||||
* src data is either buffered or compressed into @dstBuffer.
|
||||
@@ -1072,9 +1075,7 @@ static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
|
||||
if (compressOptionsPtr->stableSrc) {
|
||||
cctxPtr->tmpIn = cctxPtr->tmpBuff; /* src is stable : dictionary remains in src across invocations */
|
||||
} else {
|
||||
int const realDictSize = LZ4F_localSaveDict(cctxPtr);
|
||||
assert(0 <= realDictSize && realDictSize <= 64 KB);
|
||||
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
|
||||
LZ4F_localSaveDict(cctxPtr);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1084,8 +1085,7 @@ static size_t LZ4F_compressUpdateImpl(LZ4F_cctx* cctxPtr,
|
||||
{
|
||||
/* only preserve 64KB within internal buffer. Ensures there is enough room for next block.
|
||||
* note: this situation necessarily implies lastBlockCompressed==fromTmpBuffer */
|
||||
int const realDictSize = LZ4F_localSaveDict(cctxPtr);
|
||||
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
|
||||
LZ4F_localSaveDict(cctxPtr);
|
||||
assert((cctxPtr->tmpIn + blockSize) <= (cctxPtr->tmpBuff + cctxPtr->maxBufferSize));
|
||||
}
|
||||
|
||||
@@ -1150,7 +1150,7 @@ size_t LZ4F_uncompressedUpdate(LZ4F_cctx* cctxPtr,
|
||||
|
||||
/*! LZ4F_flush() :
|
||||
* When compressed data must be sent immediately, without waiting for a block to be filled,
|
||||
* invoke LZ4_flush(), which will immediately compress any remaining data stored within LZ4F_cctx.
|
||||
* invoke LZ4F_flush(), which will immediately compress any remaining data stored within LZ4F_cctx.
|
||||
* The result of the function is the number of bytes written into dstBuffer.
|
||||
* It can be zero, this means there was no data left within LZ4F_cctx.
|
||||
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
|
||||
@@ -1164,6 +1164,8 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
|
||||
BYTE* dstPtr = dstStart;
|
||||
compressFunc_t compress;
|
||||
|
||||
DEBUGLOG(5, "LZ4F_flush: %zu buffered bytes (saved dict size = %i) (dstCapacity=%u)",
|
||||
cctxPtr->tmpInSize, (int)(cctxPtr->tmpIn - cctxPtr->tmpBuff), (unsigned)dstCapacity);
|
||||
if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */
|
||||
RETURN_ERROR_IF(cctxPtr->cStage != 1, compressionState_uninitialized);
|
||||
RETURN_ERROR_IF(dstCapacity < (cctxPtr->tmpInSize + BHSize + BFSize), dstMaxSize_tooSmall);
|
||||
@@ -1185,9 +1187,9 @@ size_t LZ4F_flush(LZ4F_cctx* cctxPtr,
|
||||
cctxPtr->tmpInSize = 0;
|
||||
|
||||
/* keep tmpIn within limits */
|
||||
if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) { /* necessarily LZ4F_blockLinked */
|
||||
int const realDictSize = LZ4F_localSaveDict(cctxPtr);
|
||||
cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize;
|
||||
if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) {
|
||||
assert(cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked);
|
||||
LZ4F_localSaveDict(cctxPtr);
|
||||
}
|
||||
|
||||
return (size_t)(dstPtr - dstStart);
|
||||
|
181
lib/lz4hc.c
181
lib/lz4hc.c
@@ -262,6 +262,30 @@ static void LZ4HC_init_internal (LZ4HC_CCtx_internal* hc4, const BYTE* start)
|
||||
/**************************************
|
||||
* Encode
|
||||
**************************************/
|
||||
#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 2)
|
||||
# define RAWLOG(...) fprintf(stderr, __VA_ARGS__)
|
||||
void LZ4HC_hexOut(const void* src, size_t len)
|
||||
{
|
||||
const BYTE* p = (const BYTE*)src;
|
||||
size_t n;
|
||||
for (n=0; n<len; n++) {
|
||||
RAWLOG("%02X ", p[n]);
|
||||
}
|
||||
RAWLOG(" \n");
|
||||
}
|
||||
|
||||
# define HEX_CMP(_lev, _ptr, _ref, _len) \
|
||||
if (LZ4_DEBUG >= _lev) { \
|
||||
RAWLOG("match bytes: "); \
|
||||
LZ4HC_hexOut(_ptr, _len); \
|
||||
RAWLOG("ref bytes: "); \
|
||||
LZ4HC_hexOut(_ref, _len); \
|
||||
}
|
||||
|
||||
#else
|
||||
# define HEX_CMP(l,p,r,_l)
|
||||
#endif
|
||||
|
||||
/* LZ4HC_encodeSequence() :
|
||||
* @return : 0 if ok,
|
||||
* 1 if buffer issue detected */
|
||||
@@ -278,47 +302,49 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
|
||||
#define op (*_op)
|
||||
#define anchor (*_anchor)
|
||||
|
||||
size_t length;
|
||||
BYTE* const token = op++;
|
||||
|
||||
#if defined(LZ4_DEBUG) && (LZ4_DEBUG >= 6)
|
||||
static const BYTE* start = NULL;
|
||||
static U32 totalCost = 0;
|
||||
U32 const pos = (start==NULL) ? 0 : (U32)(anchor - start);
|
||||
U32 const pos = (start==NULL) ? 0 : (U32)(anchor - start); /* only works for single segment */
|
||||
U32 const ll = (U32)(ip - anchor);
|
||||
U32 const llAdd = (ll>=15) ? ((ll-15) / 255) + 1 : 0;
|
||||
U32 const mlAdd = (matchLength>=19) ? ((matchLength-19) / 255) + 1 : 0;
|
||||
U32 const cost = 1 + llAdd + ll + 2 + mlAdd;
|
||||
if (start==NULL) start = anchor; /* only works for single segment */
|
||||
/* g_debuglog_enable = (pos >= 2228) & (pos <= 2262); */
|
||||
DEBUGLOG(6, "pos:%7u -- literals:%4u, match:%4i, offset:%5i, cost:%4u + %5u",
|
||||
pos,
|
||||
(U32)(ip - anchor), matchLength, offset,
|
||||
cost, totalCost);
|
||||
# if 1 /* only works on single segment data */
|
||||
HEX_CMP(7, ip, ip-offset, matchLength);
|
||||
# endif
|
||||
totalCost += cost;
|
||||
#endif
|
||||
|
||||
/* Encode Literal length */
|
||||
length = (size_t)(ip - anchor);
|
||||
LZ4_STATIC_ASSERT(notLimited == 0);
|
||||
/* Check output limit */
|
||||
if (limit && ((op + (length / 255) + length + (2 + 1 + LASTLITERALS)) > oend)) {
|
||||
DEBUGLOG(6, "Not enough room to write %i literals (%i bytes remaining)",
|
||||
(int)length, (int)(oend - op));
|
||||
return 1;
|
||||
}
|
||||
if (length >= RUN_MASK) {
|
||||
size_t len = length - RUN_MASK;
|
||||
*token = (RUN_MASK << ML_BITS);
|
||||
for(; len >= 255 ; len -= 255) *op++ = 255;
|
||||
*op++ = (BYTE)len;
|
||||
} else {
|
||||
*token = (BYTE)(length << ML_BITS);
|
||||
}
|
||||
{ size_t litLen = (size_t)(ip - anchor);
|
||||
LZ4_STATIC_ASSERT(notLimited == 0);
|
||||
/* Check output limit */
|
||||
if (limit && ((op + (litLen / 255) + litLen + (2 + 1 + LASTLITERALS)) > oend)) {
|
||||
DEBUGLOG(6, "Not enough room to write %i literals (%i bytes remaining)",
|
||||
(int)litLen, (int)(oend - op));
|
||||
return 1;
|
||||
}
|
||||
if (litLen >= RUN_MASK) {
|
||||
size_t len = litLen - RUN_MASK;
|
||||
*token = (RUN_MASK << ML_BITS);
|
||||
for(; len >= 255 ; len -= 255) *op++ = 255;
|
||||
*op++ = (BYTE)len;
|
||||
} else {
|
||||
*token = (BYTE)(litLen << ML_BITS);
|
||||
}
|
||||
|
||||
/* Copy Literals */
|
||||
LZ4_wildCopy8(op, anchor, op + length);
|
||||
op += length;
|
||||
/* Copy Literals */
|
||||
LZ4_wildCopy8(op, anchor, op + litLen);
|
||||
op += litLen;
|
||||
}
|
||||
|
||||
/* Encode Offset */
|
||||
assert(offset <= LZ4_DISTANCE_MAX );
|
||||
@@ -327,20 +353,20 @@ LZ4_FORCE_INLINE int LZ4HC_encodeSequence (
|
||||
|
||||
/* Encode MatchLength */
|
||||
assert(matchLength >= MINMATCH);
|
||||
length = (size_t)matchLength - MINMATCH;
|
||||
if (limit && (op + (length / 255) + (1 + LASTLITERALS) > oend)) {
|
||||
DEBUGLOG(6, "Not enough room to write match length");
|
||||
return 1; /* Check output limit */
|
||||
}
|
||||
if (length >= ML_MASK) {
|
||||
*token += ML_MASK;
|
||||
length -= ML_MASK;
|
||||
for(; length >= 510 ; length -= 510) { *op++ = 255; *op++ = 255; }
|
||||
if (length >= 255) { length -= 255; *op++ = 255; }
|
||||
*op++ = (BYTE)length;
|
||||
} else {
|
||||
*token += (BYTE)(length);
|
||||
}
|
||||
{ size_t mlCode = (size_t)matchLength - MINMATCH;
|
||||
if (limit && (op + (mlCode / 255) + (1 + LASTLITERALS) > oend)) {
|
||||
DEBUGLOG(6, "Not enough room to write match length");
|
||||
return 1; /* Check output limit */
|
||||
}
|
||||
if (mlCode >= ML_MASK) {
|
||||
*token += ML_MASK;
|
||||
mlCode -= ML_MASK;
|
||||
for(; mlCode >= 510 ; mlCode -= 510) { *op++ = 255; *op++ = 255; }
|
||||
if (mlCode >= 255) { mlCode -= 255; *op++ = 255; }
|
||||
*op++ = (BYTE)mlCode;
|
||||
} else {
|
||||
*token += (BYTE)(mlCode);
|
||||
} }
|
||||
|
||||
/* Prepare next loop */
|
||||
ip += matchLength;
|
||||
@@ -519,6 +545,12 @@ static LZ4MID_searchIntoDict_f select_searchDict_function(const LZ4HC_CCtx_inter
|
||||
return LZ4MID_searchHCDict;
|
||||
}
|
||||
|
||||
/* preconditions:
|
||||
* - *srcSizePtr within [1, LZ4_MAX_INPUT_SIZE]
|
||||
* - src is valid
|
||||
* - maxOutputSize >= 1
|
||||
* - dst is valid
|
||||
*/
|
||||
static int LZ4MID_compress (
|
||||
LZ4HC_CCtx_internal* const ctx,
|
||||
const char* const src,
|
||||
@@ -550,18 +582,16 @@ static int LZ4MID_compress (
|
||||
unsigned matchLength;
|
||||
unsigned matchDistance;
|
||||
|
||||
/* input sanitization */
|
||||
DEBUGLOG(5, "LZ4MID_compress (%i bytes)", *srcSizePtr);
|
||||
|
||||
/* preconditions verifications */
|
||||
if (dict == usingDictCtxHc) DEBUGLOG(5, "usingDictCtxHc");
|
||||
assert(*srcSizePtr >= 0);
|
||||
if (*srcSizePtr) assert(src != NULL);
|
||||
if (maxOutputSize) assert(dst != NULL);
|
||||
if (*srcSizePtr < 0) return 0; /* invalid */
|
||||
if (maxOutputSize < 0) return 0; /* invalid */
|
||||
if (*srcSizePtr > LZ4_MAX_INPUT_SIZE) {
|
||||
/* forbidden: no input is allowed to be that large */
|
||||
return 0;
|
||||
}
|
||||
assert(*srcSizePtr > 0);
|
||||
assert(*srcSizePtr <= LZ4_MAX_INPUT_SIZE);
|
||||
assert(src != NULL);
|
||||
assert(maxOutputSize >= 1);
|
||||
assert(dst != NULL);
|
||||
|
||||
if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */
|
||||
if (*srcSizePtr < LZ4_minLength)
|
||||
goto _lz4mid_last_literals; /* Input too small, no compression (all literals) */
|
||||
@@ -940,6 +970,7 @@ LZ4HC_InsertAndGetWiderMatch (
|
||||
offset = (int)(ipIndex - matchIndex);
|
||||
sBack = back;
|
||||
DEBUGLOG(7, "Found match of len=%i within prefix, offset=%i, back=%i", longest, offset, -back);
|
||||
HEX_CMP(7, ip + back, ip + back - offset, (size_t)matchLength);
|
||||
} } }
|
||||
} else { /* lowestMatchIndex <= matchIndex < dictLimit : within Ext Dict */
|
||||
const BYTE* const matchPtr = dictStart + (matchIndex - dictIdx);
|
||||
@@ -959,6 +990,7 @@ LZ4HC_InsertAndGetWiderMatch (
|
||||
offset = (int)(ipIndex - matchIndex);
|
||||
sBack = back;
|
||||
DEBUGLOG(7, "Found match of len=%i within dict, offset=%i, back=%i", longest, offset, -back);
|
||||
HEX_CMP(7, ip + back, matchPtr + back, (size_t)matchLength);
|
||||
} } }
|
||||
|
||||
if (chainSwap && matchLength==longest) { /* better match => select a better chain */
|
||||
@@ -1118,10 +1150,16 @@ LZ4HC_InsertAndFindBestMatch(LZ4HC_CCtx_internal* const hc4, /* Index table wi
|
||||
}
|
||||
|
||||
|
||||
/* preconditions:
|
||||
* - *srcSizePtr within [1, LZ4_MAX_INPUT_SIZE]
|
||||
* - src is valid
|
||||
* - maxOutputSize >= 1
|
||||
* - dst is valid
|
||||
*/
|
||||
LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
|
||||
LZ4HC_CCtx_internal* const ctx,
|
||||
const char* const source,
|
||||
char* const dest,
|
||||
const char* const src,
|
||||
char* const dst,
|
||||
int* srcSizePtr,
|
||||
int const maxOutputSize,
|
||||
int maxNbAttempts,
|
||||
@@ -1132,14 +1170,14 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
|
||||
const int inputSize = *srcSizePtr;
|
||||
const int patternAnalysis = (maxNbAttempts > 128); /* levels 9+ */
|
||||
|
||||
const BYTE* ip = (const BYTE*) source;
|
||||
const BYTE* ip = (const BYTE*)src;
|
||||
const BYTE* anchor = ip;
|
||||
const BYTE* const iend = ip + inputSize;
|
||||
const BYTE* const mflimit = iend - MFLIMIT;
|
||||
const BYTE* const matchlimit = (iend - LASTLITERALS);
|
||||
|
||||
BYTE* optr = (BYTE*) dest;
|
||||
BYTE* op = (BYTE*) dest;
|
||||
BYTE* optr = (BYTE*) dst;
|
||||
BYTE* op = (BYTE*) dst;
|
||||
BYTE* oend = op + maxOutputSize;
|
||||
|
||||
const BYTE* start0;
|
||||
@@ -1150,6 +1188,13 @@ LZ4_FORCE_INLINE int LZ4HC_compress_hashChain (
|
||||
|
||||
/* init */
|
||||
DEBUGLOG(5, "LZ4HC_compress_hashChain (dict?=>%i)", dict);
|
||||
|
||||
/* preconditions verifications */
|
||||
assert(*srcSizePtr >= 1);
|
||||
assert(src != NULL);
|
||||
assert(maxOutputSize >= 1);
|
||||
assert(dst != NULL);
|
||||
|
||||
*srcSizePtr = 0;
|
||||
if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */
|
||||
if (inputSize < LZ4_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
|
||||
@@ -1334,8 +1379,8 @@ _last_literals:
|
||||
}
|
||||
|
||||
/* End */
|
||||
*srcSizePtr = (int) (((const char*)ip) - source);
|
||||
return (int) (((char*)op)-dest);
|
||||
*srcSizePtr = (int) (((const char*)ip) - src);
|
||||
return (int) (((char*)op)-dst);
|
||||
|
||||
_dest_overflow:
|
||||
if (limit == fillOutput) {
|
||||
@@ -1370,7 +1415,7 @@ static int LZ4HC_compress_optimal( LZ4HC_CCtx_internal* ctx,
|
||||
const dictCtx_directive dict,
|
||||
const HCfavor_e favorDecSpeed);
|
||||
|
||||
LZ4_FORCE_INLINE int
|
||||
static int
|
||||
LZ4HC_compress_generic_internal (
|
||||
LZ4HC_CCtx_internal* const ctx,
|
||||
const char* const src,
|
||||
@@ -1382,11 +1427,15 @@ LZ4HC_compress_generic_internal (
|
||||
const dictCtx_directive dict
|
||||
)
|
||||
{
|
||||
DEBUGLOG(5, "LZ4HC_compress_generic_internal(src=%p, srcSize=%d)",
|
||||
src, *srcSizePtr);
|
||||
DEBUGLOG(5, "LZ4HC_compress_generic_internal(src=%p, srcSize=%d, dstCapacity=%d)",
|
||||
src, *srcSizePtr, dstCapacity);
|
||||
|
||||
if (limit == fillOutput && dstCapacity < 1) return 0; /* Impossible to store anything */
|
||||
/* input sanitization */
|
||||
if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size (too large or negative) */
|
||||
if (dstCapacity < 1) return 0; /* Invalid: impossible to store anything */
|
||||
assert(dst); /* since dstCapacity >= 1, dst must be valid */
|
||||
if (*srcSizePtr == 0) { *dst = 0; return 1; }
|
||||
assert(src != NULL); /* since *srcSizePtr >= 1, src must be valid */
|
||||
|
||||
ctx->end += *srcSizePtr;
|
||||
{ cParams_t const cParam = LZ4HC_getCLevelParams(cLevel);
|
||||
@@ -1820,6 +1869,13 @@ LZ4HC_FindLongerMatch(LZ4HC_CCtx_internal* const ctx,
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* preconditions:
|
||||
* - *srcSizePtr within [1, LZ4_MAX_INPUT_SIZE]
|
||||
* - src is valid
|
||||
* - maxOutputSize >= 1
|
||||
* - dst is valid
|
||||
*/
|
||||
static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
|
||||
const char* const source,
|
||||
char* dst,
|
||||
@@ -1837,7 +1893,7 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
|
||||
#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
|
||||
LZ4HC_optimal_t* const opt = (LZ4HC_optimal_t*)ALLOC(sizeof(LZ4HC_optimal_t) * (LZ4_OPT_NUM + TRAILING_LITERALS));
|
||||
#else
|
||||
LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* ~64 KB, which is a bit large for stack... */
|
||||
LZ4HC_optimal_t opt[LZ4_OPT_NUM + TRAILING_LITERALS]; /* ~64 KB, which can be a bit large for some stacks... */
|
||||
#endif
|
||||
|
||||
const BYTE* ip = (const BYTE*) source;
|
||||
@@ -1852,10 +1908,17 @@ static int LZ4HC_compress_optimal ( LZ4HC_CCtx_internal* ctx,
|
||||
int ovoff = 0;
|
||||
|
||||
/* init */
|
||||
DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity);
|
||||
#if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1
|
||||
if (opt == NULL) goto _return_label;
|
||||
#endif
|
||||
DEBUGLOG(5, "LZ4HC_compress_optimal(dst=%p, dstCapa=%u)", dst, (unsigned)dstCapacity);
|
||||
|
||||
/* preconditions verifications */
|
||||
assert(dstCapacity > 0);
|
||||
assert(dst != NULL);
|
||||
assert(*srcSizePtr > 0);
|
||||
assert(source != NULL);
|
||||
|
||||
*srcSizePtr = 0;
|
||||
if (limit == fillOutput) oend -= LASTLITERALS; /* Hack for support LZ4 format restriction */
|
||||
if (sufficient_len >= LZ4_OPT_NUM) sufficient_len = LZ4_OPT_NUM-1;
|
||||
|
@@ -75,6 +75,7 @@ static const U32 nbTestsDefault = 256 KB;
|
||||
static const U32 prime1 = 2654435761U;
|
||||
static const U32 prime2 = 2246822519U;
|
||||
|
||||
static const U64 extCrcSeed = 1;
|
||||
|
||||
/*-************************************
|
||||
* Macros
|
||||
@@ -298,7 +299,7 @@ static int unitTests(U32 seed, double compressibility)
|
||||
goto _output_error;
|
||||
}
|
||||
FUZ_fillCompressibleNoiseBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, randState);
|
||||
crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
|
||||
crcOrig = XXH64(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, extCrcSeed);
|
||||
|
||||
/* LZ4F_compressBound() : special case : srcSize == 0 */
|
||||
DISPLAYLEVEL(3, "LZ4F_compressBound(0) = ");
|
||||
@@ -371,7 +372,7 @@ static int unitTests(U32 seed, double compressibility)
|
||||
|
||||
DISPLAYLEVEL(3, "Single Pass decompression : ");
|
||||
CHECK( LZ4F_decompress(dCtx, decodedBuffer, &decodedBufferSize, compressedBuffer, &compressedBufferSize, NULL) );
|
||||
{ U64 const crcDest = XXH64(decodedBuffer, decodedBufferSize, 1);
|
||||
{ U64 const crcDest = XXH64(decodedBuffer, decodedBufferSize, extCrcSeed);
|
||||
if (crcDest != crcOrig) goto _output_error; }
|
||||
DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedBufferSize);
|
||||
|
||||
@@ -398,7 +399,7 @@ static int unitTests(U32 seed, double compressibility)
|
||||
if (decResult != 0) goto _output_error; /* should finish now */
|
||||
op += oSize;
|
||||
if (op>oend) { DISPLAY("decompression write overflow \n"); goto _output_error; }
|
||||
{ U64 const crcDest = XXH64(decodedBuffer, (size_t)(op-ostart), 1);
|
||||
{ U64 const crcDest = XXH64(decodedBuffer, (size_t)(op-ostart), extCrcSeed);
|
||||
if (crcDest != crcOrig) goto _output_error;
|
||||
} }
|
||||
|
||||
@@ -471,7 +472,7 @@ static int unitTests(U32 seed, double compressibility)
|
||||
op += oSize;
|
||||
ip += iSize;
|
||||
}
|
||||
{ U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
|
||||
{ U64 const crcDest = XXH64(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, extCrcSeed);
|
||||
if (crcDest != crcOrig) goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(3, "Regenerated %u/%u bytes \n", (unsigned)(op-ostart), (unsigned)COMPRESSIBLE_NOISE_LENGTH);
|
||||
@@ -515,7 +516,7 @@ static int unitTests(U32 seed, double compressibility)
|
||||
ip += iSize;
|
||||
}
|
||||
{ size_t const decodedSize = (size_t)(op - ostart);
|
||||
U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1);
|
||||
U64 const crcDest = XXH64(decodedBuffer, decodedSize, extCrcSeed);
|
||||
if (crcDest != crcOrig) goto _output_error;
|
||||
DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
|
||||
}
|
||||
@@ -571,8 +572,8 @@ static int unitTests(U32 seed, double compressibility)
|
||||
CHECK( LZ4F_decompress(dctx, decodedBuffer, &decodedSize, compressedBuffer, &iSize, NULL) );
|
||||
if (decodedSize != testSize) goto _output_error;
|
||||
if (iSize != cSize) goto _output_error;
|
||||
{ U64 const crcDest = XXH64(decodedBuffer, decodedSize, 1);
|
||||
U64 const crcSrc = XXH64(CNBuffer, testSize, 1);
|
||||
{ U64 const crcDest = XXH64(decodedBuffer, decodedSize, extCrcSeed);
|
||||
U64 const crcSrc = XXH64(CNBuffer, testSize, extCrcSeed);
|
||||
if (crcDest != crcSrc) goto _output_error;
|
||||
}
|
||||
DISPLAYLEVEL(3, "Regenerated %u bytes \n", (U32)decodedSize);
|
||||
@@ -994,15 +995,28 @@ static void locateBuffDiff(const void* buff1, const void* buff2, size_t size, o_
|
||||
#undef CHECK
|
||||
#define CHECK(cond, ...) { if (cond) { EXIT_MSG(__VA_ARGS__); } }
|
||||
|
||||
static LZ4F_errorCode_t LZ4F_returnErrorCode(LZ4F_errorCodes code)
|
||||
{
|
||||
return (LZ4F_errorCode_t)-(ptrdiff_t)code;
|
||||
}
|
||||
|
||||
#define RETURN_ERROR(e) return LZ4F_returnErrorCode(LZ4F_ERROR_ ## e)
|
||||
|
||||
|
||||
/* @o_scenario: output behavior: contiguous, non-contiguous, or overwrite
|
||||
* @crcOrig: xxh64 of original data using extCrcSeed
|
||||
* @return : 0 on success, or LZ4F_ErrorCode
|
||||
* */
|
||||
size_t test_lz4f_decompression_wBuffers(
|
||||
const void* cSrc, size_t cSize,
|
||||
void* dst, size_t dstCapacity, o_scenario_e o_scenario,
|
||||
void* dst, size_t dstCapacity,
|
||||
o_scenario_e o_scenario,
|
||||
const void* srcRef, size_t decompressedSize,
|
||||
U64 crcOrig,
|
||||
U32* const randState,
|
||||
LZ4F_dctx* const dCtx,
|
||||
U32 seed, U32 testNb,
|
||||
int findErrorPos)
|
||||
int shouldSucceed)
|
||||
{
|
||||
const BYTE* ip = (const BYTE*)cSrc;
|
||||
const BYTE* const iend = ip + cSize;
|
||||
@@ -1015,7 +1029,8 @@ size_t test_lz4f_decompression_wBuffers(
|
||||
size_t totalOut = 0;
|
||||
size_t moreToFlush = 0;
|
||||
XXH64_state_t xxh64;
|
||||
XXH64_reset(&xxh64, 1);
|
||||
XXH64_reset(&xxh64, extCrcSeed);
|
||||
o_scenario = o_contiguous;
|
||||
assert(ip < iend);
|
||||
while (ip < iend) {
|
||||
unsigned const nbBitsI = (FUZ_rand(randState) % (maxBits-1)) + 1;
|
||||
@@ -1047,7 +1062,7 @@ size_t test_lz4f_decompression_wBuffers(
|
||||
moreToFlush = LZ4F_decompress(dCtx, tmpop, &oSize, tmpip, &iSize, &dOptions);
|
||||
free(iBuffer);
|
||||
}
|
||||
DISPLAYLEVEL(7, "oSize=%u, readSize=%u \n", (unsigned)oSize, (unsigned)iSize);
|
||||
DISPLAYLEVEL(7, "oSize=%u, readSize=%u \n", (unsigned)oSize, (unsigned)iSize);
|
||||
|
||||
if (sentinelTest) {
|
||||
CHECK(op[oSizeMax] != mark, "op[oSizeMax] = %02X != %02X : "
|
||||
@@ -1055,17 +1070,21 @@ size_t test_lz4f_decompression_wBuffers(
|
||||
op[oSizeMax], mark);
|
||||
}
|
||||
if (LZ4F_getErrorCode(moreToFlush) == LZ4F_ERROR_contentChecksum_invalid) {
|
||||
if (findErrorPos) DISPLAYLEVEL(2, "checksum error detected \n");
|
||||
if (findErrorPos) locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
|
||||
if (shouldSucceed) DISPLAYLEVEL(1, "checksum error reported after decompressing segment with LZ4F_decompress() \n");
|
||||
if (shouldSucceed) locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
|
||||
RETURN_ERROR(contentChecksum_invalid);
|
||||
}
|
||||
if (LZ4F_isError(moreToFlush)) {
|
||||
if (shouldSucceed) DISPLAYLEVEL(1, "error reported after decompressing segment with LZ4F_decompress() \n");
|
||||
return moreToFlush;
|
||||
}
|
||||
if (LZ4F_isError(moreToFlush)) return moreToFlush;
|
||||
|
||||
XXH64_update(&xxh64, op, oSize);
|
||||
totalOut += oSize;
|
||||
op += oSize;
|
||||
ip += iSize;
|
||||
if (o_scenario == o_noncontiguous) {
|
||||
if (op == oend) return LZ4F_ERROR_GENERIC; /* can theoretically happen with bogus data */
|
||||
if (op == oend) RETURN_ERROR(GENERIC); /* can theoretically happen with bogus data */
|
||||
op++; /* create a gap between consecutive output */
|
||||
}
|
||||
if (o_scenario==o_overwrite) op = (BYTE*)dst; /* overwrite destination */
|
||||
@@ -1073,17 +1092,19 @@ size_t test_lz4f_decompression_wBuffers(
|
||||
&& (iSize == 0)) /* no input consumed */
|
||||
break;
|
||||
}
|
||||
if (moreToFlush != 0) return LZ4F_ERROR_decompressionFailed;
|
||||
if (moreToFlush != 0) RETURN_ERROR(decompressionFailed);
|
||||
if (totalOut) { /* otherwise, it's a skippable frame */
|
||||
U64 const crcDecoded = XXH64_digest(&xxh64);
|
||||
if (crcDecoded != crcOrig) {
|
||||
if (findErrorPos) locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
|
||||
return LZ4F_ERROR_contentChecksum_invalid;
|
||||
if (shouldSucceed) DISPLAYLEVEL(1, "checksum error detected at end of decompression \n");
|
||||
if (shouldSucceed) locateBuffDiff(srcRef, dst, decompressedSize, o_scenario);
|
||||
RETURN_ERROR(contentChecksum_invalid);
|
||||
} }
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/* @return : 0 on success, or LZ4F_ErrorCode */
|
||||
size_t test_lz4f_decompression(const void* cSrc, size_t cSize,
|
||||
const void* srcRef, size_t decompressedSize,
|
||||
U64 crcOrig,
|
||||
@@ -1095,7 +1116,7 @@ size_t test_lz4f_decompression(const void* cSrc, size_t cSize,
|
||||
o_scenario_e const o_scenario = (o_scenario_e)(FUZ_rand(randState) % 3); /* 0 : contiguous; 1 : non-contiguous; 2 : dst overwritten */
|
||||
/* tighten dst buffer conditions */
|
||||
size_t const dstCapacity = (o_scenario == o_noncontiguous) ?
|
||||
(decompressedSize * 2) + 128 :
|
||||
(decompressedSize * 2) + 128 : /* should provide enough room to insert spaces beween blocks */
|
||||
decompressedSize;
|
||||
size_t result;
|
||||
void* const dstBuffer = malloc(dstCapacity);
|
||||
@@ -1152,7 +1173,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
|
||||
size_t const srcStartId = FUZ_rand(&randState) % (CNBufferLength - srcSize);
|
||||
const BYTE* const srcStart = (const BYTE*)CNBuffer + srcStartId;
|
||||
unsigned const neverFlush = (FUZ_rand(&randState) & 15) == 1;
|
||||
U64 const crcOrig = XXH64(srcStart, srcSize, 1);
|
||||
U64 const crcOrig = XXH64(srcStart, srcSize, extCrcSeed);
|
||||
LZ4F_preferences_t prefs;
|
||||
const LZ4F_preferences_t* prefsPtr = &prefs;
|
||||
size_t cSize;
|
||||
@@ -1200,11 +1221,9 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
|
||||
size_t iSize = MIN(sampleMax, (size_t)(iend-ip));
|
||||
size_t const oSize = LZ4F_compressBound(iSize, prefsPtr);
|
||||
cOptions.stableSrc = ((FUZ_rand(&randState) & 3) == 1);
|
||||
DISPLAYLEVEL(6, "Sending %u bytes to compress (stableSrc:%u) \n",
|
||||
(unsigned)iSize, cOptions.stableSrc);
|
||||
|
||||
#if 1
|
||||
/* insert uncompressed segment */
|
||||
/* test inserting an uncompressed block */
|
||||
if ( (iSize>0)
|
||||
&& !neverFlush /* do not mess with compressBound when neverFlush is set */
|
||||
&& prefsPtr != NULL /* prefs are set */
|
||||
@@ -1212,6 +1231,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
|
||||
&& (FUZ_rand(&randState) & 15) == 1 ) {
|
||||
size_t const uSize = FUZ_rand(&randState) % iSize;
|
||||
size_t const flushedSize = LZ4F_uncompressedUpdate(cCtx, op, (size_t)(oend-op), ip, uSize, &cOptions);
|
||||
DISPLAYLEVEL(6, "Actually sending %u bytes as uncompressed \n", (unsigned)uSize);
|
||||
CHECK(LZ4F_isError(flushedSize), "Insert uncompressed data failed (error %i : %s)",
|
||||
(int)flushedSize, LZ4F_getErrorName(flushedSize));
|
||||
op += flushedSize;
|
||||
@@ -1220,6 +1240,8 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
|
||||
}
|
||||
#endif
|
||||
|
||||
DISPLAYLEVEL(6, "Sending %u bytes to compress or buffer (stableSrc:%u) \n",
|
||||
(unsigned)iSize, cOptions.stableSrc);
|
||||
{ size_t const flushedSize = LZ4F_compressUpdate(cCtx, op, oSize, ip, iSize, &cOptions);
|
||||
CHECK(LZ4F_isError(flushedSize), "Compression failed (error %i : %s)",
|
||||
(int)flushedSize, LZ4F_getErrorName(flushedSize));
|
||||
@@ -1230,7 +1252,7 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
|
||||
{ unsigned const forceFlush = neverFlush ? 0 : ((FUZ_rand(&randState) & 3) == 1);
|
||||
if (forceFlush) {
|
||||
size_t const flushSize = LZ4F_flush(cCtx, op, (size_t)(oend-op), &cOptions);
|
||||
DISPLAYLEVEL(6,"flushing %u bytes \n", (unsigned)flushSize);
|
||||
DISPLAYLEVEL(6, "flushing %u bytes \n", (unsigned)flushSize);
|
||||
CHECK(LZ4F_isError(flushSize), "Compression failed (error %i)", (int)flushSize);
|
||||
op += flushSize;
|
||||
if ((FUZ_rand(&randState) % 1024) == 3) {
|
||||
@@ -1244,7 +1266,10 @@ int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressi
|
||||
op += 4;
|
||||
} } } }
|
||||
} /* while (ip<iend) */
|
||||
CHECK(op>=oend, "LZ4F_compressFrameBound overflow");
|
||||
|
||||
/* check compressBound guarantees */
|
||||
if (neverFlush) CHECK(op>=oend, "LZ4F_compressFrameBound overflow");
|
||||
|
||||
{ size_t const dstEndSafeSize = LZ4F_compressBound(0, prefsPtr);
|
||||
int const tooSmallDstEnd = ((FUZ_rand(&randState) & 31) == 3);
|
||||
size_t const dstEndTooSmallSize = (FUZ_rand(&randState) % dstEndSafeSize) + 1;
|
||||
|
232
tests/fuzzer.c
232
tests/fuzzer.c
@@ -309,6 +309,10 @@ static void FUZ_findDiff(const void* buff1, const void* buff2)
|
||||
const BYTE* const b1 = (const BYTE*)buff1;
|
||||
const BYTE* const b2 = (const BYTE*)buff2;
|
||||
size_t u = 0;
|
||||
if ((buff1 == NULL) || (buff2 == NULL)) {
|
||||
DISPLAY(" one of the buffers is NULL \n");
|
||||
return;
|
||||
}
|
||||
while (b1[u]==b2[u]) u++;
|
||||
DISPLAY("\nWrong Byte at position %u \n", (unsigned)u);
|
||||
}
|
||||
@@ -375,13 +379,13 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
cycleNb++) {
|
||||
U32 testNb = 0;
|
||||
U32 randState = FUZ_rand(&coreRandState) ^ PRIME3;
|
||||
int const blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1)) + 1;
|
||||
int const blockSize = (FUZ_rand(&randState) % (FUZ_MAX_BLOCK_SIZE-1));
|
||||
int const blockStart = (int)(FUZ_rand(&randState) % (U32)(COMPRESSIBLE_NOISE_LENGTH - blockSize - 1)) + 1;
|
||||
int const dictSizeRand = FUZ_rand(&randState) % FUZ_MAX_DICT_SIZE;
|
||||
int const dictSize = MIN(dictSizeRand, blockStart - 1);
|
||||
int const compressionLevel = FUZ_rand(&randState) % (LZ4HC_CLEVEL_MAX+1);
|
||||
const char* block = ((char*)CNBuffer) + blockStart;
|
||||
const char* dict = block - dictSize;
|
||||
const char* block = blockSize ? ((char*)CNBuffer) + blockStart : NULL;
|
||||
const char* dict = (char*)CNBuffer + blockStart - dictSize;
|
||||
int compressedSize, HCcompressedSize;
|
||||
int blockContinueCompressedSize;
|
||||
U32 const crcOrig = XXH32(block, (size_t)blockSize, 0);
|
||||
@@ -392,7 +396,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
/* Compression tests */
|
||||
if ( ((FUZ_rand(&randState) & 63) == 2)
|
||||
&& ((size_t)blockSize < labSize) ) {
|
||||
memcpy(lowAddrBuffer, block, blockSize);
|
||||
if (blockSize) { assert(block); memcpy(lowAddrBuffer, block, blockSize); }
|
||||
block = (const char*)lowAddrBuffer;
|
||||
}
|
||||
|
||||
@@ -448,7 +452,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
FUZ_DISPLAYTEST();
|
||||
decodedBuffer[srcSize] = canary;
|
||||
{ int const dSize = LZ4_decompress_safe(compressedBuffer, decodedBuffer, cSize, srcSize);
|
||||
FUZ_CHECKTEST(dSize<0, "LZ4_decompress_safe failed (%i) on data compressed by LZ4_compressHC_destSize", dSize);
|
||||
FUZ_CHECKTEST(dSize<0, "LZ4_decompress_safe failed (%i) on data compressed by LZ4_compress_HC_destSize", dSize);
|
||||
FUZ_CHECKTEST(dSize!=srcSize, "LZ4_decompress_safe failed : decompressed %i bytes, was supposed to decompress %i bytes", dSize, srcSize);
|
||||
}
|
||||
FUZ_CHECKTEST(decodedBuffer[srcSize] != canary, "LZ4_decompress_safe overwrite dst buffer !");
|
||||
@@ -487,16 +491,20 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
|
||||
FUZ_DISPLAYTEST("test LZ4_compress_destSize_extState() with too small dest buffer (must succeed, compress less than full input)");
|
||||
{ int inputSize = blockSize;
|
||||
int const r3 = LZ4_compress_destSize_extState(stateLZ4, block, compressedBuffer, &inputSize, r-1, 8);
|
||||
/* note: LZ4_compress_destSize() still requires a targetDstSize > 0, since it's the minimum valid output size produced by LZ4 */
|
||||
int targetDstSize = r-1 < 1 ? 1 : r-1;
|
||||
int const r3 = LZ4_compress_destSize_extState(stateLZ4, block, compressedBuffer, &inputSize, targetDstSize, 8);
|
||||
FUZ_CHECKTEST(r3==0, "LZ4_compress_destSize_extState() failed");
|
||||
FUZ_CHECKTEST(inputSize>=blockSize, "LZ4_compress_destSize_extState() should consume less than full input");
|
||||
if (targetDstSize < r)
|
||||
FUZ_CHECKTEST(inputSize>=blockSize, "LZ4_compress_destSize_extState() should consume less than full input");
|
||||
}
|
||||
}
|
||||
|
||||
/* Test compression using fast reset external state*/
|
||||
FUZ_DISPLAYTEST("test LZ4_compress_fast_extState_fastReset()");
|
||||
{ int const r = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
|
||||
FUZ_CHECKTEST(r==0, "LZ4_compress_fast_extState_fastReset() failed"); }
|
||||
FUZ_CHECKTEST(r==0, "LZ4_compress_fast_extState_fastReset() failed");
|
||||
}
|
||||
|
||||
/* Test compression */
|
||||
FUZ_DISPLAYTEST("test LZ4_compress_default()");
|
||||
@@ -529,12 +537,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
}
|
||||
|
||||
/* Test decoding with one byte missing => must fail */
|
||||
FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short");
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
{ int const r = LZ4_decompress_fast(cBuffer_exact, decodedBuffer, blockSize-1);
|
||||
FUZ_CHECKTEST(r>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
|
||||
if (blockSize >= 1) {
|
||||
FUZ_DISPLAYTEST("LZ4_decompress_fast() with output buffer 1-byte too short");
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
{ int const r = LZ4_decompress_fast(cBuffer_exact, decodedBuffer, blockSize-1);
|
||||
FUZ_CHECKTEST(r>=0, "LZ4_decompress_fast should have failed, due to Output Size being too small");
|
||||
}
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1]!=0, "LZ4_decompress_fast overrun specified output buffer");
|
||||
}
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1]!=0, "LZ4_decompress_fast overrun specified output buffer");
|
||||
|
||||
/* Test decoding with one byte too much => must fail */
|
||||
FUZ_DISPLAYTEST();
|
||||
@@ -568,12 +578,14 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
}
|
||||
|
||||
/* Test decoding with output size being one byte too short => must fail */
|
||||
FUZ_DISPLAYTEST();
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
{ int const r = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize-1);
|
||||
FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
|
||||
if (blockSize >= 1) {
|
||||
FUZ_DISPLAYTEST();
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
{ int const r = LZ4_decompress_safe(cBuffer_exact, decodedBuffer, compressedSize, blockSize-1);
|
||||
FUZ_CHECKTEST(r>=0, "LZ4_decompress_safe should have failed, due to Output Size being one byte too short");
|
||||
}
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
|
||||
}
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe overrun specified output buffer size");
|
||||
|
||||
/* Test decoding with output size being 10 bytes too short => must fail */
|
||||
FUZ_DISPLAYTEST();
|
||||
@@ -638,58 +650,60 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
}
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize], "LZ4_decompress_safe overrun specified output buffer size");
|
||||
|
||||
/* Test partial decoding => must work */
|
||||
FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial");
|
||||
{ size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
|
||||
int const targetSize = (int)((size_t)blockSize - missingOutBytes);
|
||||
size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
|
||||
int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
|
||||
char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
|
||||
int const decResult = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial failed despite valid input data (error:%i)", decResult);
|
||||
FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial: corruption detected in regenerated data");
|
||||
}
|
||||
if (blockSize) {
|
||||
/* Test partial decoding => must work */
|
||||
FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial");
|
||||
{ size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
|
||||
int const targetSize = (int)((size_t)blockSize - missingOutBytes);
|
||||
size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
|
||||
int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
|
||||
char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
|
||||
int const decResult = LZ4_decompress_safe_partial(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial failed despite valid input data (error:%i)", decResult);
|
||||
FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial: corruption detected in regenerated data");
|
||||
}
|
||||
|
||||
/* Partial decompression using dictionary. */
|
||||
FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict using no dict");
|
||||
{ size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
|
||||
int const targetSize = (int)((size_t)blockSize - missingOutBytes);
|
||||
size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
|
||||
int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
|
||||
char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
|
||||
int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, NULL, 0);
|
||||
FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
|
||||
FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
|
||||
}
|
||||
/* Partial decompression using dictionary. */
|
||||
FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict using no dict");
|
||||
{ size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
|
||||
int const targetSize = (int)((size_t)blockSize - missingOutBytes);
|
||||
size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
|
||||
int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
|
||||
char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
|
||||
int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, NULL, 0);
|
||||
FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
|
||||
FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
|
||||
}
|
||||
|
||||
FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict() using prefix as dict");
|
||||
{ size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
|
||||
int const targetSize = (int)((size_t)blockSize - missingOutBytes);
|
||||
size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
|
||||
int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
|
||||
char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
|
||||
int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, decodedBuffer, dictSize);
|
||||
FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
|
||||
FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
|
||||
}
|
||||
FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict() using prefix as dict");
|
||||
{ size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
|
||||
int const targetSize = (int)((size_t)blockSize - missingOutBytes);
|
||||
size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
|
||||
int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
|
||||
char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
|
||||
int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, decodedBuffer, dictSize);
|
||||
FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
|
||||
FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
|
||||
}
|
||||
|
||||
FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict() using external dict");
|
||||
{ size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
|
||||
int const targetSize = (int)((size_t)blockSize - missingOutBytes);
|
||||
size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
|
||||
int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
|
||||
char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
|
||||
int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, dict, dictSize);
|
||||
FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
|
||||
FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
|
||||
FUZ_DISPLAYTEST("test LZ4_decompress_safe_partial_usingDict() using external dict");
|
||||
{ size_t const missingOutBytes = FUZ_rand(&randState) % (unsigned)blockSize;
|
||||
int const targetSize = (int)((size_t)blockSize - missingOutBytes);
|
||||
size_t const extraneousInBytes = FUZ_rand(&randState) % 2;
|
||||
int const inCSize = (int)((size_t)compressedSize + extraneousInBytes);
|
||||
char const sentinel = decodedBuffer[targetSize] = block[targetSize] ^ 0x5A;
|
||||
int const decResult = LZ4_decompress_safe_partial_usingDict(compressedBuffer, decodedBuffer, inCSize, targetSize, blockSize, dict, dictSize);
|
||||
FUZ_CHECKTEST(decResult<0, "LZ4_decompress_safe_partial_usingDict failed despite valid input data (error:%i)", decResult);
|
||||
FUZ_CHECKTEST(decResult != targetSize, "LZ4_decompress_safe_partial_usingDict did not regenerated required amount of data (%i < %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(decodedBuffer[targetSize] != sentinel, "LZ4_decompress_safe_partial_usingDict overwrite beyond requested size (though %i <= %i <= %i)", decResult, targetSize, blockSize);
|
||||
FUZ_CHECKTEST(memcmp(block, decodedBuffer, (size_t)targetSize), "LZ4_decompress_safe_partial_usingDict: corruption detected in regenerated data");
|
||||
}
|
||||
}
|
||||
|
||||
/* Test Compression with limited output size */
|
||||
@@ -754,7 +768,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
|
||||
/* Decompress with dictionary as prefix */
|
||||
FUZ_DISPLAYTEST("test LZ4_decompress_fast_usingDict() with dictionary as prefix");
|
||||
memcpy(decodedBuffer, dict, dictSize);
|
||||
if (dictSize) memcpy(decodedBuffer, dict, dictSize);
|
||||
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer+dictSize, blockSize, decodedBuffer, dictSize);
|
||||
FUZ_CHECKTEST(ret!=blockContinueCompressedSize, "LZ4_decompress_fast_usingDict did not read all compressed block input");
|
||||
{ U32 const crcCheck = XXH32(decodedBuffer+dictSize, (size_t)blockSize, 0);
|
||||
@@ -780,6 +794,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
|
||||
FUZ_DISPLAYTEST("LZ4_compress_fast_continue() with dictionary and output buffer too short by one byte");
|
||||
LZ4_loadDict(&LZ4dictBody, dict, dictSize);
|
||||
assert(blockContinueCompressedSize >= 1);
|
||||
ret = LZ4_compress_fast_continue(&LZ4dictBody, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
|
||||
FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using ExtDict should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
|
||||
|
||||
@@ -814,17 +829,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
|
||||
}
|
||||
|
||||
FUZ_DISPLAYTEST();
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
|
||||
if (blockSize >= 1) {
|
||||
FUZ_DISPLAYTEST();
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
|
||||
|
||||
FUZ_DISPLAYTEST();
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
|
||||
FUZ_DISPLAYTEST();
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
|
||||
}
|
||||
|
||||
FUZ_DISPLAYTEST();
|
||||
{ int const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
|
||||
@@ -864,6 +881,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
FUZ_DISPLAYTEST("LZ4_compress_fast_continue() after LZ4_attach_dictionary(), but output buffer is 1 byte too short");
|
||||
LZ4_resetStream_fast(&LZ4_stream);
|
||||
LZ4_attach_dictionary(&LZ4_stream, &LZ4dictBody);
|
||||
assert(blockContinueCompressedSize >= 1);
|
||||
ret = LZ4_compress_fast_continue(&LZ4_stream, block, compressedBuffer, blockSize, blockContinueCompressedSize-1, 1);
|
||||
FUZ_CHECKTEST(ret>0, "LZ4_compress_fast_continue using extDictCtx should fail : one missing byte for output buffer : %i written, %i buffer", ret, blockContinueCompressedSize);
|
||||
/* note : context is no longer dirty after a failed compressed block */
|
||||
@@ -908,17 +926,19 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
FUZ_CHECKTEST(crcCheck!=crcOrig, "LZ4_decompress_safe_usingDict corrupted decoded data");
|
||||
}
|
||||
|
||||
FUZ_DISPLAYTEST();
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
|
||||
if (blockSize >= 1) {
|
||||
FUZ_DISPLAYTEST();
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
ret = LZ4_decompress_fast_usingDict(compressedBuffer, decodedBuffer, blockSize-1, dict, dictSize);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_fast_usingDict should have failed : wrong original size (-1 byte)");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_fast_usingDict overrun specified output buffer size");
|
||||
|
||||
FUZ_DISPLAYTEST();
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
|
||||
FUZ_DISPLAYTEST();
|
||||
decodedBuffer[blockSize-1] = 0;
|
||||
ret = LZ4_decompress_safe_usingDict(compressedBuffer, decodedBuffer, blockContinueCompressedSize, blockSize-1, dict, dictSize);
|
||||
FUZ_CHECKTEST(ret>=0, "LZ4_decompress_safe_usingDict should have failed : not enough output size (-1 byte)");
|
||||
FUZ_CHECKTEST(decodedBuffer[blockSize-1], "LZ4_decompress_safe_usingDict overrun specified output buffer size");
|
||||
}
|
||||
|
||||
FUZ_DISPLAYTEST("LZ4_decompress_safe_usingDict with a too small output buffer");
|
||||
{ int const missingBytes = (FUZ_rand(&randState) & 0xF) + 2;
|
||||
@@ -941,6 +961,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
|
||||
FUZ_DISPLAYTEST("LZ4_compress_HC_continue with same external dictionary, but output buffer 1 byte too short");
|
||||
LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
|
||||
assert(blockContinueCompressedSize>=1);
|
||||
ret = LZ4_compress_HC_continue(LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
|
||||
FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDict should fail : one missing byte for output buffer (expected %i, but result=%i)", blockContinueCompressedSize, ret);
|
||||
/* note : context is no longer dirty after a failed compressed block */
|
||||
@@ -977,6 +998,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
FUZ_DISPLAYTEST();
|
||||
LZ4_resetStreamHC_fast (LZ4_streamHC, compressionLevel);
|
||||
LZ4_attach_HC_dictionary(LZ4_streamHC, LZ4dictHC);
|
||||
assert(blockContinueCompressedSize>=1);
|
||||
ret = LZ4_compress_HC_continue(LZ4_streamHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
|
||||
FUZ_CHECKTEST(ret>0, "LZ4_compress_HC_continue using ExtDictCtx should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
|
||||
/* note : context is no longer dirty after a failed compressed block */
|
||||
@@ -1014,7 +1036,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
|
||||
|
||||
/* Compress HC continue destSize */
|
||||
FUZ_DISPLAYTEST();
|
||||
{ int const availableSpace = (int)(FUZ_rand(&randState) % (U32)blockSize) + 5;
|
||||
{ int const availableSpace = (blockSize ? (int)(FUZ_rand(&randState) % (U32)blockSize) : 0) + 5;
|
||||
int consumedSize = blockSize;
|
||||
FUZ_DISPLAYTEST();
|
||||
LZ4_loadDictHC(LZ4dictHC, dict, dictSize);
|
||||
@@ -1297,6 +1319,30 @@ static void FUZ_unitTests(int compressionLevel)
|
||||
} }
|
||||
}
|
||||
|
||||
DISPLAYLEVEL(3, "LZ4_compress_HC() with NULL input:");
|
||||
{ int const maxCSize = LZ4_compressBound(0);
|
||||
int level;
|
||||
for (level=LZ4HC_CLEVEL_MIN; level <= LZ4HC_CLEVEL_MAX; level++) {
|
||||
int const cSize = LZ4_compress_HC(NULL, testCompressed, 0, maxCSize, level);
|
||||
FUZ_CHECKTEST(!(cSize==1 && testCompressed[0]==0),
|
||||
"compressing empty should give byte 0"
|
||||
" (maxCSize == %i) (cSize == %i) (byte == 0x%02X)",
|
||||
maxCSize, cSize, testCompressed[0]);
|
||||
}
|
||||
}
|
||||
DISPLAYLEVEL(3, " OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "LZ4_compress_HC() with both NULL input and output:");
|
||||
{ int level;
|
||||
for (level=LZ4HC_CLEVEL_MIN; level <= LZ4HC_CLEVEL_MAX; level++) {
|
||||
int const cSize = LZ4_compress_HC(NULL, NULL, 0, 0, level);
|
||||
FUZ_CHECKTEST(cSize != 0,
|
||||
"compressing into NULL must fail"
|
||||
" (cSize == %i != 0)", cSize);
|
||||
}
|
||||
}
|
||||
DISPLAYLEVEL(3, " OK \n");
|
||||
|
||||
DISPLAYLEVEL(3, "LZ4_initStreamHC with multiple valid alignments : ");
|
||||
{ typedef struct {
|
||||
LZ4_streamHC_t hc1;
|
||||
@@ -1420,10 +1466,10 @@ static void FUZ_unitTests(int compressionLevel)
|
||||
LZ4_resetStreamHC_fast(&sHC, compressionLevel);
|
||||
LZ4_loadDictHC(&sHC, testInput, segSize);
|
||||
result1 = LZ4_compress_HC_continue(&sHC, testInput + segSize, testCompressed, segSize, segSize -1);
|
||||
FUZ_CHECKTEST(result1==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result1);
|
||||
FUZ_CHECKTEST(result1==0, "LZ4_compress_HC_continue() dictionary compression failed : result = %i", result1);
|
||||
FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
|
||||
result2 = LZ4_compress_HC_continue(&sHC, testInput + 2*(size_t)segSize, testCompressed+result1, segSize, segSize-1);
|
||||
FUZ_CHECKTEST(result2==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result2);
|
||||
FUZ_CHECKTEST(result2==0, "LZ4_compress_HC_continue() dictionary compression failed : result = %i", result2);
|
||||
FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
|
||||
|
||||
result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result1, segSize, testInput, segSize);
|
||||
@@ -1439,7 +1485,7 @@ static void FUZ_unitTests(int compressionLevel)
|
||||
LZ4_resetStreamHC_fast(&sHC, compressionLevel);
|
||||
LZ4_loadDictHC(&sHC, testInput, 32 KB);
|
||||
result = LZ4_compress_HC_continue(&sHC, testInput + 64 KB, testCompressed, testCompressedSize, testCompressedSize-1);
|
||||
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() remote dictionary failed : result = %i", result);
|
||||
FUZ_CHECKTEST(result==0, "LZ4_compress_HC_continue() remote dictionary failed : result = %i", result);
|
||||
FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
|
||||
|
||||
result = LZ4_decompress_safe_usingDict(testCompressed, testVerify, result, testCompressedSize, testInput, 32 KB);
|
||||
@@ -1471,7 +1517,7 @@ static void FUZ_unitTests(int compressionLevel)
|
||||
crcOrig = XXH64_digest(&crcOrigState);
|
||||
assert(segSize <= INT_MAX);
|
||||
result = LZ4_compress_HC_continue(&sHC, testInput + segStart, testCompressed, (int)segSize, LZ4_compressBound((int)segSize));
|
||||
FUZ_CHECKTEST(result==0, "LZ4_compressHC_limitedOutput_continue() dictionary compression failed : result = %i", result);
|
||||
FUZ_CHECKTEST(result==0, "LZ4_compress_HC_continue() dictionary compression failed : result = %i", result);
|
||||
FUZ_CHECKTEST(sHC.internal_donotuse.dirty, "Context should be clean");
|
||||
|
||||
result = LZ4_decompress_safe_usingDict(testCompressed, dst, result, (int)segSize, dict, (int)dictSize);
|
||||
|
Reference in New Issue
Block a user