josh/lz4
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed MT incompatibility with --content-size

Browse Source 
also:
provide MT-friendly time measurements for Linux/posix
This commit is contained in:
Yann Collet
2023-12-29 12:18:52 -08:00
parent b45de8b49b
commit 34e9895cde
2 changed files with 19 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
programs/lz4io.c
View File

@@ -115,7 +115,9 @@ static TIME_t g_time = { 0 };
static void LZ4IO_finalTimeDisplay(TIME_t timeStart, clock_t cpuStart, unsigned long long size)
{
#ifdef LZ4IO_MULTITHREAD
if (TIME_support_MT_measurements())
if (!TIME_support_MT_measurements()) {
DISPLAYLEVEL(5, "time measurements not compatible with multithreading \n");
} else
#endif
{
Duration_ns duration_ns = TIME_clockSpan_ns(timeStart);
@@ -1216,15 +1218,15 @@ LZ4IO_compressFilename_extRess_MT(unsigned long long* inStreamSize,
}
/* End of Frame mark */
{ size_t endSize = LZ4F_compressEnd(ctx, dstBuffer, dstBufferSize, NULL);
if (LZ4F_isError(endSize))
END_PROCESS(48, "End of frame error : %s", LZ4F_getErrorName(endSize));
{ size_t endSize = 4;
assert(dstBufferSize >= 8);
memset(dstBuffer, 0, 4);
if (checksum) {
/* handle frame checksum externally
* note: LZ4F_compressEnd already wrote a (bogus) checksum */
U32 const crc = XXH32_digest(xxh32);
assert(endSize >= 4);
LZ4IO_writeLE32( (char*)dstBuffer + endSize - 4, crc);
LZ4IO_writeLE32( (char*)dstBuffer + 4, crc);
endSize = 8;
}
if (fwrite(dstBuffer, 1, endSize, dstFile) != endSize)
END_PROCESS(49, "Write error : cannot write end of frame");
@@ -1412,8 +1414,7 @@ LZ4IO_compressFilename_extRess(unsigned long long* inStreamSize,
{
#if defined(LZ4IO_MULTITHREAD)
/* do NOT employ multi-threading in the following scenarios: */
if ( (io_prefs->contentSizeFlag) /* content size present in frame header*/
|| (io_prefs->blockIndependence == LZ4F_blockLinked) /* blocks are not independent */
if ( (io_prefs->blockIndependence == LZ4F_blockLinked) /* blocks are not independent */
|| (ress.cdict)) /* dictionary compression */
return LZ4IO_compressFilename_extRess_ST(inStreamSize, ress, srcFileName, dstFileName, compressionLevel, io_prefs);
@@ -1459,6 +1460,7 @@ int LZ4IO_compressMultipleFilenames(
unsigned long long totalProcessed = 0;
TIME_t timeStart = TIME_getTime();
clock_t cpuStart = clock();
DISPLAY("LZ4IO_compressMultipleFilenames (%i files) \n", ifntSize);
if (dstFileName == NULL) return ifntSize; /* not enough memory */
ress = LZ4IO_createCResources(prefs);

23
programs/timefn.c
View File

@@ -78,34 +78,30 @@ TIME_t TIME_getTime(void)
}
}
/* POSIX.1-2001 (optional) */
#elif defined(CLOCK_PROCESS_CPUTIME_ID)
#elif defined(CLOCK_MONOTONIC)
/* Note: CLOCK_PROCESS_CPUTIME_ID is only fine for single-thread performance,
* which is the scenario we focus on for the time being.
* For multi-threading performance, prefer CLOCK_MONOTONIC. */
# define TIME_MT_MEASUREMENTS_NOT_SUPPORTED
# include <stdio.h> /* perror */
# include <stdlib.h> /* abort */
#include <stdlib.h> /* abort */
#include <stdio.h> /* perror */
TIME_t TIME_getTime(void)
{
/* time must be initialized, othersize it may fail msan test.
* No good reason, likely a limitation of timespec_get() for some target */
struct timespec time = { 0, 0 };
if (clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &time) != 0) {
perror("timefn::clock_gettime(CLOCK_PROCESS_CPUTIME_ID)");
if (clock_gettime(CLOCK_MONOTONIC, &time) != 0) {
perror("timefn::clock_gettime(CLOCK_MONOTONIC)");
abort();
}
{
TIME_t r;
{ TIME_t r;
r.t = (Duration_ns)time.tv_sec * 1000000000ULL
+ (Duration_ns)time.tv_nsec;
return r;
}
}
/* C11 requires support of timespec_get().
* However, FreeBSD 11 claims C11 compliance while lacking timespec_get().
* Double confirm timespec_get() support by checking the definition of TIME_UTC.
@@ -119,8 +115,7 @@ TIME_t TIME_getTime(void)
TIME_t TIME_getTime(void)
{
/* time must be initialized, othersize it may fail msan test.
* No good reason, likely a limitation of timespec_get() for some target */
/* time must be initialized, othersize it may fail msan test */
struct timespec time = { 0, 0 };
if (timespec_get(&time, TIME_UTC) != TIME_UTC) {
perror("timefn::timespec_get(TIME_UTC)");