josh/lz4
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1446 from lz4/listStdout

Browse Source 
Ensure --list output is sent to stdout
This commit is contained in:
Yann Collet
2024-07-15 16:31:55 -07:00
committed by GitHub
parent f580770cf6 b7062669e6
commit 8cde298b62
4 changed files with 42 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
programs/lz4.1.md
View File

@@ -113,7 +113,8 @@ only the latest one will be applied.
* `--list`:
List information about .lz4 files.
note : current implementation is limited to single-frame .lz4 files.
For detailed information on files with multiple frames, use `-v`.
`--list` automatically triggers `-m` modifier.
### Operation modifiers

2
programs/lz4cli.c
View File

@@ -474,7 +474,7 @@ int main(int argCount, const char** argv)
if (!strcmp(argument, "--no-crc")) { LZ4IO_setStreamChecksumMode(prefs, 0); LZ4IO_setBlockChecksumMode(prefs, 0); BMK_skipChecksums(1); continue; }
if (!strcmp(argument, "--content-size")) { LZ4IO_setContentSize(prefs, 1); continue; }
if (!strcmp(argument, "--no-content-size")) { LZ4IO_setContentSize(prefs, 0); continue; }
if (!strcmp(argument, "--list")) { mode = om_list; continue; }
if (!strcmp(argument, "--list")) { mode = om_list; multiple_inputs = 1; continue; }
if (!strcmp(argument, "--sparse")) { LZ4IO_setSparseFile(prefs, 2); continue; }
if (!strcmp(argument, "--no-sparse")) { LZ4IO_setSparseFile(prefs, 0); continue; }
if (!strcmp(argument, "--favor-decSpeed")) { LZ4IO_favorDecSpeed(prefs, 1); continue; }

72
programs/lz4io.c
View File

@@ -2705,7 +2705,7 @@ static const char* LZ4IO_baseName(const char* input_filename)
* + report nb of blocks, hence max. possible decompressed size (when not reported in header)
*/
static LZ4IO_infoResult
LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filename)
LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filename, int displayNow)
{
LZ4IO_infoResult result = LZ4IO_format_not_known; /* default result (error) */
unsigned char buffer[LZ4F_HEADER_SIZE_MAX];
@@ -2758,23 +2758,22 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
if (totalBlocksSize) {
char bTypeBuffer[5];
LZ4IO_blockTypeID(frameInfo.lz4FrameInfo.blockSizeID, frameInfo.lz4FrameInfo.blockMode, bTypeBuffer);
DISPLAYLEVEL(3, " %6llu %14s %5s %8s",
if (displayNow) DISPLAYOUT(" %6llu %14s %5s %8s",
cfinfo->frameCount + 1,
LZ4IO_frameTypeNames[frameInfo.frameType],
bTypeBuffer,
frameInfo.lz4FrameInfo.contentChecksumFlag ? "XXH32" : "-");
if (frameInfo.lz4FrameInfo.contentSize) {
{ double const ratio = (double)(totalBlocksSize + hSize) / (double)frameInfo.lz4FrameInfo.contentSize * 100;
DISPLAYLEVEL(3, " %20llu %20llu %9.2f%%\n",
totalBlocksSize + hSize,
frameInfo.lz4FrameInfo.contentSize,
ratio);
}
double const ratio = (double)(totalBlocksSize + hSize) / (double)frameInfo.lz4FrameInfo.contentSize * 100;
if (displayNow) DISPLAYOUT(" %20llu %20llu %9.2f%%\n",
totalBlocksSize + hSize,
frameInfo.lz4FrameInfo.contentSize,
ratio);
/* Now we've consumed frameInfo we can use it to store the total contentSize */
frameInfo.lz4FrameInfo.contentSize += cfinfo->frameSummary.lz4FrameInfo.contentSize;
}
else {
DISPLAYLEVEL(3, " %20llu %20s %9s \n", totalBlocksSize + hSize, "-", "-");
if (displayNow) DISPLAYOUT(" %20llu %20s %9s \n", totalBlocksSize + hSize, "-", "-");
cfinfo->allContentSize = 0;
}
result = LZ4IO_LZ4F_OK;
@@ -2792,7 +2791,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
break;
}
if (totalBlocksSize) {
DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20llu %20s %9s\n",
if (displayNow) DISPLAYOUT(" %6llu %14s %5s %8s %20llu %20s %9s\n",
cfinfo->frameCount + 1,
LZ4IO_frameTypeNames[frameInfo.frameType],
"-", "-",
@@ -2814,7 +2813,7 @@ LZ4IO_getCompressedFileInfo(LZ4IO_cFileInfo_t* cfinfo, const char* input_filenam
int const errorNb = fseek_u32(finput, size, SEEK_CUR);
if (errorNb != 0)
END_PROCESS(43, "Stream error : cannot skip skippable area");
DISPLAYLEVEL(3, " %6llu %14s %5s %8s %20u %20s %9s\n",
if (displayNow) DISPLAYOUT(" %6llu %14s %5s %8s %20u %20s %9s\n",
cfinfo->frameCount + 1,
"SkippableFrame",
"-", "-", size + 8, "-", "-");
@@ -2846,7 +2845,7 @@ int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx)
int result = 0;
size_t idx = 0;
if (g_displayLevel < 3) {
DISPLAYOUT("%10s %14s %5s %11s %13s %9s %s\n",
DISPLAYOUT("%10s %14s %5s %11s %13s %8s %s\n",
"Frames", "Type", "Block", "Compressed", "Uncompressed", "Ratio", "Filename");
}
for (; idx < ifnIdx; idx++) {
@@ -2857,34 +2856,39 @@ int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx)
DISPLAYLEVEL(1, "lz4: %s is not a regular file \n", inFileNames[idx]);
return 1;
}
DISPLAYLEVEL(3, "%s(%llu/%llu)\n", cfinfo.fileName, (unsigned long long)idx + 1, (unsigned long long)ifnIdx);
DISPLAYLEVEL(3, " %6s %14s %5s %8s %20s %20s %9s\n",
"Frame", "Type", "Block", "Checksum", "Compressed", "Uncompressed", "Ratio")
{ LZ4IO_infoResult const op_result = LZ4IO_getCompressedFileInfo(&cfinfo, inFileNames[idx]);
if (g_displayLevel >= 3) {
/* verbose mode */
DISPLAYOUT("%s(%llu/%llu)\n", cfinfo.fileName, (unsigned long long)idx + 1, (unsigned long long)ifnIdx);
DISPLAYOUT(" %6s %14s %5s %8s %20s %20s %9s\n",
"Frame", "Type", "Block", "Checksum", "Compressed", "Uncompressed", "Ratio");
}
{ LZ4IO_infoResult const op_result = LZ4IO_getCompressedFileInfo(&cfinfo, inFileNames[idx], g_displayLevel >= 3);
if (op_result != LZ4IO_LZ4F_OK) {
assert(op_result == LZ4IO_format_not_known);
DISPLAYLEVEL(1, "lz4: %s: File format not recognized \n", inFileNames[idx]);
return 1;
} }
DISPLAYLEVEL(3, "\n");
if (g_displayLevel >= 3) {
DISPLAYOUT("\n");
}
if (g_displayLevel < 3) {
/* Display Summary */
{ char buffers[3][10];
DISPLAYOUT("%10llu %14s %5s %11s %13s ",
cfinfo.frameCount,
cfinfo.eqFrameTypes ? LZ4IO_frameTypeNames[cfinfo.frameSummary.frameType] : "-" ,
cfinfo.eqBlockTypes ? LZ4IO_blockTypeID(cfinfo.frameSummary.lz4FrameInfo.blockSizeID,
cfinfo.frameSummary.lz4FrameInfo.blockMode, buffers[0]) : "-",
LZ4IO_toHuman((long double)cfinfo.fileSize, buffers[1]),
cfinfo.allContentSize ? LZ4IO_toHuman((long double)cfinfo.frameSummary.lz4FrameInfo.contentSize, buffers[2]) : "-");
if (cfinfo.allContentSize) {
double const ratio = (double)cfinfo.fileSize / (double)cfinfo.frameSummary.lz4FrameInfo.contentSize * 100;
DISPLAYOUT("%9.2f%% %s \n", ratio, cfinfo.fileName);
} else {
DISPLAYOUT("%9s %s\n",
"-",
cfinfo.fileName);
} } } /* if (g_displayLevel < 3) */
/* Display summary */
char buffers[3][10];
DISPLAYOUT("%10llu %14s %5s %11s %13s ",
cfinfo.frameCount,
cfinfo.eqFrameTypes ? LZ4IO_frameTypeNames[cfinfo.frameSummary.frameType] : "-" ,
cfinfo.eqBlockTypes ? LZ4IO_blockTypeID(cfinfo.frameSummary.lz4FrameInfo.blockSizeID,
cfinfo.frameSummary.lz4FrameInfo.blockMode, buffers[0]) : "-",
LZ4IO_toHuman((long double)cfinfo.fileSize, buffers[1]),
cfinfo.allContentSize ? LZ4IO_toHuman((long double)cfinfo.frameSummary.lz4FrameInfo.contentSize, buffers[2]) : "-");
if (cfinfo.allContentSize) {
double const ratio = (double)cfinfo.fileSize / (double)cfinfo.frameSummary.lz4FrameInfo.contentSize * 100;
DISPLAYOUT("%8.2f%% %s \n", ratio, cfinfo.fileName);
} else {
DISPLAYOUT("%8s %s\n",
"-",
cfinfo.fileName);
} } /* if (g_displayLevel < 3) */
} /* for (; idx < ifnIdx; idx++) */
return result;

24
tests/test-lz4-list.py
View File

@@ -138,24 +138,11 @@ class TestVerbose(unittest.TestCase):
self.cvinfo.file_frame_map = concat_file_list
self.cvinfo.compressed_size = os.path.getsize(f"{TEMP}/test_list_concat-all.lz4")
def test_filename(self):
for i, vinfo in enumerate(self.vinfo_list):
self.assertRegex(vinfo.filename, f"^test_list_.*({i + 1}/{len(self.vinfo_list)})")
def test_frame_number(self):
for vinfo in self.vinfo_list:
for i, frame_info in enumerate(vinfo.frame_list):
self.assertEqual(frame_info["frame"], str(i + 1), frame_info["line"])
def test_frame_type(self):
for i, frame_info in enumerate(self.cvinfo.frame_list):
if "-lz4f-" in self.cvinfo.file_frame_map[i]:
self.assertEqual(self.cvinfo.frame_list[i]["type"], "LZ4Frame", self.cvinfo.frame_list[i]["line"])
elif "-legc-" in self.cvinfo.file_frame_map[i]:
self.assertEqual(self.cvinfo.frame_list[i]["type"], "LegacyFrame", self.cvinfo.frame_list[i]["line"])
elif "-skip-" in self.cvinfo.file_frame_map[i]:
self.assertEqual(self.cvinfo.frame_list[i]["type"], "SkippableFrame", self.cvinfo.frame_list[i]["line"])
def test_block(self):
for i, frame_info in enumerate(self.cvinfo.frame_list):
if "--BD" in self.cvinfo.file_frame_map[i]:
@@ -168,15 +155,6 @@ class TestVerbose(unittest.TestCase):
if "-lz4f-" in self.cvinfo.file_frame_map[i] and "--no-frame-crc" not in self.cvinfo.file_frame_map[i]:
self.assertEqual(self.cvinfo.frame_list[i]["checksum"], "XXH32", self.cvinfo.frame_list[i]["line"])
def test_compressed(self):
total = 0
for i, frame_info in enumerate(self.cvinfo.frame_list):
if "-2f-" not in self.cvinfo.file_frame_map[i]:
expected_size = os.path.getsize(self.cvinfo.file_frame_map[i])
self.assertEqual(self.cvinfo.frame_list[i]["compressed"], str(expected_size), self.cvinfo.frame_list[i]["line"])
total += int(self.cvinfo.frame_list[i]["compressed"])
self.assertEqual(total, self.cvinfo.compressed_size, f"Expected total sum ({total}) to match {self.cvinfo.filename} filesize")
def test_uncompressed(self):
for i, frame_info in enumerate(self.cvinfo.frame_list):
ffm = self.cvinfo.file_frame_map[i]
@@ -227,7 +205,7 @@ def execute(command, print_command=True, print_output=False, print_error=True):
if stderr_lines and not print_output and print_error:
print(stderr_lines)
errout(f"Failed to run: {command}, {stdout_lines + stderr_lines}\n")
return (stdout_lines + stderr_lines).splitlines()
return (stdout_lines).splitlines()
def cleanup(silent=False):