Yu Watanabe
dac2b39405
Previously, even when the compressed data was truncated, or had trailing garbage, the decompressor might not detect it. To fix the issue, update the check if stream is finished. With this change, now decompressor_push() must be called with size == 0 to finalize the stream.
707 lines
34 KiB
C
707 lines
34 KiB
C
/* SPDX-License-Identifier: LGPL-2.1-or-later */
|
|
|
|
#include <stdlib.h>
|
|
#include <sys/stat.h>
|
|
#include <unistd.h>
|
|
|
|
#include "alloc-util.h"
|
|
#include "argv-util.h"
|
|
#include "compress.h"
|
|
#include "copy.h"
|
|
#include "fd-util.h"
|
|
#include "io-util.h"
|
|
#include "path-util.h"
|
|
#include "random-util.h"
|
|
#include "tests.h"
|
|
#include "tmpfile-util.h"
|
|
|
|
#define HUGE_SIZE (4096*1024)
|
|
|
|
static const char text[] =
|
|
"text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
|
|
"foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
|
|
static char data[512] = "random\0";
|
|
static char *huge = NULL;
|
|
static const char *srcfile;
|
|
|
|
static const char* cat_for_compression(Compression c) {
|
|
switch (c) {
|
|
case COMPRESSION_XZ: return "xzcat";
|
|
case COMPRESSION_LZ4: return "lz4cat";
|
|
case COMPRESSION_ZSTD: return "zstdcat";
|
|
case COMPRESSION_GZIP: return "zcat";
|
|
case COMPRESSION_BZIP2: return "bzcat";
|
|
default: return NULL;
|
|
}
|
|
}
|
|
|
|
TEST(compress_decompress_blob) {
|
|
for (Compression c = 0; c < _COMPRESSION_MAX; c++) {
|
|
if (c == COMPRESSION_NONE || !compression_supported(c))
|
|
continue;
|
|
|
|
const char *label = compression_to_string(c);
|
|
|
|
for (size_t t = 0; t < 2; t++) {
|
|
const char *input = t == 0 ? text : data;
|
|
size_t input_len = t == 0 ? sizeof(text) : sizeof(data);
|
|
|
|
char compressed[sizeof(data) * 4];
|
|
size_t csize;
|
|
_cleanup_free_ char *decompressed = NULL;
|
|
|
|
log_info("/* testing %s %s blob compression/decompression */", label, input);
|
|
|
|
ASSERT_OK(compress_blob(c, input, input_len, compressed, sizeof(compressed), &csize, -1));
|
|
|
|
ASSERT_OK_ZERO(decompress_blob(c, compressed, csize, (void **) &decompressed, &csize, 0));
|
|
ASSERT_NOT_NULL(decompressed);
|
|
ASSERT_EQ(memcmp(decompressed, input, input_len), 0);
|
|
|
|
ASSERT_FAIL(decompress_blob(c, "garbage", 7, (void **) &decompressed, &csize, 0));
|
|
}
|
|
}
|
|
}
|
|
|
|
TEST(decompress_blob_zstd_frame_at_buffer_boundary) {
|
|
/* Regression test: a frame whose decompressed size lands exactly on the output buffer boundary
|
|
* used to be misreported as -EBADMSG. ZSTD_decompressStream() returns 0 (frame complete) just as
|
|
* the buffer fills, and the decompressor must recognize completion rather than grow the buffer
|
|
* and call the decoder again on the now-empty input, which it would misread as a truncated next
|
|
* frame. The checksum is stripped so that end-of-input coincides with frame completion (with a
|
|
* checksum the trailer is read after the content, hiding the boundary).
|
|
*
|
|
* decompress_blob() sizes its first output buffer to greedy_realloc(ZSTD_DStreamOutSize()) (the
|
|
* compressed content is tiny, so the src_size*2 term never dominates), so replicate that here and
|
|
* size the content to the buffer's exact usable size to make the frame finish precisely when it
|
|
* fills. The size is derived from ZSTD_DStreamOutSize() rather than hardcoded, so the coverage
|
|
* tracks the implementation instead of silently drifting if that value ever changes. */
|
|
|
|
if (!compression_supported(COMPRESSION_ZSTD))
|
|
return (void) log_tests_skipped("zstd is not supported");
|
|
|
|
size_t out_size = zstd_dstream_out_size();
|
|
ASSERT_GT(out_size, 0u);
|
|
|
|
_cleanup_free_ void *probe = NULL;
|
|
ASSERT_NOT_NULL(greedy_realloc(&probe, out_size, 1));
|
|
size_t input_size = MALLOC_SIZEOF_SAFE(probe);
|
|
|
|
_cleanup_free_ char *input = malloc(input_size);
|
|
ASSERT_NOT_NULL(input);
|
|
for (size_t i = 0; i < input_size; i++)
|
|
input[i] = 'A' + (i % 26);
|
|
|
|
_cleanup_close_ int src = -EBADF, dst = -EBADF;
|
|
_cleanup_(unlink_tempfilep) char
|
|
src_pattern[] = "/tmp/systemd-test.zstd-boundary-src.XXXXXX",
|
|
dst_pattern[] = "/tmp/systemd-test.zstd-boundary.XXXXXX";
|
|
struct stat st;
|
|
|
|
ASSERT_OK(src = mkostemp_safe(src_pattern));
|
|
ASSERT_OK(loop_write(src, input, input_size));
|
|
ASSERT_EQ(lseek(src, 0, SEEK_SET), (off_t) 0);
|
|
ASSERT_OK(dst = mkostemp_safe(dst_pattern));
|
|
ASSERT_OK(compress_stream(COMPRESSION_ZSTD, src, dst, UINT64_MAX, NULL));
|
|
|
|
ASSERT_OK_ERRNO(fstat(dst, &st));
|
|
_cleanup_free_ uint8_t *compressed = malloc(st.st_size);
|
|
ASSERT_NOT_NULL(compressed);
|
|
ASSERT_EQ(lseek(dst, 0, SEEK_SET), (off_t) 0);
|
|
ASSERT_EQ(loop_read(dst, compressed, st.st_size, true), (ssize_t) st.st_size);
|
|
|
|
/* This relies on compress_stream() emitting a content checksum (bit 2 of the frame header
|
|
* descriptor, with a 4-byte trailer). Assert that assumption rather than silently dropping
|
|
* structural bytes should it ever change. */
|
|
ASSERT_TRUE(compressed[4] & 0x04);
|
|
compressed[4] &= ~0x04; /* Clear the checksum flag and drop the trailer below. */
|
|
|
|
_cleanup_free_ void *decompressed = NULL;
|
|
size_t dsize = 0;
|
|
ASSERT_OK_ZERO(decompress_blob(COMPRESSION_ZSTD, compressed, st.st_size - 4, &decompressed, &dsize, 0));
|
|
ASSERT_EQ(dsize, input_size);
|
|
ASSERT_EQ(memcmp(decompressed, input, input_size), 0);
|
|
}
|
|
|
|
TEST(decompress_blob_zstd_unknown_size) {
|
|
/* Frames produced by the streaming compressor — like those the 'zstd' CLI and compressed kernel
|
|
* images use — don't record the decompressed size in the frame header, so
|
|
* ZSTD_getFrameContentSize() returns ZSTD_CONTENTSIZE_UNKNOWN. Make sure decompress_blob() and
|
|
* decompress_startswith() handle that instead of bailing out with -EBADMSG. */
|
|
|
|
if (!compression_supported(COMPRESSION_ZSTD))
|
|
return (void) log_tests_skipped("zstd is not supported");
|
|
|
|
_cleanup_free_ char *input = NULL;
|
|
size_t input_size = U64_MB; /* > ZSTD_DStreamOutSize(), so the output buffer must grow
|
|
* several times and pass through zstd's buffered-flush mode */
|
|
|
|
ASSERT_NOT_NULL(input = malloc(input_size));
|
|
for (size_t i = 0; i < input_size; i++)
|
|
input[i] = 'A' + (i % 26);
|
|
|
|
_cleanup_close_ int src = -EBADF, dst = -EBADF;
|
|
_cleanup_(unlink_tempfilep) char
|
|
src_pattern[] = "/tmp/systemd-test.zstd-unknown-src.XXXXXX",
|
|
pattern[] = "/tmp/systemd-test.zstd-unknown.XXXXXX";
|
|
struct stat st;
|
|
|
|
ASSERT_OK(src = mkostemp_safe(src_pattern));
|
|
ASSERT_OK(loop_write(src, input, input_size));
|
|
ASSERT_EQ(lseek(src, 0, SEEK_SET), (off_t) 0);
|
|
|
|
/* compress_stream() never records the content size in the frame header. */
|
|
ASSERT_OK(dst = mkostemp_safe(pattern));
|
|
ASSERT_OK(compress_stream(COMPRESSION_ZSTD, src, dst, UINT64_MAX, NULL));
|
|
|
|
ASSERT_OK_ERRNO(fstat(dst, &st));
|
|
|
|
_cleanup_free_ void *compressed = NULL;
|
|
ASSERT_NOT_NULL(compressed = malloc(st.st_size));
|
|
ASSERT_EQ(lseek(dst, 0, SEEK_SET), (off_t) 0);
|
|
ASSERT_EQ(loop_read(dst, compressed, st.st_size, true), (ssize_t) st.st_size);
|
|
|
|
/* Full blob decompression must reconstruct the input exactly. */
|
|
_cleanup_free_ void *decompressed = NULL;
|
|
size_t dsize = 0;
|
|
ASSERT_OK_ZERO(decompress_blob(COMPRESSION_ZSTD, compressed, st.st_size, &decompressed, &dsize, 0));
|
|
ASSERT_EQ(dsize, input_size);
|
|
ASSERT_EQ(memcmp(decompressed, input, input_size), 0);
|
|
|
|
/* Decompression capped via dst_max must stop at the requested size. */
|
|
decompressed = mfree(decompressed);
|
|
dsize = 0;
|
|
ASSERT_OK_ZERO(decompress_blob(COMPRESSION_ZSTD, compressed, st.st_size, &decompressed, &dsize, 4096));
|
|
ASSERT_EQ(dsize, 4096u);
|
|
ASSERT_EQ(memcmp(decompressed, input, 4096), 0);
|
|
|
|
/* decompress_startswith() must still find the prefix without a recorded content size. */
|
|
_cleanup_free_ void *buf = NULL;
|
|
ASSERT_OK_POSITIVE(decompress_startswith(COMPRESSION_ZSTD, compressed, st.st_size, &buf, input, 4096, input[4096]));
|
|
ASSERT_OK_ZERO(decompress_startswith(COMPRESSION_ZSTD, compressed, st.st_size, &buf, input, 4096, 0xff));
|
|
|
|
/* A concatenated multi-frame stream must be fully decoded, not silently truncated at the first
|
|
* frame boundary. */
|
|
size_t csize = st.st_size;
|
|
_cleanup_free_ void *multiframe = malloc(csize * 2);
|
|
ASSERT_NOT_NULL(multiframe);
|
|
memcpy(multiframe, compressed, csize);
|
|
memcpy((uint8_t*) multiframe + csize, compressed, csize);
|
|
|
|
decompressed = mfree(decompressed);
|
|
dsize = 0;
|
|
ASSERT_OK_ZERO(decompress_blob(COMPRESSION_ZSTD, multiframe, csize * 2, &decompressed, &dsize, 0));
|
|
ASSERT_EQ(dsize, input_size * 2);
|
|
ASSERT_EQ(memcmp(decompressed, input, input_size), 0);
|
|
ASSERT_EQ(memcmp((uint8_t*) decompressed + input_size, input, input_size), 0);
|
|
|
|
/* Kernel zboot payloads are usually compressed without a content checksum, which is the case that
|
|
* exercises the buffer-growth/buffered-flush logic without the checksum trailer masking
|
|
* end-of-input. Turn our stream into a non-checksummed one (clear bit 2 of the frame header
|
|
* descriptor, drop the 4-byte trailer) and make sure it still round-trips. */
|
|
((uint8_t*) compressed)[4] &= ~0x04;
|
|
decompressed = mfree(decompressed);
|
|
dsize = 0;
|
|
ASSERT_OK_ZERO(decompress_blob(COMPRESSION_ZSTD, compressed, csize - 4, &decompressed, &dsize, 0));
|
|
ASSERT_EQ(dsize, input_size);
|
|
ASSERT_EQ(memcmp(decompressed, input, input_size), 0);
|
|
|
|
/* A valid but short unknown-size stream whose content is shorter than the requested prefix must
|
|
* report "too short" (0), not a corruption error. */
|
|
_cleanup_close_ int short_src = -EBADF, short_dst = -EBADF;
|
|
_cleanup_(unlink_tempfilep) char
|
|
short_src_pattern[] = "/tmp/systemd-test.zstd-short-src.XXXXXX",
|
|
short_pattern[] = "/tmp/systemd-test.zstd-short.XXXXXX";
|
|
static const char short_text[] = "hi";
|
|
struct stat short_st;
|
|
|
|
ASSERT_OK(short_src = mkostemp_safe(short_src_pattern));
|
|
ASSERT_OK(loop_write(short_src, short_text, sizeof(short_text)));
|
|
ASSERT_EQ(lseek(short_src, 0, SEEK_SET), (off_t) 0);
|
|
ASSERT_OK(short_dst = mkostemp_safe(short_pattern));
|
|
ASSERT_OK(compress_stream(COMPRESSION_ZSTD, short_src, short_dst, UINT64_MAX, NULL));
|
|
ASSERT_OK_ERRNO(fstat(short_dst, &short_st));
|
|
|
|
_cleanup_free_ void *short_compressed = malloc(short_st.st_size);
|
|
ASSERT_NOT_NULL(short_compressed);
|
|
ASSERT_EQ(lseek(short_dst, 0, SEEK_SET), (off_t) 0);
|
|
ASSERT_EQ(loop_read(short_dst, short_compressed, short_st.st_size, true), (ssize_t) short_st.st_size);
|
|
|
|
_cleanup_free_ void *short_buf = NULL;
|
|
ASSERT_OK_ZERO(decompress_startswith(COMPRESSION_ZSTD, short_compressed, short_st.st_size, &short_buf,
|
|
"hello world", strlen("hello world"), 'x'));
|
|
}
|
|
|
|
TEST(decompress_startswith) {
|
|
for (Compression c = 0; c < _COMPRESSION_MAX; c++) {
|
|
if (c == COMPRESSION_NONE || !compression_supported(c))
|
|
continue;
|
|
|
|
const char *label = compression_to_string(c);
|
|
|
|
struct { const char *buf; size_t len; bool may_fail; } inputs[] = {
|
|
{ text, sizeof(text), false },
|
|
{ data, sizeof(data), true },
|
|
{ huge, HUGE_SIZE, true },
|
|
};
|
|
|
|
for (size_t t = 0; t < ELEMENTSOF(inputs); t++) {
|
|
char *compressed;
|
|
_cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL;
|
|
size_t csize, len;
|
|
int r;
|
|
|
|
log_info("/* testing decompress_startswith with %s on %.20s */", label, inputs[t].buf);
|
|
|
|
compressed = compressed1 = malloc(512);
|
|
ASSERT_NOT_NULL(compressed1);
|
|
r = compress_blob(c, inputs[t].buf, inputs[t].len, compressed, 512, &csize, -1);
|
|
if (r == -ENOBUFS) {
|
|
log_info_errno(r, "compression failed: %m");
|
|
ASSERT_TRUE(inputs[t].may_fail);
|
|
|
|
compressed = compressed2 = malloc(20000);
|
|
ASSERT_NOT_NULL(compressed2);
|
|
r = compress_blob(c, inputs[t].buf, inputs[t].len, compressed, 20000, &csize, -1);
|
|
}
|
|
if (r == -ENOBUFS) {
|
|
log_info_errno(r, "compression failed again: %m");
|
|
ASSERT_TRUE(inputs[t].may_fail);
|
|
continue;
|
|
}
|
|
ASSERT_OK(r);
|
|
|
|
len = strlen(inputs[t].buf);
|
|
|
|
ASSERT_OK_POSITIVE(decompress_startswith(c, compressed, csize, (void **) &decompressed, inputs[t].buf, len, '\0'));
|
|
ASSERT_OK_ZERO(decompress_startswith(c, compressed, csize, (void **) &decompressed, inputs[t].buf, len, 'w'));
|
|
ASSERT_OK_POSITIVE(decompress_startswith(c, compressed, csize, (void **) &decompressed, inputs[t].buf, len - 1, inputs[t].buf[len-1]));
|
|
ASSERT_OK_ZERO(decompress_startswith(c, compressed, csize, (void **) &decompressed, inputs[t].buf, len - 1, 'w'));
|
|
}
|
|
}
|
|
}
|
|
|
|
TEST(decompress_startswith_large) {
|
|
/* Test decompress_startswith with large data to exercise the buffer growth path. */
|
|
|
|
_cleanup_free_ char *large = NULL;
|
|
size_t large_size = 8 * 1024;
|
|
|
|
ASSERT_NOT_NULL(large = malloc(large_size));
|
|
for (size_t i = 0; i < large_size; i++)
|
|
large[i] = 'A' + (i % 26);
|
|
|
|
for (Compression c = 0; c < _COMPRESSION_MAX; c++) {
|
|
if (c == COMPRESSION_NONE || !compression_supported(c))
|
|
continue;
|
|
|
|
_cleanup_free_ char *compressed = NULL;
|
|
size_t csize;
|
|
|
|
log_info("/* decompress_startswith_large with %s */", compression_to_string(c));
|
|
|
|
ASSERT_NOT_NULL(compressed = malloc(large_size));
|
|
int r = compress_blob(c, large, large_size, compressed, large_size, &csize, -1);
|
|
if (r == -ENOBUFS) {
|
|
log_info_errno(r, "compression failed: %m");
|
|
continue;
|
|
}
|
|
ASSERT_OK(r);
|
|
|
|
_cleanup_free_ void *buf = NULL;
|
|
|
|
ASSERT_OK_POSITIVE(decompress_startswith(c, compressed, csize, &buf, large, 1, large[1]));
|
|
ASSERT_OK_ZERO(decompress_startswith(c, compressed, csize, &buf, large, 1, 0xff));
|
|
ASSERT_OK_POSITIVE(decompress_startswith(c, compressed, csize, &buf, large, 512, large[512]));
|
|
ASSERT_OK_ZERO(decompress_startswith(c, compressed, csize, &buf, large, 512, 0xff));
|
|
ASSERT_OK_POSITIVE(decompress_startswith(c, compressed, csize, &buf, large, 4096, large[4096]));
|
|
ASSERT_OK_ZERO(decompress_startswith(c, compressed, csize, &buf, large, 4096, 0xff));
|
|
}
|
|
}
|
|
|
|
TEST(decompress_startswith_short) {
|
|
#define TEXT "HUGE=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
|
|
|
|
for (Compression c = 0; c < _COMPRESSION_MAX; c++) {
|
|
if (c == COMPRESSION_NONE || !compression_supported(c))
|
|
continue;
|
|
|
|
char buf[1024];
|
|
size_t csize;
|
|
|
|
log_info("/* decompress_startswith_short with %s */", compression_to_string(c));
|
|
|
|
ASSERT_OK(compress_blob(c, TEXT, sizeof TEXT, buf, sizeof buf, &csize, -1));
|
|
|
|
for (size_t i = 1; i < strlen(TEXT); i++) {
|
|
_cleanup_free_ void *buf2 = NULL;
|
|
|
|
ASSERT_NOT_NULL(buf2 = malloc(i));
|
|
|
|
ASSERT_OK_POSITIVE(decompress_startswith(c, buf, csize, &buf2, TEXT, i, TEXT[i]));
|
|
ASSERT_OK_ZERO(decompress_startswith(c, buf, csize, &buf2, TEXT, i, 'y'));
|
|
}
|
|
}
|
|
#undef TEXT
|
|
}
|
|
|
|
TEST(compress_decompress_stream) {
|
|
for (Compression c = 0; c < _COMPRESSION_MAX; c++) {
|
|
if (c == COMPRESSION_NONE || !compression_supported(c))
|
|
continue;
|
|
|
|
const char *cat = cat_for_compression(c);
|
|
if (!cat)
|
|
continue;
|
|
|
|
int r = find_executable(cat, NULL);
|
|
if (r < 0) {
|
|
log_error_errno(r, "Skipping %s, could not find %s binary: %m",
|
|
compression_to_string(c), cat);
|
|
continue;
|
|
}
|
|
|
|
_cleanup_close_ int src = -EBADF, dst = -EBADF, dst2 = -EBADF;
|
|
_cleanup_(unlink_tempfilep) char
|
|
pattern[] = "/tmp/systemd-test.compressed.XXXXXX",
|
|
pattern2[] = "/tmp/systemd-test.compressed.XXXXXX";
|
|
_cleanup_free_ char *cmd = NULL, *cmd2 = NULL;
|
|
struct stat st = {};
|
|
uint64_t uncompressed_size;
|
|
|
|
log_debug("/* testing %s stream compression */", compression_to_string(c));
|
|
|
|
ASSERT_OK(src = open(srcfile, O_RDONLY|O_CLOEXEC));
|
|
ASSERT_OK(dst = mkostemp_safe(pattern));
|
|
|
|
ASSERT_OK(compress_stream(c, src, dst, -1, &uncompressed_size));
|
|
|
|
ASSERT_OK_POSITIVE(asprintf(&cmd, "%s %s | diff '%s' -", cat, pattern, srcfile));
|
|
ASSERT_OK_ZERO(system(cmd));
|
|
|
|
ASSERT_OK(dst2 = mkostemp_safe(pattern2));
|
|
|
|
ASSERT_OK_ZERO_ERRNO(stat(srcfile, &st));
|
|
ASSERT_EQ((uint64_t) st.st_size, uncompressed_size);
|
|
|
|
ASSERT_OK_ERRNO(lseek(dst, 0, SEEK_SET));
|
|
ASSERT_OK_ZERO(decompress_stream(c, dst, dst2, st.st_size));
|
|
|
|
ASSERT_OK_POSITIVE(asprintf(&cmd2, "diff '%s' %s", srcfile, pattern2));
|
|
ASSERT_OK_ZERO(system(cmd2));
|
|
|
|
log_debug("/* test faulty decompression */");
|
|
|
|
ASSERT_OK_ERRNO(lseek(dst, 1, SEEK_SET));
|
|
ASSERT_ERROR(decompress_stream(c, dst, dst2, st.st_size), EBADMSG);
|
|
|
|
ASSERT_OK_ERRNO(lseek(dst, 0, SEEK_SET));
|
|
ASSERT_OK_ERRNO(lseek(dst2, 0, SEEK_SET));
|
|
ASSERT_ERROR(decompress_stream(c, dst, dst2, st.st_size - 1), EFBIG);
|
|
}
|
|
}
|
|
|
|
struct decompressor_test_data {
|
|
uint8_t *buf;
|
|
size_t size;
|
|
};
|
|
|
|
static int test_decompressor_callback(const void *p, size_t size, void *userdata) {
|
|
struct decompressor_test_data *d = ASSERT_PTR(userdata);
|
|
|
|
if (!GREEDY_REALLOC(d->buf, d->size + size))
|
|
return -ENOMEM;
|
|
|
|
memcpy(d->buf + d->size, p, size);
|
|
d->size += size;
|
|
return 0;
|
|
}
|
|
|
|
static void compare_fd(int fda, int fdb) {
|
|
struct stat sta, stb;
|
|
|
|
/* Verify apparent size matches */
|
|
ASSERT_OK_ERRNO(fstat(fda, &sta));
|
|
ASSERT_OK_ERRNO(fstat(fdb, &stb));
|
|
ASSERT_EQ(sta.st_size, stb.st_size);
|
|
|
|
/* Verify content matches by comparing bytes */
|
|
ASSERT_OK_EQ_ERRNO(lseek(fda, 0, SEEK_SET), (off_t) 0);
|
|
ASSERT_OK_EQ_ERRNO(lseek(fdb, 0, SEEK_SET), (off_t) 0);
|
|
|
|
for (off_t offset = 0; offset < sta.st_size;) {
|
|
uint8_t bufa[4096], bufb[4096];
|
|
size_t to_read = MIN((size_t) (sta.st_size - offset), sizeof(bufa));
|
|
|
|
ASSERT_OK(loop_read_exact(fda, bufa, to_read, /* do_poll= */ false));
|
|
ASSERT_OK(loop_read_exact(fdb, bufb, to_read, /* do_poll= */ false));
|
|
ASSERT_EQ(memcmp(bufa, bufb, to_read), 0);
|
|
offset += to_read;
|
|
}
|
|
}
|
|
|
|
TEST(decompress_stream_sparse) {
|
|
for (Compression c = 0; c < _COMPRESSION_MAX; c++) {
|
|
if (c == COMPRESSION_NONE || !compression_supported(c))
|
|
continue;
|
|
|
|
_cleanup_close_ int src = -EBADF, compressed = -EBADF, decompressed = -EBADF;
|
|
_cleanup_(unlink_tempfilep) char
|
|
pattern_src[] = "/tmp/systemd-test.sparse-src.XXXXXX",
|
|
pattern_compressed[] = "/tmp/systemd-test.sparse-compressed.XXXXXX",
|
|
pattern_decompressed[] = "/tmp/systemd-test.sparse-decompressed.XXXXXX";
|
|
/* Create a sparse-like input: 4K of data, 64K of zeros, 4K of data, 64K trailing zeros.
|
|
* Total apparent size: 136K, but most of it is zeros. */
|
|
uint8_t data_block[4096];
|
|
struct stat st_src, st_decompressed;
|
|
uint64_t uncompressed_size;
|
|
|
|
log_debug("/* testing %s sparse decompression */", compression_to_string(c));
|
|
|
|
random_bytes(data_block, sizeof(data_block));
|
|
|
|
ASSERT_OK(src = mkostemp_safe(pattern_src));
|
|
|
|
/* Write: 4K data, 64K zeros, 4K data, 64K zeros */
|
|
ASSERT_OK(loop_write(src, data_block, sizeof(data_block)));
|
|
ASSERT_OK_ERRNO(ftruncate(src, sizeof(data_block) + 65536));
|
|
ASSERT_OK_ERRNO(lseek(src, sizeof(data_block) + 65536, SEEK_SET));
|
|
ASSERT_OK(loop_write(src, data_block, sizeof(data_block)));
|
|
ASSERT_OK_ERRNO(ftruncate(src, 2 * sizeof(data_block) + 2 * 65536));
|
|
ASSERT_EQ(lseek(src, 0, SEEK_SET), (off_t) 0);
|
|
|
|
ASSERT_OK_ERRNO(fstat(src, &st_src));
|
|
ASSERT_EQ(st_src.st_size, 2 * (off_t) sizeof(data_block) + 2 * 65536);
|
|
|
|
/* Compress */
|
|
ASSERT_OK(compressed = mkostemp_safe(pattern_compressed));
|
|
ASSERT_OK(compress_stream(c, src, compressed, -1, &uncompressed_size));
|
|
ASSERT_EQ((uint64_t) st_src.st_size, uncompressed_size);
|
|
|
|
/* Decompress to a regular file (sparse writes auto-detected) */
|
|
ASSERT_OK(decompressed = mkostemp_safe(pattern_decompressed));
|
|
ASSERT_EQ(lseek(compressed, 0, SEEK_SET), (off_t) 0);
|
|
ASSERT_OK_ZERO(decompress_stream(c, compressed, decompressed, st_src.st_size));
|
|
|
|
compare_fd(src, decompressed);
|
|
|
|
ASSERT_OK_ERRNO(fstat(decompressed, &st_decompressed));
|
|
/* Verify the decompressed file is actually sparse (uses less disk than apparent size).
|
|
* st_blocks is in 512-byte units. The file has 128K of zeros, so disk usage should be
|
|
* noticeably less than the apparent size if sparse writes worked.
|
|
* Only assert if the filesystem supports holes (SEEK_HOLE). */
|
|
log_debug("%s sparse decompression: apparent=%jd disk=%jd",
|
|
compression_to_string(c),
|
|
(intmax_t) st_decompressed.st_size,
|
|
(intmax_t) st_decompressed.st_blocks * 512);
|
|
if (lseek(decompressed, 0, SEEK_HOLE) < st_decompressed.st_size)
|
|
ASSERT_LT(st_decompressed.st_blocks * 512, st_decompressed.st_size);
|
|
else
|
|
log_debug("Filesystem does not support holes, skipping sparsity check");
|
|
|
|
/* Check if the decompressor clears previous data.
|
|
* The original data is 4K data, 64K zeros, 4K data, 64K zeros.
|
|
* Now, fill the file with 4K zeros, 64K data, 4K zeros, 64K data. */
|
|
log_debug("/* testing %s sparse decompression to an existing file */", compression_to_string(c));
|
|
|
|
ASSERT_OK_ERRNO(ftruncate(decompressed, 0));
|
|
ASSERT_OK_ERRNO(ftruncate(decompressed, 4096));
|
|
ASSERT_OK_ERRNO(lseek(decompressed, 4096, SEEK_SET));
|
|
for (unsigned i = 0; i < 16; i++)
|
|
ASSERT_OK(loop_write(decompressed, data_block, sizeof(data_block)));
|
|
ASSERT_OK_ERRNO(ftruncate(decompressed, 4096 + 16 * sizeof(data_block) + 4096));
|
|
ASSERT_OK_ERRNO(lseek(decompressed, 4096 + 16 * sizeof(data_block) + 4096, SEEK_SET));
|
|
for (unsigned i = 0; i < 16; i++)
|
|
ASSERT_OK(loop_write(decompressed, data_block, sizeof(data_block)));
|
|
|
|
ASSERT_OK_ERRNO(lseek(compressed, 0, SEEK_SET));
|
|
ASSERT_OK_ERRNO(lseek(decompressed, 0, SEEK_SET));
|
|
ASSERT_OK_ZERO(decompress_stream(c, compressed, decompressed, st_src.st_size));
|
|
|
|
compare_fd(src, decompressed);
|
|
|
|
log_debug("/* testing %s sparse decompression with multiple streams */", compression_to_string(c));
|
|
{
|
|
_cleanup_close_ int multi = -EBADF, compressed_multi = -EBADF, decompressed_multi = -EBADF;
|
|
_cleanup_(unlink_tempfilep) char
|
|
pattern_multi[] = "/tmp/systemd-test.sparse-multi.XXXXXX",
|
|
pattern_compressed_multi[] = "/tmp/systemd-test.sparse-compressed_multi.XXXXXX",
|
|
pattern_decompressed_multi[] = "/tmp/systemd-test.sparse-decompressed_multi.XXXXXX";
|
|
|
|
ASSERT_OK(multi = mkostemp_safe(pattern_multi));
|
|
ASSERT_OK(compressed_multi = mkostemp_safe(pattern_compressed_multi));
|
|
ASSERT_OK(decompressed_multi = mkostemp_safe(pattern_decompressed_multi));
|
|
|
|
for (unsigned i = 0; i < 4; i++) {
|
|
ASSERT_OK(copy_bytes(src, multi, UINT64_MAX, COPY_SEEK0_SOURCE));
|
|
ASSERT_OK(copy_bytes(compressed, compressed_multi, UINT64_MAX, COPY_SEEK0_SOURCE));
|
|
}
|
|
|
|
ASSERT_EQ(lseek(compressed_multi, 0, SEEK_SET), (off_t) 0);
|
|
ASSERT_OK(decompress_stream(c, compressed_multi, decompressed_multi, UINT64_MAX));
|
|
|
|
compare_fd(multi, decompressed_multi);
|
|
}
|
|
|
|
log_debug("/* testing %s sparse decompression with garbage at the end */", compression_to_string(c));
|
|
|
|
off_t end = ASSERT_OK_ERRNO(lseek(compressed, 0, SEEK_END));
|
|
ASSERT_OK(loop_write(compressed, "a", 1));
|
|
ASSERT_OK_ERRNO(lseek(compressed, 0, SEEK_SET));
|
|
ASSERT_OK_ERRNO(lseek(decompressed, 0, SEEK_SET));
|
|
ASSERT_ERROR(decompress_stream(c, compressed, decompressed, st_src.st_size), EBADMSG);
|
|
|
|
log_debug("/* testing %s sparse decompression with truncated data */", compression_to_string(c));
|
|
|
|
ASSERT_OK_ERRNO(ftruncate(compressed, end - 1));
|
|
ASSERT_OK_ERRNO(lseek(compressed, 0, SEEK_SET));
|
|
ASSERT_OK_ERRNO(lseek(decompressed, 0, SEEK_SET));
|
|
ASSERT_ERROR(decompress_stream(c, compressed, decompressed, st_src.st_size), EBADMSG);
|
|
|
|
log_debug("/* testing %s sparse decompression with zero length stream */", compression_to_string(c));
|
|
|
|
ASSERT_OK_ERRNO(ftruncate(compressed, 0));
|
|
ASSERT_OK_ERRNO(lseek(compressed, 0, SEEK_SET));
|
|
ASSERT_OK_ERRNO(lseek(decompressed, 0, SEEK_SET));
|
|
ASSERT_ERROR(decompress_stream(c, compressed, decompressed, st_src.st_size), EBADMSG);
|
|
|
|
/* Test all-zeros input: entire output should be a hole */
|
|
log_debug("/* testing %s sparse decompression of all-zeros */", compression_to_string(c));
|
|
{
|
|
_cleanup_close_ int zsrc = -EBADF, zcompressed = -EBADF, zdecompressed = -EBADF;
|
|
_cleanup_(unlink_tempfilep) char
|
|
zp_src[] = "/tmp/systemd-test.sparse-zero-src.XXXXXX",
|
|
zp_compressed[] = "/tmp/systemd-test.sparse-zero-compressed.XXXXXX",
|
|
zp_decompressed[] = "/tmp/systemd-test.sparse-zero-decompressed.XXXXXX";
|
|
struct stat zst;
|
|
uint64_t zsize, zsize_expected = 65536;
|
|
|
|
ASSERT_OK(zsrc = mkostemp_safe(zp_src));
|
|
ASSERT_OK_ERRNO(ftruncate(zsrc, zsize_expected));
|
|
ASSERT_EQ(lseek(zsrc, 0, SEEK_SET), (off_t) 0);
|
|
|
|
ASSERT_OK(zcompressed = mkostemp_safe(zp_compressed));
|
|
ASSERT_OK(compress_stream(c, zsrc, zcompressed, -1, &zsize));
|
|
ASSERT_EQ(zsize, zsize_expected);
|
|
|
|
ASSERT_OK(zdecompressed = mkostemp_safe(zp_decompressed));
|
|
ASSERT_EQ(lseek(zcompressed, 0, SEEK_SET), (off_t) 0);
|
|
ASSERT_OK_ZERO(decompress_stream(c, zcompressed, zdecompressed, zsize_expected));
|
|
|
|
compare_fd(zsrc, zdecompressed);
|
|
|
|
ASSERT_OK_ERRNO(fstat(zdecompressed, &zst));
|
|
/* All zeros — disk usage should be minimal */
|
|
log_debug("%s all-zeros sparse: apparent=%jd disk=%jd",
|
|
compression_to_string(c), (intmax_t) zst.st_size, (intmax_t) zst.st_blocks * 512);
|
|
if (lseek(zdecompressed, 0, SEEK_HOLE) < zst.st_size)
|
|
ASSERT_LT(zst.st_blocks * 512, zst.st_size);
|
|
else
|
|
log_debug("Filesystem does not support holes, skipping sparsity check");
|
|
}
|
|
|
|
/* Test data ending with non-zero bytes: ftruncate should be a no-op */
|
|
log_debug("/* testing %s sparse decompression ending with data */", compression_to_string(c));
|
|
{
|
|
_cleanup_close_ int dsrc = -EBADF, dcompressed = -EBADF, ddecompressed = -EBADF;
|
|
_cleanup_(unlink_tempfilep) char
|
|
dp_src[] = "/tmp/systemd-test.sparse-end-src.XXXXXX",
|
|
dp_compressed[] = "/tmp/systemd-test.sparse-end-compressed.XXXXXX",
|
|
dp_decompressed[] = "/tmp/systemd-test.sparse-end-decompressed.XXXXXX";
|
|
uint64_t dsize;
|
|
|
|
/* 64K zeros followed by 4K random data */
|
|
ASSERT_OK(dsrc = mkostemp_safe(dp_src));
|
|
ASSERT_OK_ERRNO(ftruncate(dsrc, 65536));
|
|
ASSERT_OK_ERRNO(lseek(dsrc, 65536, SEEK_SET));
|
|
ASSERT_OK(loop_write(dsrc, data_block, sizeof(data_block)));
|
|
ASSERT_EQ(lseek(dsrc, 0, SEEK_SET), (off_t) 0);
|
|
|
|
ASSERT_OK(dcompressed = mkostemp_safe(dp_compressed));
|
|
ASSERT_OK(compress_stream(c, dsrc, dcompressed, -1, &dsize));
|
|
ASSERT_EQ(dsize, (uint64_t) (65536 + sizeof(data_block)));
|
|
|
|
ASSERT_OK(ddecompressed = mkostemp_safe(dp_decompressed));
|
|
ASSERT_EQ(lseek(dcompressed, 0, SEEK_SET), (off_t) 0);
|
|
ASSERT_OK_ZERO(decompress_stream(c, dcompressed, ddecompressed, dsize));
|
|
|
|
compare_fd(dsrc, ddecompressed);
|
|
}
|
|
}
|
|
}
|
|
|
|
TEST(compressor_decompressor_push_api) {
|
|
for (Compression c = 0; c < _COMPRESSION_MAX; c++) {
|
|
if (c == COMPRESSION_NONE || !compression_supported(c))
|
|
continue;
|
|
|
|
log_info("/* testing %s Compressor/Decompressor push API */", compression_to_string(c));
|
|
|
|
_cleanup_(compressor_freep) Compressor *compressor = NULL;
|
|
_cleanup_(compressor_freep) Decompressor *decompressor = NULL;
|
|
_cleanup_free_ void *compressed = NULL, *finish_buf = NULL;
|
|
size_t compressed_size = 0, compressed_alloc = 0;
|
|
size_t finish_size = 0, finish_alloc = 0;
|
|
|
|
/* Compress */
|
|
ASSERT_OK(compressor_new(&compressor, c));
|
|
ASSERT_EQ(compressor_type(compressor), c);
|
|
|
|
ASSERT_OK(compressor_start(compressor, text, sizeof(text), &compressed, &compressed_size, &compressed_alloc));
|
|
ASSERT_OK(compressor_finish(compressor, &finish_buf, &finish_size, &finish_alloc));
|
|
|
|
size_t total_compressed = compressed_size + finish_size;
|
|
_cleanup_free_ void *full_compressed = malloc(total_compressed);
|
|
ASSERT_NOT_NULL(full_compressed);
|
|
memcpy(full_compressed, compressed, compressed_size);
|
|
if (finish_size > 0)
|
|
memcpy((uint8_t*) full_compressed + compressed_size, finish_buf, finish_size);
|
|
|
|
compressor = compressor_free(compressor);
|
|
|
|
/* Decompress via detect + push and verify content */
|
|
ASSERT_OK_POSITIVE(decompressor_detect(&decompressor, full_compressed, total_compressed));
|
|
ASSERT_EQ(compressor_type(decompressor), c);
|
|
|
|
struct decompressor_test_data result = {};
|
|
for (unsigned i = 0; i < 4; i++)
|
|
ASSERT_OK(decompressor_push(decompressor, full_compressed, total_compressed, test_decompressor_callback, &result));
|
|
ASSERT_OK(decompressor_push(decompressor, /* data= */ NULL, /* size= */ 0, test_decompressor_callback, &result));
|
|
|
|
ASSERT_EQ(result.size, sizeof(text) * 4);
|
|
for (unsigned i = 0; i < 4; i++)
|
|
ASSERT_EQ(memcmp(result.buf + sizeof(text) * i, text, sizeof(text)), 0);
|
|
free(result.buf);
|
|
|
|
decompressor = compressor_free(decompressor);
|
|
}
|
|
|
|
/* Test compressor_type on NULL */
|
|
ASSERT_EQ(compressor_type(NULL), _COMPRESSION_INVALID);
|
|
|
|
/* Test decompressor_force_off */
|
|
_cleanup_(compressor_freep) Decompressor *d = NULL;
|
|
ASSERT_OK(decompressor_force_off(&d));
|
|
ASSERT_EQ(compressor_type(d), COMPRESSION_NONE);
|
|
d = compressor_free(d);
|
|
|
|
/* Test decompressor_detect returning 0 on too-small input */
|
|
ASSERT_OK_ZERO(decompressor_detect(&d, "x", 1));
|
|
ASSERT_NULL(d);
|
|
}
|
|
|
|
static int intro(void) {
|
|
srcfile = saved_argc > 1 ? saved_argv[1] : saved_argv[0];
|
|
|
|
ASSERT_NOT_NULL(huge = malloc(HUGE_SIZE));
|
|
memcpy(huge, "HUGE=", STRLEN("HUGE="));
|
|
memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
|
|
huge[HUGE_SIZE - 1] = '\0';
|
|
|
|
random_bytes(data + 7, sizeof(data) - 7);
|
|
|
|
return 0;
|
|
}
|
|
|
|
DEFINE_TEST_MAIN_WITH_INTRO(LOG_DEBUG, intro);
|