From f08796065d863a536791bf5cd96c8b8e1a65c339 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sat, 28 Mar 2026 23:21:18 +0000 Subject: [PATCH] compress: consolidate all compression into compress.c with dlopen Move the push-based streaming compression API from import-compress.c into compress.c and delete import-compress.c/h. This consolidates all compression code in one place and makes all compression libraries (liblzma, liblz4, libzstd, libz, libbz2) runtime-loaded via dlopen instead of directly linked. Introduce opaque Compressor/Decompressor types backed by a heap- allocated struct defined only in compress.c, keeping all third-party library headers out of compress.h. Rewrite the per-codec fd-to-fd stream functions as thin wrappers around the push API via generic compress_stream()/decompress_stream() taking a Compression type parameter. Integrate LZ4 into this framework using the LZ4 Frame API, eliminating all LZ4 special-casing. Extend the Compression enum with COMPRESSION_GZIP and COMPRESSION_BZIP2 and add the corresponding blob, startswith, and stream functions for both. Rename the ImportCompress types and functions: ImportCompressType becomes the existing Compression enum, ImportCompress becomes Compressor (with Decompressor typedef), and all import_compress_*/import_uncompress_* become compressor_*/decompressor_*. Rename dlopen_lzma() to dlopen_xz() for consistency. Make compression_to_string() return lowercase by default. Add INT_MAX/UINT_MAX overflow checks for LZ4, zlib, and bzip2 blob functions where the codec API uses narrower integer types than our uint64_t parameters. Migrate test-compress.c and test-compress-benchmark.c to the TEST() macro framework, new assertion macros, and codec-generic loops instead of per-codec duplication. Co-developed-by: Claude Opus 4.6 --- meson.build | 2 + src/basic/compress.c | 2452 ++++++++++++----- src/basic/compress.h | 128 +- src/basic/meson.build | 4 +- src/boot/test-bcd.c | 2 +- src/coredump/coredump-submit.c | 4 +- src/coredump/coredumpctl.c | 2 +- src/fundamental/macro-fundamental.h | 6 + src/import/export-raw.c | 25 +- src/import/export-raw.h | 4 +- src/import/export-tar.c | 23 +- src/import/export-tar.h | 4 +- src/import/export.c | 34 +- src/import/import-common.c | 5 +- src/import/import-common.h | 2 - src/import/import-compress.c | 611 ---- src/import/import-compress.h | 54 - src/import/import-raw.c | 19 +- src/import/import-tar.c | 17 +- src/import/meson.build | 5 - src/import/pull-common.c | 2 + src/import/pull-job.c | 11 +- src/import/pull-job.h | 6 +- src/import/pull-oci.c | 2 + src/import/pull-raw.c | 2 + src/import/pull-tar.c | 2 + src/import/qcow2-util.c | 22 +- src/journal-remote/journal-compression-util.c | 2 +- src/journal-remote/journal-remote-main.c | 4 +- src/journal-remote/journal-upload-journal.c | 2 +- src/journal-remote/journal-upload.c | 10 +- src/libsystemd/sd-journal/journal-file.c | 2 +- src/test/test-compress-benchmark.c | 193 +- src/test/test-compress.c | 863 +++--- src/test/test-dlopen-so.c | 4 +- ...EST-04-JOURNAL.SYSTEMD_JOURNAL_COMPRESS.sh | 17 +- 36 files changed, 2473 insertions(+), 2074 deletions(-) delete mode 100644 src/import/import-compress.c delete mode 100644 src/import/import-compress.h diff --git a/meson.build b/meson.build index 659aecd421..5e01f22d5b 100644 --- a/meson.build +++ b/meson.build @@ -1373,6 +1373,7 @@ conf.set10('HAVE_DWFL_SET_SYSROOT', libz = dependency('zlib', required : get_option('zlib')) conf.set10('HAVE_ZLIB', libz.found()) +libz_cflags = libz.partial_dependency(includes: true, compile_args: true) feature = get_option('bzip2') libbzip2 = dependency('bzip2', @@ -1382,6 +1383,7 @@ if not libbzip2.found() libbzip2 = cc.find_library('bz2', required : feature) endif conf.set10('HAVE_BZIP2', libbzip2.found()) +libbzip2_cflags = libbzip2.partial_dependency(includes: true, compile_args: true) libxz = dependency('liblzma', required : get_option('xz')) diff --git a/src/basic/compress.c b/src/basic/compress.c index 5f00f968a2..251eb02fbb 100644 --- a/src/basic/compress.c +++ b/src/basic/compress.c @@ -2,38 +2,65 @@ #include #include -#include #include #include -#if HAVE_LZ4 -#include -#include -#include -#endif - #if HAVE_XZ #include #endif +#if HAVE_LZ4 +#include +#include +#include +#endif + #if HAVE_ZSTD #include #include #endif +#if HAVE_ZLIB +#include +#endif + +#if HAVE_BZIP2 +#include +#endif + #include "sd-dlopen.h" #include "alloc-util.h" #include "bitfield.h" #include "compress.h" #include "dlfcn-util.h" -#include "fileio.h" #include "io-util.h" #include "log.h" #include "string-table.h" -#include "string-util.h" #include "unaligned.h" +#if HAVE_XZ +static void *lzma_dl = NULL; + +static DLSYM_PROTOTYPE(lzma_code) = NULL; +static DLSYM_PROTOTYPE(lzma_easy_encoder) = NULL; +static DLSYM_PROTOTYPE(lzma_end) = NULL; +static DLSYM_PROTOTYPE(lzma_stream_buffer_encode) = NULL; +static DLSYM_PROTOTYPE(lzma_stream_decoder) = NULL; +static DLSYM_PROTOTYPE(lzma_lzma_preset) = NULL; + +/* We can’t just do _cleanup_(sym_lzma_end) because a compiler bug makes + * this fail with: + * ../src/basic/compress.c: In function ‘decompress_blob_xz’: + * ../src/basic/compress.c:304:9: error: cleanup argument not a function + * 304 | _cleanup_(sym_lzma_end) lzma_stream s = LZMA_STREAM_INIT; + * | ^~~~~~~~~ + */ +static inline void lzma_end_wrapper(lzma_stream *ls) { + sym_lzma_end(ls); +} +#endif + #if HAVE_LZ4 static void *lz4_dl = NULL; @@ -48,16 +75,14 @@ static DLSYM_PROTOTYPE(LZ4F_freeCompressionContext) = NULL; static DLSYM_PROTOTYPE(LZ4F_freeDecompressionContext) = NULL; static DLSYM_PROTOTYPE(LZ4F_isError) = NULL; static DLSYM_PROTOTYPE(LZ4_compress_HC) = NULL; -/* These are used in test-compress.c so we don't make them static. */ -// NOLINTBEGIN(misc-use-internal-linkage) -DLSYM_PROTOTYPE(LZ4_compress_default) = NULL; -DLSYM_PROTOTYPE(LZ4_decompress_safe) = NULL; -DLSYM_PROTOTYPE(LZ4_decompress_safe_partial) = NULL; -DLSYM_PROTOTYPE(LZ4_versionNumber) = NULL; -// NOLINTEND(misc-use-internal-linkage) +static DLSYM_PROTOTYPE(LZ4_compress_default) = NULL; +static DLSYM_PROTOTYPE(LZ4_decompress_safe) = NULL; +static DLSYM_PROTOTYPE(LZ4_decompress_safe_partial) = NULL; +static DLSYM_PROTOTYPE(LZ4_versionNumber) = NULL; -DEFINE_TRIVIAL_CLEANUP_FUNC_FULL_RENAME(LZ4F_compressionContext_t, sym_LZ4F_freeCompressionContext, LZ4F_freeCompressionContextp, NULL); -DEFINE_TRIVIAL_CLEANUP_FUNC_FULL_RENAME(LZ4F_decompressionContext_t, sym_LZ4F_freeDecompressionContext, LZ4F_freeDecompressionContextp, NULL); +static const LZ4F_preferences_t lz4_preferences = { + .frameInfo.blockSizeID = 5, +}; #endif #if HAVE_ZSTD @@ -80,7 +105,6 @@ static DLSYM_PROTOTYPE(ZSTD_getErrorName) = NULL; static DLSYM_PROTOTYPE(ZSTD_getFrameContentSize) = NULL; static DLSYM_PROTOTYPE(ZSTD_isError) = NULL; -DEFINE_TRIVIAL_CLEANUP_FUNC_FULL_RENAME(ZSTD_CCtx*, sym_ZSTD_freeCCtx, ZSTD_freeCCtxp, NULL); DEFINE_TRIVIAL_CLEANUP_FUNC_FULL_RENAME(ZSTD_DCtx*, sym_ZSTD_freeDCtx, ZSTD_freeDCtxp, NULL); static int zstd_ret_to_errno(size_t ret) { @@ -95,53 +119,146 @@ static int zstd_ret_to_errno(size_t ret) { } #endif -#if HAVE_XZ -static void *lzma_dl = NULL; +#if HAVE_ZLIB +static void *zlib_dl = NULL; -static DLSYM_PROTOTYPE(lzma_code) = NULL; -static DLSYM_PROTOTYPE(lzma_easy_encoder) = NULL; -static DLSYM_PROTOTYPE(lzma_end) = NULL; -static DLSYM_PROTOTYPE(lzma_stream_buffer_encode) = NULL; -static DLSYM_PROTOTYPE(lzma_stream_decoder) = NULL; -static DLSYM_PROTOTYPE(lzma_lzma_preset) = NULL; +static DLSYM_PROTOTYPE(deflateInit2_) = NULL; +static DLSYM_PROTOTYPE(deflate) = NULL; +static DLSYM_PROTOTYPE(deflateEnd) = NULL; +static DLSYM_PROTOTYPE(inflateInit2_) = NULL; +static DLSYM_PROTOTYPE(inflate) = NULL; +static DLSYM_PROTOTYPE(inflateEnd) = NULL; -/* We can't just do _cleanup_(sym_lzma_end) because a compiler bug makes - * this fail with: - * ../src/basic/compress.c: In function ‘decompress_blob_xz’: - * ../src/basic/compress.c:304:9: error: cleanup argument not a function - * 304 | _cleanup_(sym_lzma_end) lzma_stream s = LZMA_STREAM_INIT; - * | ^~~~~~~~~ - */ -static inline void lzma_end_wrapper(lzma_stream *ls) { - sym_lzma_end(ls); +static inline void deflateEnd_wrapper(z_stream *s) { + sym_deflateEnd(s); +} + +static inline void inflateEnd_wrapper(z_stream *s) { + sym_inflateEnd(s); } #endif -#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t)) +#if HAVE_BZIP2 +static void *bzip2_dl = NULL; -static const char* const compression_table[_COMPRESSION_MAX] = { - [COMPRESSION_NONE] = "NONE", - [COMPRESSION_XZ] = "XZ", - [COMPRESSION_LZ4] = "LZ4", - [COMPRESSION_ZSTD] = "ZSTD", +static DLSYM_PROTOTYPE(BZ2_bzCompressInit) = NULL; +static DLSYM_PROTOTYPE(BZ2_bzCompress) = NULL; +static DLSYM_PROTOTYPE(BZ2_bzCompressEnd) = NULL; +static DLSYM_PROTOTYPE(BZ2_bzDecompressInit) = NULL; +static DLSYM_PROTOTYPE(BZ2_bzDecompress) = NULL; +static DLSYM_PROTOTYPE(BZ2_bzDecompressEnd) = NULL; + +static inline void BZ2_bzCompressEnd_wrapper(bz_stream *s) { + sym_BZ2_bzCompressEnd(s); +} + +static inline void BZ2_bzDecompressEnd_wrapper(bz_stream *s) { + sym_BZ2_bzDecompressEnd(s); +} +#endif + +/* Opaque Compressor/Decompressor struct definition */ +struct Compressor { + Compression type; + bool encoding; + union { +#if HAVE_XZ + lzma_stream xz; +#endif +#if HAVE_LZ4 + struct { + LZ4F_compressionContext_t c_lz4; + void *lz4_header; /* stashed frame header from LZ4F_compressBegin */ + size_t lz4_header_size; + }; + LZ4F_decompressionContext_t d_lz4; +#endif +#if HAVE_ZSTD + ZSTD_CCtx *c_zstd; + ZSTD_DCtx *d_zstd; +#endif +#if HAVE_ZLIB + z_stream gzip; +#endif +#if HAVE_BZIP2 + bz_stream bzip2; +#endif + }; }; -static const char* const compression_lowercase_table[_COMPRESSION_MAX] = { - [COMPRESSION_NONE] = "none", - [COMPRESSION_XZ] = "xz", - [COMPRESSION_LZ4] = "lz4", - [COMPRESSION_ZSTD] = "zstd", +#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t)) + +/* zlib windowBits value for gzip format: MAX_WBITS (15) + 16 to enable gzip header detection/generation */ +#define ZLIB_WBITS_GZIP (15 + 16) + +static const char* const compression_table[_COMPRESSION_MAX] = { + [COMPRESSION_NONE] = "uncompressed", /* backwards compatibility with importd */ + [COMPRESSION_XZ] = "xz", + [COMPRESSION_LZ4] = "lz4", + [COMPRESSION_ZSTD] = "zstd", + [COMPRESSION_GZIP] = "gzip", + [COMPRESSION_BZIP2] = "bzip2", +}; + +static const char* const compression_uppercase_table[_COMPRESSION_MAX] = { + [COMPRESSION_NONE] = "NONE", /* backwards compatibility with SYSTEMD_JOURNAL_COMPRESS=NONE */ + [COMPRESSION_XZ] = "XZ", + [COMPRESSION_LZ4] = "LZ4", + [COMPRESSION_ZSTD] = "ZSTD", + [COMPRESSION_GZIP] = "GZIP", + [COMPRESSION_BZIP2] = "BZIP2", +}; + +static const char* const compression_extension_table[_COMPRESSION_MAX] = { + [COMPRESSION_NONE] = "", + [COMPRESSION_XZ] = ".xz", + [COMPRESSION_LZ4] = ".lz4", + [COMPRESSION_ZSTD] = ".zst", + [COMPRESSION_GZIP] = ".gz", + [COMPRESSION_BZIP2] = ".bz2", }; DEFINE_STRING_TABLE_LOOKUP(compression, Compression); -DEFINE_STRING_TABLE_LOOKUP(compression_lowercase, Compression); +DEFINE_STRING_TABLE_LOOKUP(compression_uppercase, Compression); +DEFINE_STRING_TABLE_LOOKUP(compression_extension, Compression); + +Compression compression_from_string_harder(const char *s) { + Compression c; + + assert(s); + + c = compression_from_string(s); + if (c >= 0) + return c; + + return compression_uppercase_from_string(s); +} + +Compression compression_from_filename(const char *filename) { + Compression c; + const char *e; + + assert(filename); + + e = strrchr(filename, '.'); + if (!e) + return COMPRESSION_NONE; + + c = compression_extension_from_string(e); + if (c < 0) + return COMPRESSION_NONE; + + return c; +} bool compression_supported(Compression c) { static const unsigned supported = (1U << COMPRESSION_NONE) | (1U << COMPRESSION_XZ) * HAVE_XZ | (1U << COMPRESSION_LZ4) * HAVE_LZ4 | - (1U << COMPRESSION_ZSTD) * HAVE_ZSTD; + (1U << COMPRESSION_ZSTD) * HAVE_ZSTD | + (1U << COMPRESSION_GZIP) * HAVE_ZLIB | + (1U << COMPRESSION_BZIP2) * HAVE_BZIP2; assert(c >= 0); assert(c < _COMPRESSION_MAX); @@ -149,7 +266,7 @@ bool compression_supported(Compression c) { return BIT_SET(supported, c); } -int dlopen_lzma(void) { +int dlopen_xz(void) { #if HAVE_XZ SD_ELF_NOTE_DLOPEN( "lzma", @@ -171,56 +288,6 @@ int dlopen_lzma(void) { #endif } -int compress_blob_xz(const void *src, uint64_t src_size, - void *dst, size_t dst_alloc_size, size_t *dst_size, int level) { - - assert(src); - assert(src_size > 0); - assert(dst); - assert(dst_alloc_size > 0); - assert(dst_size); - -#if HAVE_XZ - lzma_options_lzma opt = { - 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT, - LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4 - }; - lzma_filter filters[] = { - { LZMA_FILTER_LZMA2, &opt }, - { LZMA_VLI_UNKNOWN, NULL } - }; - lzma_ret ret; - size_t out_pos = 0; - int r; - - r = dlopen_lzma(); - if (r < 0) - return r; - - if (level >= 0) { - r = sym_lzma_lzma_preset(&opt, (uint32_t) level); - if (r < 0) - return r; - } - - /* Returns < 0 if we couldn't compress the data or the - * compressed result is longer than the original */ - - if (src_size < 80) - return -ENOBUFS; - - ret = sym_lzma_stream_buffer_encode(filters, LZMA_CHECK_NONE, NULL, - src, src_size, dst, &out_pos, dst_alloc_size); - if (ret != LZMA_OK) - return -ENOBUFS; - - *dst_size = out_pos; - return 0; -#else - return -EPROTONOSUPPORT; -#endif -} - int dlopen_lz4(void) { #if HAVE_LZ4 SD_ELF_NOTE_DLOPEN( @@ -252,43 +319,6 @@ int dlopen_lz4(void) { #endif } -int compress_blob_lz4(const void *src, uint64_t src_size, - void *dst, size_t dst_alloc_size, size_t *dst_size, int level) { - - assert(src); - assert(src_size > 0); - assert(dst); - assert(dst_alloc_size > 0); - assert(dst_size); - -#if HAVE_LZ4 - int r; - - r = dlopen_lz4(); - if (r < 0) - return r; - /* Returns < 0 if we couldn't compress the data or the - * compressed result is longer than the original */ - - if (src_size < 9) - return -ENOBUFS; - - if (level <= 0) - r = sym_LZ4_compress_default(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8); - else - r = sym_LZ4_compress_HC(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8, level); - if (r <= 0) - return -ENOBUFS; - - unaligned_write_le64(dst, src_size); - *dst_size = r + 8; - - return 0; -#else - return -EPROTONOSUPPORT; -#endif -} - int dlopen_zstd(void) { #if HAVE_ZSTD SD_ELF_NOTE_DLOPEN( @@ -321,9 +351,159 @@ int dlopen_zstd(void) { #endif } -int compress_blob_zstd( - const void *src, uint64_t src_size, - void *dst, size_t dst_alloc_size, size_t *dst_size, int level) { +int dlopen_zlib(void) { +#if HAVE_ZLIB + SD_ELF_NOTE_DLOPEN( + "zlib", + "Support gzip compression and decompression", + SD_ELF_NOTE_DLOPEN_PRIORITY_SUGGESTED, + "libz.so.1"); + + return dlopen_many_sym_or_warn( + &zlib_dl, + "libz.so.1", LOG_DEBUG, + DLSYM_ARG(deflateInit2_), + DLSYM_ARG(deflate), + DLSYM_ARG(deflateEnd), + DLSYM_ARG(inflateInit2_), + DLSYM_ARG(inflate), + DLSYM_ARG(inflateEnd)); +#else + return -EOPNOTSUPP; +#endif +} + +int dlopen_bzip2(void) { +#if HAVE_BZIP2 + SD_ELF_NOTE_DLOPEN( + "bzip2", + "Support bzip2 compression and decompression", + SD_ELF_NOTE_DLOPEN_PRIORITY_SUGGESTED, + "libbz2.so.1"); + + return dlopen_many_sym_or_warn( + &bzip2_dl, + "libbz2.so.1", LOG_DEBUG, + DLSYM_ARG(BZ2_bzCompressInit), + DLSYM_ARG(BZ2_bzCompress), + DLSYM_ARG(BZ2_bzCompressEnd), + DLSYM_ARG(BZ2_bzDecompressInit), + DLSYM_ARG(BZ2_bzDecompress), + DLSYM_ARG(BZ2_bzDecompressEnd)); +#else + return -EOPNOTSUPP; +#endif +} + +static int compress_blob_xz( + const void *src, + uint64_t src_size, + void *dst, + size_t dst_alloc_size, + size_t *dst_size, + int level) { + + assert(src); + assert(src_size > 0); + assert(dst); + assert(dst_alloc_size > 0); + assert(dst_size); + +#if HAVE_XZ + lzma_options_lzma opt = { + 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT, + LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4 + }; + lzma_filter filters[] = { + { LZMA_FILTER_LZMA2, &opt }, + { LZMA_VLI_UNKNOWN, NULL } + }; + lzma_ret ret; + size_t out_pos = 0; + int r; + + r = dlopen_xz(); + if (r < 0) + return r; + + if (level >= 0) { + r = sym_lzma_lzma_preset(&opt, (uint32_t) level); + if (r < 0) + return r; + } + + /* Returns < 0 if we couldn't compress the data or the + * compressed result is longer than the original */ + + if (src_size < 80) + return -ENOBUFS; + + ret = sym_lzma_stream_buffer_encode(filters, LZMA_CHECK_NONE, NULL, + src, src_size, dst, &out_pos, dst_alloc_size); + if (ret != LZMA_OK) + return -ENOBUFS; + + *dst_size = out_pos; + return 0; +#else + return -EPROTONOSUPPORT; +#endif +} + +static int compress_blob_lz4( + const void *src, + uint64_t src_size, + void *dst, + size_t dst_alloc_size, + size_t *dst_size, + int level) { + + assert(src); + assert(src_size > 0); + assert(dst); + assert(dst_alloc_size > 0); + assert(dst_size); + +#if HAVE_LZ4 + int r; + + r = dlopen_lz4(); + if (r < 0) + return r; + /* Returns < 0 if we couldn't compress the data or the + * compressed result is longer than the original */ + + if (src_size < 9) + return -ENOBUFS; + + if (src_size > INT_MAX) + return -EFBIG; + if (dst_alloc_size > INT_MAX) + dst_alloc_size = INT_MAX; + + if (level <= 0) + r = sym_LZ4_compress_default(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8); + else + r = sym_LZ4_compress_HC(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8, level); + if (r <= 0) + return -ENOBUFS; + + unaligned_write_le64(dst, src_size); + *dst_size = r + 8; + + return 0; +#else + return -EPROTONOSUPPORT; +#endif +} + +static int compress_blob_zstd( + const void *src, + uint64_t src_size, + void *dst, + size_t dst_alloc_size, + size_t *dst_size, + int level) { assert(src); assert(src_size > 0); @@ -350,11 +530,125 @@ int compress_blob_zstd( #endif } -int decompress_blob_xz( +static int compress_blob_gzip(const void *src, uint64_t src_size, + void *dst, size_t dst_alloc_size, size_t *dst_size, int level) { + + assert(src); + assert(src_size > 0); + assert(dst); + assert(dst_alloc_size > 0); + assert(dst_size); + +#if HAVE_ZLIB + int r; + + r = dlopen_zlib(); + if (r < 0) + return r; + + if (src_size > UINT_MAX) + return -EFBIG; + if (dst_alloc_size > UINT_MAX) + dst_alloc_size = UINT_MAX; + + _cleanup_(deflateEnd_wrapper) z_stream s = {}; + + r = sym_deflateInit2_(&s, level < 0 ? Z_DEFAULT_COMPRESSION : level, + /* method= */ Z_DEFLATED, + /* windowBits= */ ZLIB_WBITS_GZIP, + /* memLevel= */ 8, + /* strategy= */ Z_DEFAULT_STRATEGY, + ZLIB_VERSION, (int) sizeof(s)); + if (r != Z_OK) + return -ENOMEM; + + s.next_in = (void*) src; + s.avail_in = src_size; + s.next_out = dst; + s.avail_out = dst_alloc_size; + + r = sym_deflate(&s, Z_FINISH); + if (r != Z_STREAM_END) + return -ENOBUFS; + + *dst_size = dst_alloc_size - s.avail_out; + return 0; +#else + return -EPROTONOSUPPORT; +#endif +} + +static int compress_blob_bzip2( + const void *src, uint64_t src_size, + void *dst, size_t dst_alloc_size, size_t *dst_size, int level) { + + assert(src); + assert(src_size > 0); + assert(dst); + assert(dst_alloc_size > 0); + assert(dst_size); + +#if HAVE_BZIP2 + int r; + + r = dlopen_bzip2(); + if (r < 0) + return r; + + if (src_size > UINT_MAX) + return -EFBIG; + if (dst_alloc_size > UINT_MAX) + dst_alloc_size = UINT_MAX; + + _cleanup_(BZ2_bzCompressEnd_wrapper) bz_stream s = {}; + + r = sym_BZ2_bzCompressInit(&s, level < 0 ? 9 : level, /* verbosity= */ 0, /* workFactor= */ 0); + if (r != BZ_OK) + return -ENOMEM; + + s.next_in = (char*) src; + s.avail_in = src_size; + s.next_out = (char*) dst; + s.avail_out = dst_alloc_size; + + r = sym_BZ2_bzCompress(&s, BZ_FINISH); + + if (r != BZ_STREAM_END) + return -ENOBUFS; + + *dst_size = dst_alloc_size - s.avail_out; + return 0; +#else + return -EPROTONOSUPPORT; +#endif +} + +int compress_blob( + Compression compression, + const void *src, uint64_t src_size, + void *dst, size_t dst_alloc_size, size_t *dst_size, int level) { + + switch (compression) { + case COMPRESSION_XZ: + return compress_blob_xz(src, src_size, dst, dst_alloc_size, dst_size, level); + case COMPRESSION_LZ4: + return compress_blob_lz4(src, src_size, dst, dst_alloc_size, dst_size, level); + case COMPRESSION_ZSTD: + return compress_blob_zstd(src, src_size, dst, dst_alloc_size, dst_size, level); + case COMPRESSION_GZIP: + return compress_blob_gzip(src, src_size, dst, dst_alloc_size, dst_size, level); + case COMPRESSION_BZIP2: + return compress_blob_bzip2(src, src_size, dst, dst_alloc_size, dst_size, level); + default: + return -EOPNOTSUPP; + } +} + +static int decompress_blob_xz( const void *src, uint64_t src_size, void **dst, - size_t* dst_size, + size_t *dst_size, size_t dst_max) { assert(src); @@ -365,12 +659,12 @@ int decompress_blob_xz( #if HAVE_XZ int r; - r = dlopen_lzma(); + r = dlopen_xz(); if (r < 0) return r; _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT; - lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, 0); + lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, /* flags= */ 0); if (ret != LZMA_OK) return -ENOMEM; @@ -416,11 +710,11 @@ int decompress_blob_xz( #endif } -int decompress_blob_lz4( +static int decompress_blob_lz4( const void *src, uint64_t src_size, void **dst, - size_t* dst_size, + size_t *dst_size, size_t dst_max) { assert(src); @@ -439,6 +733,9 @@ int decompress_blob_lz4( if (src_size <= 8) return -EBADMSG; + if (src_size - 8 > INT_MAX) + return -EFBIG; + size = unaligned_read_le64(src); if (size < 0 || (unsigned) size != unaligned_read_le64(src)) return -EFBIG; @@ -457,7 +754,7 @@ int decompress_blob_lz4( #endif } -int decompress_blob_zstd( +static int decompress_blob_zstd( const void *src, uint64_t src_size, void **dst, @@ -515,12 +812,146 @@ int decompress_blob_zstd( #endif } +static int decompress_blob_gzip( + const void *src, + uint64_t src_size, + void **dst, + size_t *dst_size, + size_t dst_max) { + + assert(src); + assert(src_size > 0); + assert(dst); + assert(dst_size); + +#if HAVE_ZLIB + int r; + + r = dlopen_zlib(); + if (r < 0) + return r; + + if (src_size > UINT_MAX) + return -EFBIG; + + _cleanup_(inflateEnd_wrapper) z_stream s = {}; + + r = sym_inflateInit2_(&s, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(s)); + if (r != Z_OK) + return -ENOMEM; + + size_t space = MIN3(src_size * 2, dst_max ?: SIZE_MAX, (size_t) UINT_MAX); + if (!greedy_realloc(dst, space, 1)) + return -ENOMEM; + + s.next_in = (void*) src; + s.avail_in = src_size; + s.next_out = *dst; + s.avail_out = space; + + for (;;) { + size_t used; + + r = sym_inflate(&s, Z_NO_FLUSH); + if (r == Z_STREAM_END) + break; + if (!IN_SET(r, Z_OK, Z_BUF_ERROR)) + return -EBADMSG; + + if (dst_max > 0 && (space - s.avail_out) >= dst_max) + break; + if (dst_max > 0 && space == dst_max) + return -ENOBUFS; + + used = space - s.avail_out; + space = MIN3(2 * space, dst_max ?: SIZE_MAX, UINT_MAX); + if (!greedy_realloc(dst, space, 1)) + return -ENOMEM; + + s.avail_out = space - used; + s.next_out = *(uint8_t**)dst + used; + } + + *dst_size = space - s.avail_out; + return 0; +#else + return -EPROTONOSUPPORT; +#endif +} + +static int decompress_blob_bzip2( + const void *src, + uint64_t src_size, + void **dst, + size_t *dst_size, + size_t dst_max) { + + assert(src); + assert(src_size > 0); + assert(dst); + assert(dst_size); + +#if HAVE_BZIP2 + int r; + + r = dlopen_bzip2(); + if (r < 0) + return r; + + if (src_size > UINT_MAX) + return -EFBIG; + + _cleanup_(BZ2_bzDecompressEnd_wrapper) bz_stream s = {}; + + r = sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0); + if (r != BZ_OK) + return -ENOMEM; + + size_t space = MIN3(src_size * 2, dst_max ?: SIZE_MAX, (size_t) UINT_MAX); + if (!greedy_realloc(dst, space, 1)) + return -ENOMEM; + + s.next_in = (char*) src; + s.avail_in = src_size; + s.next_out = (char*) *dst; + s.avail_out = space; + + for (;;) { + size_t used; + + r = sym_BZ2_bzDecompress(&s); + if (r == BZ_STREAM_END) + break; + if (r != BZ_OK) + return -EBADMSG; + + if (dst_max > 0 && (space - s.avail_out) >= dst_max) + break; + if (dst_max > 0 && space == dst_max) + return -ENOBUFS; + + used = space - s.avail_out; + space = MIN3(2 * space, dst_max ?: SIZE_MAX, (size_t) UINT_MAX); + if (!greedy_realloc(dst, space, 1)) + return -ENOMEM; + + s.avail_out = space - used; + s.next_out = (char*) *dst + used; + } + + *dst_size = space - s.avail_out; + return 0; +#else + return -EPROTONOSUPPORT; +#endif +} + int decompress_blob( Compression compression, const void *src, uint64_t src_size, void **dst, - size_t* dst_size, + size_t *dst_size, size_t dst_max) { switch (compression) { @@ -536,12 +967,62 @@ int decompress_blob( return decompress_blob_zstd( src, src_size, dst, dst_size, dst_max); + case COMPRESSION_GZIP: + return decompress_blob_gzip( + src, src_size, + dst, dst_size, dst_max); + case COMPRESSION_BZIP2: + return decompress_blob_bzip2( + src, src_size, + dst, dst_size, dst_max); default: return -EPROTONOSUPPORT; } } -int decompress_startswith_xz( +int decompress_zlib_raw( + const void *src, + uint64_t src_size, + void *dst, + size_t dst_size, + int wbits) { + +#if HAVE_ZLIB + int r; + + r = dlopen_zlib(); + if (r < 0) + return r; + + if (src_size > UINT_MAX) + return -EFBIG; + if (dst_size > UINT_MAX) + return -EFBIG; + + _cleanup_(inflateEnd_wrapper) z_stream s = { + .next_in = (void*) src, + .avail_in = src_size, + .next_out = dst, + .avail_out = dst_size, + }; + + r = sym_inflateInit2_(&s, /* windowBits= */ wbits, ZLIB_VERSION, (int) sizeof(s)); + if (r != Z_OK) + return -EIO; + + r = sym_inflate(&s, Z_FINISH); + size_t produced = (uint8_t*) s.next_out - (uint8_t*) dst; + + if (r != Z_STREAM_END || produced != dst_size) + return -EBADMSG; + + return 0; +#else + return -EPROTONOSUPPORT; +#endif +} + +static int decompress_startswith_xz( const void *src, uint64_t src_size, void **buffer, @@ -560,12 +1041,12 @@ int decompress_startswith_xz( #if HAVE_XZ int r; - r = dlopen_lzma(); + r = dlopen_xz(); if (r < 0) return r; _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT; - lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, 0); + lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, /* flags= */ 0); if (ret != LZMA_OK) return -EBADMSG; @@ -607,7 +1088,7 @@ int decompress_startswith_xz( #endif } -int decompress_startswith_lz4( +static int decompress_startswith_lz4( const void *src, uint64_t src_size, void **buffer, @@ -634,6 +1115,9 @@ int decompress_startswith_lz4( if (src_size <= 8) return -EBADMSG; + if (src_size - 8 > INT_MAX) + return -EFBIG; + if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1))) return -ENOMEM; allocated = MALLOC_SIZEOF_SAFE(*buffer); @@ -680,7 +1164,7 @@ int decompress_startswith_lz4( #endif } -int decompress_startswith_zstd( +static int decompress_startswith_zstd( const void *src, uint64_t src_size, void **buffer, @@ -738,8 +1222,7 @@ int decompress_startswith_zstd( #endif } -int decompress_startswith( - Compression compression, +static int decompress_startswith_gzip( const void *src, uint64_t src_size, void **buffer, @@ -747,212 +1230,234 @@ int decompress_startswith( size_t prefix_len, uint8_t extra) { - switch (compression) { + assert(src); + assert(src_size > 0); + assert(buffer); + assert(prefix); - case COMPRESSION_XZ: - return decompress_startswith_xz( - src, src_size, - buffer, - prefix, prefix_len, - extra); - - case COMPRESSION_LZ4: - return decompress_startswith_lz4( - src, src_size, - buffer, - prefix, prefix_len, - extra); - case COMPRESSION_ZSTD: - return decompress_startswith_zstd( - src, src_size, - buffer, - prefix, prefix_len, - extra); - default: - return -EBADMSG; - } -} - -int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) { - assert(fdf >= 0); - assert(fdt >= 0); - -#if HAVE_XZ +#if HAVE_ZLIB int r; - r = dlopen_lzma(); + r = dlopen_zlib(); if (r < 0) return r; - _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT; - lzma_ret ret = sym_lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64); - if (ret != LZMA_OK) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Failed to initialize XZ encoder: code %u", - ret); + if (src_size > UINT_MAX) + return -EFBIG; + + _cleanup_(inflateEnd_wrapper) z_stream s = {}; + + r = sym_inflateInit2_(&s, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(s)); + if (r != Z_OK) + return -EBADMSG; + + if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1))) + return -ENOMEM; + + size_t allocated = MALLOC_SIZEOF_SAFE(*buffer); + + s.next_in = (void*) src; + s.avail_in = src_size; + + s.next_out = *buffer; + s.avail_out = MIN(allocated, (size_t) UINT_MAX); - uint8_t buf[BUFSIZ], out[BUFSIZ]; - lzma_action action = LZMA_RUN; for (;;) { - if (s.avail_in == 0 && action == LZMA_RUN) { - size_t m = sizeof(buf); - ssize_t n; + r = sym_inflate(&s, Z_FINISH); - if (max_bytes != UINT64_MAX && (uint64_t) m > max_bytes) - m = (size_t) max_bytes; + if (!IN_SET(r, Z_OK, Z_STREAM_END, Z_BUF_ERROR)) + return -EBADMSG; - n = read(fdf, buf, m); - if (n < 0) - return -errno; - if (n == 0) - action = LZMA_FINISH; - else { - s.next_in = buf; - s.avail_in = n; + if (allocated - s.avail_out >= prefix_len + 1) + return memcmp(*buffer, prefix, prefix_len) == 0 && + ((const uint8_t*) *buffer)[prefix_len] == extra; - if (max_bytes != UINT64_MAX) { - assert(max_bytes >= (uint64_t) n); - max_bytes -= n; - } - } - } + if (r == Z_STREAM_END) + return 0; - if (s.avail_out == 0) { - s.next_out = out; - s.avail_out = sizeof(out); - } + size_t used = allocated - s.avail_out; - ret = sym_lzma_code(&s, action); - if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END)) - return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), - "Compression failed: code %u", - ret); + if (!(greedy_realloc(buffer, allocated * 2, 1))) + return -ENOMEM; - if (s.avail_out == 0 || ret == LZMA_STREAM_END) { - ssize_t n, k; - - n = sizeof(out) - s.avail_out; - - k = loop_write(fdt, out, n); - if (k < 0) - return k; - - if (ret == LZMA_STREAM_END) { - if (ret_uncompressed_size) - *ret_uncompressed_size = s.total_in; - - if (s.total_in == 0) - log_debug("XZ compression finished (no input data)"); - else - log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)", - s.total_in, s.total_out, - (double) s.total_out / s.total_in * 100); - - return 0; - } - } + allocated = MALLOC_SIZEOF_SAFE(*buffer); + s.avail_out = MIN(allocated - used, (size_t) UINT_MAX); + s.next_out = *(uint8_t**)buffer + used; } #else return -EPROTONOSUPPORT; #endif } -#define LZ4_BUFSIZE (512*1024u) +static int decompress_startswith_bzip2( + const void *src, + uint64_t src_size, + void **buffer, + const void *prefix, + size_t prefix_len, + uint8_t extra) { -int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) { + assert(src); + assert(src_size > 0); + assert(buffer); + assert(prefix); -#if HAVE_LZ4 - LZ4F_errorCode_t c; - _cleanup_(LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL; - _cleanup_free_ void *in_buff = NULL; - _cleanup_free_ char *out_buff = NULL; - size_t out_allocsize, n, offset = 0, frame_size; - uint64_t total_in = 0, total_out; +#if HAVE_BZIP2 int r; - static const LZ4F_preferences_t preferences = { - .frameInfo.blockSizeID = 5, - }; - r = dlopen_lz4(); + r = dlopen_bzip2(); if (r < 0) return r; - c = sym_LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); - if (sym_LZ4F_isError(c)) + if (src_size > UINT_MAX) + return -EFBIG; + + _cleanup_(BZ2_bzDecompressEnd_wrapper) bz_stream s = {}; + + r = sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0); + if (r != BZ_OK) + return -EBADMSG; + + if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1))) return -ENOMEM; - frame_size = sym_LZ4F_compressBound(LZ4_BUFSIZE, &preferences); - out_allocsize = frame_size + 64*1024; /* add some space for header and trailer */ - out_buff = malloc(out_allocsize); - if (!out_buff) - return -ENOMEM; + size_t allocated = MALLOC_SIZEOF_SAFE(*buffer); - in_buff = malloc(LZ4_BUFSIZE); - if (!in_buff) - return -ENOMEM; + s.next_in = (char*) src; + s.avail_in = src_size; - n = offset = total_out = sym_LZ4F_compressBegin(ctx, out_buff, out_allocsize, &preferences); - if (sym_LZ4F_isError(n)) - return -EINVAL; - - log_debug("Buffer size is %zu bytes, header size %zu bytes.", out_allocsize, n); + s.next_out = *buffer; + s.avail_out = MIN(allocated, (size_t) UINT_MAX); for (;;) { - ssize_t k; + r = sym_BZ2_bzDecompress(&s); - k = loop_read(fdf, in_buff, LZ4_BUFSIZE, true); - if (k < 0) - return k; - if (k == 0) - break; - n = sym_LZ4F_compressUpdate(ctx, out_buff + offset, out_allocsize - offset, - in_buff, k, NULL); - if (sym_LZ4F_isError(n)) - return -ENOTRECOVERABLE; + if (!IN_SET(r, BZ_OK, BZ_STREAM_END)) + return -EBADMSG; - total_in += k; - offset += n; - total_out += n; + if (allocated - s.avail_out >= prefix_len + 1) + return memcmp(*buffer, prefix, prefix_len) == 0 && + ((const uint8_t*) *buffer)[prefix_len] == extra; - if (max_bytes != UINT64_MAX && total_out > (size_t) max_bytes) - return log_debug_errno(SYNTHETIC_ERRNO(EFBIG), - "Compressed stream longer than %" PRIu64 " bytes", max_bytes); + if (r == BZ_STREAM_END) + return 0; - if (out_allocsize - offset < frame_size + 4) { - k = loop_write(fdt, out_buff, offset); - if (k < 0) - return k; - offset = 0; - } + size_t used = allocated - s.avail_out; + + if (!(greedy_realloc(buffer, allocated * 2, 1))) + return -ENOMEM; + + allocated = MALLOC_SIZEOF_SAFE(*buffer); + s.avail_out = MIN(allocated - used, (size_t) UINT_MAX); + s.next_out = (char*) *buffer + used; } +#else + return -EPROTONOSUPPORT; +#endif +} - n = sym_LZ4F_compressEnd(ctx, out_buff + offset, out_allocsize - offset, NULL); - if (sym_LZ4F_isError(n)) - return -ENOTRECOVERABLE; +int decompress_startswith( + Compression compression, + const void *src, uint64_t src_size, + void **buffer, + const void *prefix, size_t prefix_len, + uint8_t extra) { - offset += n; - total_out += n; - r = loop_write(fdt, out_buff, offset); + switch (compression) { + case COMPRESSION_XZ: + return decompress_startswith_xz(src, src_size, buffer, prefix, prefix_len, extra); + case COMPRESSION_LZ4: + return decompress_startswith_lz4(src, src_size, buffer, prefix, prefix_len, extra); + case COMPRESSION_ZSTD: + return decompress_startswith_zstd(src, src_size, buffer, prefix, prefix_len, extra); + case COMPRESSION_GZIP: + return decompress_startswith_gzip(src, src_size, buffer, prefix, prefix_len, extra); + case COMPRESSION_BZIP2: + return decompress_startswith_bzip2(src, src_size, buffer, prefix, prefix_len, extra); + default: + return -EOPNOTSUPP; + } +} + +int compress_stream( + Compression type, + int fdf, int fdt, + uint64_t max_bytes, + uint64_t *ret_uncompressed_size) { + + _cleanup_(compressor_freep) Compressor *c = NULL; + _cleanup_free_ void *buf = NULL; + _cleanup_free_ uint8_t *input = NULL; + size_t buf_size = 0, buf_alloc = 0; + uint64_t total_in = 0, total_out = 0; + int r; + + assert(fdf >= 0); + assert(fdt >= 0); + + r = compressor_new(&c, type); if (r < 0) return r; + input = new(uint8_t, COMPRESS_PIPE_BUFFER_SIZE); + if (!input) + return -ENOMEM; + + for (;;) { + size_t m = COMPRESS_PIPE_BUFFER_SIZE; + ssize_t n; + + if (max_bytes != UINT64_MAX && (uint64_t) m > max_bytes) + m = (size_t) max_bytes; + + n = read(fdf, input, m); + if (n < 0) + return -errno; + + if (n == 0) { + r = compressor_finish(c, &buf, &buf_size, &buf_alloc); + if (r < 0) + return r; + + if (buf_size > 0) { + r = loop_write(fdt, buf, buf_size); + if (r < 0) + return r; + total_out += buf_size; + } + break; + } + + total_in += n; + if (max_bytes != UINT64_MAX) { + assert(max_bytes >= (uint64_t) n); + max_bytes -= n; + } + + r = compressor_start(c, input, n, &buf, &buf_size, &buf_alloc); + if (r < 0) + return r; + + if (buf_size > 0) { + r = loop_write(fdt, buf, buf_size); + if (r < 0) + return r; + total_out += buf_size; + } + } + if (ret_uncompressed_size) *ret_uncompressed_size = total_in; if (total_in == 0) - log_debug("LZ4 compression finished (no input data)"); + log_debug("%s compression finished (no input data)", compression_to_string(type)); else - log_debug("LZ4 compression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)", - total_in, total_out, - (double) total_out / total_in * 100); + log_debug("%s compression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)", + compression_to_string(type), total_in, total_out, (double) total_out / total_in * 100); return 0; -#else - return -EPROTONOSUPPORT; -#endif } -#if HAVE_COMPRESSION /* Determine whether sparse writes should be used for this fd. Sparse writes are only safe on * regular files without O_APPEND (O_APPEND ignores lseek position, which would collapse holes). */ static int should_sparse(int fd) { @@ -987,414 +1492,1049 @@ static int finalize_sparse(int fd) { return 0; } -static int maybe_sparse_write(int fd, const void *buf, size_t nbytes, bool sparse) { - int r; +/* Common helper for decompress_stream_*() wrappers */ - if (sparse) { - ssize_t k; +struct decompress_stream_userdata { + int fd; + uint64_t max_bytes; + uint64_t total_out; + bool sparse; +}; +static int decompress_stream_write_callback(const void *data, size_t size, void *userdata) { + struct decompress_stream_userdata *u = ASSERT_PTR(userdata); + + if (u->max_bytes != UINT64_MAX) { + if (u->max_bytes < size) + return -EFBIG; + u->max_bytes -= size; + } + + u->total_out += size; + + if (u->sparse) { /* Note: sparse_write() does not retry on EINTR and converts short writes to -EIO. * This is fine here since sparse mode is only used on regular files, where short * writes and EINTR are not expected in practice. */ - k = sparse_write(fd, buf, nbytes, 64); + ssize_t k = sparse_write(u->fd, data, size, 64); if (k < 0) return (int) k; - } else { - r = loop_write_full(fd, buf, nbytes, USEC_INFINITY); + return 0; + } + + return loop_write(u->fd, data, size); +} + +static int decompressor_new(Decompressor **ret, Compression type) { +#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2 + int r; +#endif + + assert(ret); + + _cleanup_(compressor_freep) Decompressor *c = new0(Decompressor, 1); + if (!c) + return -ENOMEM; + + c->type = _COMPRESSION_INVALID; + + switch (type) { + +#if HAVE_XZ + case COMPRESSION_XZ: + r = dlopen_xz(); + if (r < 0) + return r; + + if (sym_lzma_stream_decoder(&c->xz, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED) != LZMA_OK) + return -EIO; + break; +#endif + +#if HAVE_LZ4 + case COMPRESSION_LZ4: { + r = dlopen_lz4(); + if (r < 0) + return r; + + size_t rc = sym_LZ4F_createDecompressionContext(&c->d_lz4, LZ4F_VERSION); + if (sym_LZ4F_isError(rc)) + return -ENOMEM; + + break; + } +#endif + +#if HAVE_ZSTD + case COMPRESSION_ZSTD: + r = dlopen_zstd(); + if (r < 0) + return r; + + c->d_zstd = sym_ZSTD_createDCtx(); + if (!c->d_zstd) + return -ENOMEM; + break; +#endif + +#if HAVE_ZLIB + case COMPRESSION_GZIP: + r = dlopen_zlib(); + if (r < 0) + return r; + + r = sym_inflateInit2_(&c->gzip, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(c->gzip)); + if (r != Z_OK) + return -EIO; + break; +#endif + +#if HAVE_BZIP2 + case COMPRESSION_BZIP2: + r = dlopen_bzip2(); + if (r < 0) + return r; + + r = sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0); + if (r != BZ_OK) + return -EIO; + break; +#endif + + default: + return -EOPNOTSUPP; + } + + c->type = type; + c->encoding = false; + *ret = TAKE_PTR(c); + return 0; +} + +int decompress_stream( + Compression type, + int fdf, int fdt, + uint64_t max_bytes) { + + _cleanup_(compressor_freep) Decompressor *c = NULL; + _cleanup_free_ uint8_t *buf = NULL; + uint64_t total_in = 0; + int r; + + assert(fdf >= 0); + assert(fdt >= 0); + + r = decompressor_new(&c, type); + if (r < 0) + return r; + + struct decompress_stream_userdata userdata = { + .fd = fdt, + .max_bytes = max_bytes, + .sparse = should_sparse(fdt) > 0, + }; + + buf = new(uint8_t, COMPRESS_PIPE_BUFFER_SIZE); + if (!buf) + return -ENOMEM; + + for (;;) { + ssize_t n; + + n = read(fdf, buf, COMPRESS_PIPE_BUFFER_SIZE); + if (n < 0) + return -errno; + if (n == 0) + break; + + total_in += n; + + r = decompressor_push(c, buf, n, decompress_stream_write_callback, &userdata); if (r < 0) return r; } + if (total_in == 0) + return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "%s decompression failed: no data read", + compression_to_string(type)); + + if (userdata.sparse) { + r = finalize_sparse(fdt); + if (r < 0) + return r; + } + + log_debug("%s decompression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)", + compression_to_string(type), total_in, userdata.total_out, + (double) userdata.total_out / total_in * 100); + return 0; } -#endif -int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) { - assert(fdf >= 0); - assert(fdt >= 0); +int decompress_stream_by_filename(const char *filename, int fdf, int fdt, uint64_t max_bytes) { + Compression c = compression_from_filename(filename); + if (c == COMPRESSION_NONE) + return -EPROTONOSUPPORT; + + return decompress_stream(c, fdf, fdt, max_bytes); +} + +/* Push-based streaming compression/decompression context API */ + +Compressor* compressor_free(Compressor *c) { + if (!c) + return NULL; + + switch (c->type) { #if HAVE_XZ - bool sparse = should_sparse(fdt) > 0; - int r; - - r = dlopen_lzma(); - if (r < 0) - return r; - - _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT; - lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, 0); - if (ret != LZMA_OK) - return log_debug_errno(SYNTHETIC_ERRNO(ENOMEM), - "Failed to initialize XZ decoder: code %u", - ret); - - uint8_t buf[BUFSIZ], out[BUFSIZ]; - lzma_action action = LZMA_RUN; - for (;;) { - if (s.avail_in == 0 && action == LZMA_RUN) { - ssize_t n; - - n = read(fdf, buf, sizeof(buf)); - if (n < 0) - return -errno; - if (n == 0) - action = LZMA_FINISH; - else { - s.next_in = buf; - s.avail_in = n; - } - } - - if (s.avail_out == 0) { - s.next_out = out; - s.avail_out = sizeof(out); - } - - ret = sym_lzma_code(&s, action); - if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END)) - return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), - "Decompression failed: code %u", - ret); - - if (s.avail_out == 0 || ret == LZMA_STREAM_END) { - ssize_t n, k; - - n = sizeof(out) - s.avail_out; - - if (max_bytes != UINT64_MAX) { - if (max_bytes < (uint64_t) n) - return -EFBIG; - - max_bytes -= n; - } - - k = maybe_sparse_write(fdt, out, n, sparse); - if (k < 0) - return k; - - if (ret == LZMA_STREAM_END) { - if (s.total_in == 0) - log_debug("XZ decompression finished (no input data)"); - else - log_debug("XZ decompression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)", - s.total_in, s.total_out, - (double) s.total_out / s.total_in * 100); - - return sparse ? finalize_sparse(fdt) : 0; - } - } - } -#else - return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT), - "Cannot decompress file. Compiled without XZ support."); + case COMPRESSION_XZ: + sym_lzma_end(&c->xz); + break; #endif -} -int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) { #if HAVE_LZ4 - size_t c; - _cleanup_(LZ4F_freeDecompressionContextp) LZ4F_decompressionContext_t ctx = NULL; - _cleanup_free_ char *buf = NULL; - char *src; - struct stat st; - bool sparse = should_sparse(fdt) > 0; + case COMPRESSION_LZ4: + if (c->encoding) { + sym_LZ4F_freeCompressionContext(c->c_lz4); + c->c_lz4 = NULL; + c->lz4_header = mfree(c->lz4_header); + } else { + sym_LZ4F_freeDecompressionContext(c->d_lz4); + c->d_lz4 = NULL; + } + break; +#endif + +#if HAVE_ZSTD + case COMPRESSION_ZSTD: + if (c->encoding) { + sym_ZSTD_freeCCtx(c->c_zstd); + c->c_zstd = NULL; + } else { + sym_ZSTD_freeDCtx(c->d_zstd); + c->d_zstd = NULL; + } + break; +#endif + +#if HAVE_ZLIB + case COMPRESSION_GZIP: + if (c->encoding) + sym_deflateEnd(&c->gzip); + else + sym_inflateEnd(&c->gzip); + break; +#endif + +#if HAVE_BZIP2 + case COMPRESSION_BZIP2: + if (c->encoding) + sym_BZ2_bzCompressEnd(&c->bzip2); + else + sym_BZ2_bzDecompressEnd(&c->bzip2); + break; +#endif + + default: + break; + } + + return mfree(c); +} + +Compression compressor_type(const Compressor *c) { + return c ? c->type : _COMPRESSION_INVALID; +} + +int decompressor_detect(Decompressor **ret, const void *data, size_t size) { + static const uint8_t xz_signature[] = { + 0xfd, '7', 'z', 'X', 'Z', 0x00 + }; + static const uint8_t lz4_signature[] = { + 0x04, 0x22, 0x4d, 0x18 + }; + static const uint8_t zstd_signature[] = { + 0x28, 0xb5, 0x2f, 0xfd + }; + static const uint8_t gzip_signature[] = { + 0x1f, 0x8b + }; + static const uint8_t bzip2_signature[] = { + 'B', 'Z', 'h' + }; + +#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2 int r; - size_t total_in = 0, total_out = 0; +#endif - r = dlopen_lz4(); - if (r < 0) - return r; + assert(ret); - c = sym_LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION); - if (sym_LZ4F_isError(c)) + if (*ret) + return 1; + + if (size < MAX5(sizeof(xz_signature), + sizeof(gzip_signature), + sizeof(zstd_signature), + sizeof(bzip2_signature), + sizeof(lz4_signature))) + return 0; + + assert(data); + + _cleanup_(compressor_freep) Decompressor *c = new0(Decompressor, 1); + if (!c) return -ENOMEM; - if (fstat(fdf, &st) < 0) - return log_debug_errno(errno, "fstat() failed: %m"); + c->type = COMPRESSION_NONE; - if (file_offset_beyond_memory_size(st.st_size)) - return -EFBIG; - - buf = malloc(LZ4_BUFSIZE); - if (!buf) - return -ENOMEM; - - src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fdf, 0); - if (src == MAP_FAILED) - return -errno; - - while (total_in < (size_t) st.st_size) { - size_t produced = LZ4_BUFSIZE; - size_t used = st.st_size - total_in; - - c = sym_LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL); - if (sym_LZ4F_isError(c)) { - r = -EBADMSG; - goto cleanup; - } - - total_in += used; - total_out += produced; - - if (max_bytes != UINT64_MAX && total_out > (size_t) max_bytes) { - log_debug("Decompressed stream longer than %"PRIu64" bytes", max_bytes); - r = -EFBIG; - goto cleanup; - } - - r = maybe_sparse_write(fdt, buf, produced, sparse); +#if HAVE_XZ + if (c->type == COMPRESSION_NONE && memcmp(data, xz_signature, sizeof(xz_signature)) == 0) { + r = dlopen_xz(); if (r < 0) - goto cleanup; + return r; + + lzma_ret xzr = sym_lzma_stream_decoder(&c->xz, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED); + if (xzr != LZMA_OK) + return -EIO; + + c->type = COMPRESSION_XZ; } - - if (total_in == 0) - log_debug("LZ4 decompression finished (no input data)"); - else - log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)", - total_in, total_out, - (double) total_out / total_in * 100); - r = sparse ? finalize_sparse(fdt) : 0; - cleanup: - munmap(src, st.st_size); - return r; -#else - return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT), - "Cannot decompress file. Compiled without LZ4 support."); #endif -} -int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) { - assert(fdf >= 0); - assert(fdt >= 0); +#if HAVE_LZ4 + if (c->type == COMPRESSION_NONE && memcmp(data, lz4_signature, sizeof(lz4_signature)) == 0) { + r = dlopen_lz4(); + if (r < 0) + return r; + + size_t rc = sym_LZ4F_createDecompressionContext(&c->d_lz4, LZ4F_VERSION); + if (sym_LZ4F_isError(rc)) + return -ENOMEM; + + c->type = COMPRESSION_LZ4; + } +#endif #if HAVE_ZSTD - _cleanup_(ZSTD_freeCCtxp) ZSTD_CCtx *cctx = NULL; - _cleanup_free_ void *in_buff = NULL, *out_buff = NULL; - size_t in_allocsize, out_allocsize; - size_t z; - uint64_t left = max_bytes, in_bytes = 0; - int r; + if (c->type == COMPRESSION_NONE && memcmp(data, zstd_signature, sizeof(zstd_signature)) == 0) { + r = dlopen_zstd(); + if (r < 0) + return r; - r = dlopen_zstd(); - if (r < 0) - return r; + c->d_zstd = sym_ZSTD_createDCtx(); + if (!c->d_zstd) + return -ENOMEM; - /* Create the context and buffers */ - in_allocsize = sym_ZSTD_CStreamInSize(); - out_allocsize = sym_ZSTD_CStreamOutSize(); - in_buff = malloc(in_allocsize); - out_buff = malloc(out_allocsize); - cctx = sym_ZSTD_createCCtx(); - if (!cctx || !out_buff || !in_buff) + c->type = COMPRESSION_ZSTD; + } +#endif + +#if HAVE_ZLIB + if (c->type == COMPRESSION_NONE && memcmp(data, gzip_signature, sizeof(gzip_signature)) == 0) { + r = dlopen_zlib(); + if (r < 0) + return r; + + r = sym_inflateInit2_(&c->gzip, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(c->gzip)); + if (r != Z_OK) + return -EIO; + + c->type = COMPRESSION_GZIP; + } +#endif + +#if HAVE_BZIP2 + if (c->type == COMPRESSION_NONE && memcmp(data, bzip2_signature, sizeof(bzip2_signature)) == 0) { + r = dlopen_bzip2(); + if (r < 0) + return r; + + r = sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0); + if (r != BZ_OK) + return -EIO; + + c->type = COMPRESSION_BZIP2; + } +#endif + + c->encoding = false; + + log_debug("Detected compression type: %s", compression_to_string(c->type)); + *ret = TAKE_PTR(c); + return 1; +} + +int decompressor_force_off(Decompressor **ret) { + assert(ret); + + *ret = compressor_free(*ret); + + Decompressor *c = new0(Decompressor, 1); + if (!c) return -ENOMEM; - z = sym_ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); - if (sym_ZSTD_isError(z)) - log_debug("Failed to enable ZSTD checksum, ignoring: %s", sym_ZSTD_getErrorName(z)); + c->type = COMPRESSION_NONE; + c->encoding = false; + *ret = c; + return 0; +} - /* This loop read from the input file, compresses that entire chunk, - * and writes all output produced to the output file. - */ - for (;;) { - bool is_last_chunk; - ZSTD_inBuffer input = { - .src = in_buff, - .size = 0, - .pos = 0 - }; - ssize_t red; +int decompressor_push(Decompressor *c, const void *data, size_t size, DecompressorCallback callback, void *userdata) { +#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2 + _cleanup_free_ uint8_t *buffer = NULL; +#endif + int r; - red = loop_read(fdf, in_buff, in_allocsize, true); - if (red < 0) - return red; - is_last_chunk = red == 0; + assert(c); + assert(callback); - in_bytes += (size_t) red; - input.size = (size_t) red; + if (c->encoding) + return -EINVAL; - for (bool finished = false; !finished;) { - ZSTD_outBuffer output = { - .dst = out_buff, - .size = out_allocsize, - .pos = 0 - }; - size_t remaining; - ssize_t wrote; + if (size == 0) + return 1; - /* Compress into the output buffer and write all of the - * output to the file so we can reuse the buffer next - * iteration. - */ - remaining = sym_ZSTD_compressStream2( - cctx, &output, &input, - is_last_chunk ? ZSTD_e_end : ZSTD_e_continue); + assert(data); - if (sym_ZSTD_isError(remaining)) { - log_debug("ZSTD encoder failed: %s", sym_ZSTD_getErrorName(remaining)); - return zstd_ret_to_errno(remaining); +#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2 + if (c->type != COMPRESSION_NONE) { + buffer = new(uint8_t, COMPRESS_PIPE_BUFFER_SIZE); + if (!buffer) + return -ENOMEM; + } +#endif + + switch (c->type) { + + case COMPRESSION_NONE: + r = callback(data, size, userdata); + if (r < 0) + return r; + + break; + +#if HAVE_XZ + case COMPRESSION_XZ: + c->xz.next_in = data; + c->xz.avail_in = size; + + while (c->xz.avail_in > 0) { + c->xz.next_out = buffer; + c->xz.avail_out = COMPRESS_PIPE_BUFFER_SIZE; + + lzma_ret lzr = sym_lzma_code(&c->xz, LZMA_RUN); + if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END)) + return -EBADMSG; + + if (c->xz.avail_out < COMPRESS_PIPE_BUFFER_SIZE) { + r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->xz.avail_out, userdata); + if (r < 0) + return r; } - - if (left < output.pos) - return -EFBIG; - - wrote = loop_write_full(fdt, output.dst, output.pos, USEC_INFINITY); - if (wrote < 0) - return wrote; - - left -= output.pos; - - /* If we're on the last chunk we're finished when zstd - * returns 0, which means its consumed all the input AND - * finished the frame. Otherwise, we're finished when - * we've consumed all the input. - */ - finished = is_last_chunk ? (remaining == 0) : (input.pos == input.size); } - /* zstd only returns 0 when the input is completely consumed */ - assert(input.pos == input.size); - if (is_last_chunk) - break; - } - - if (ret_uncompressed_size) - *ret_uncompressed_size = in_bytes; - - if (in_bytes == 0) - log_debug("ZSTD compression finished (no input data)"); - else - log_debug("ZSTD compression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)", - in_bytes, max_bytes - left, (double) (max_bytes - left) / in_bytes * 100); - - return 0; -#else - return -EPROTONOSUPPORT; + break; #endif -} -int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) { - assert(fdf >= 0); - assert(fdt >= 0); +#if HAVE_LZ4 + case COMPRESSION_LZ4: { + const uint8_t *src = data; + size_t src_remaining = size; + + while (src_remaining > 0) { + size_t produced = COMPRESS_PIPE_BUFFER_SIZE; + size_t consumed = src_remaining; + + size_t rc = sym_LZ4F_decompress(c->d_lz4, buffer, &produced, src, &consumed, NULL); + if (sym_LZ4F_isError(rc)) + return -EBADMSG; + + if (consumed == 0 && produced == 0) + break; /* No progress possible with current input */ + + src += consumed; + src_remaining -= consumed; + + if (produced > 0) { + r = callback(buffer, produced, userdata); + if (r < 0) + return r; + } + } + + break; + } +#endif #if HAVE_ZSTD - _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = NULL; - _cleanup_free_ void *in_buff = NULL, *out_buff = NULL; - bool sparse = should_sparse(fdt) > 0; - size_t in_allocsize, out_allocsize; - size_t last_result = 0; - uint64_t left = max_bytes, in_bytes = 0; - int r; - - r = dlopen_zstd(); - if (r < 0) - return r; - /* Create the context and buffers */ - in_allocsize = sym_ZSTD_DStreamInSize(); - out_allocsize = sym_ZSTD_DStreamOutSize(); - in_buff = malloc(in_allocsize); - out_buff = malloc(out_allocsize); - dctx = sym_ZSTD_createDCtx(); - if (!dctx || !out_buff || !in_buff) - return -ENOMEM; - - /* This loop assumes that the input file is one or more concatenated - * zstd streams. This example won't work if there is trailing non-zstd - * data at the end, but streaming decompression in general handles this - * case. ZSTD_decompressStream() returns 0 exactly when the frame is - * completed, and doesn't consume input after the frame. - */ - for (;;) { - bool has_error = false; + case COMPRESSION_ZSTD: { ZSTD_inBuffer input = { - .src = in_buff, - .size = 0, - .pos = 0 + .src = (void*) data, + .size = size, }; - ssize_t red; - red = loop_read(fdf, in_buff, in_allocsize, true); - if (red < 0) - return red; - if (red == 0) - break; - - in_bytes += (size_t) red; - input.size = (size_t) red; - input.pos = 0; - - /* Given a valid frame, zstd won't consume the last byte of the - * frame until it has flushed all of the decompressed data of - * the frame. So input.pos < input.size means frame is not done - * or there is still output available. - */ while (input.pos < input.size) { ZSTD_outBuffer output = { - .dst = out_buff, - .size = out_allocsize, - .pos = 0 + .dst = buffer, + .size = COMPRESS_PIPE_BUFFER_SIZE, }; - ssize_t wrote; - /* The return code is zero if the frame is complete, but - * there may be multiple frames concatenated together. - * Zstd will automatically reset the context when a - * frame is complete. Still, calling ZSTD_DCtx_reset() - * can be useful to reset the context to a clean state, - * for instance if the last decompression call returned - * an error. - */ - last_result = sym_ZSTD_decompressStream(dctx, &output, &input); - if (sym_ZSTD_isError(last_result)) { - has_error = true; - break; + + size_t res = sym_ZSTD_decompressStream(c->d_zstd, &output, &input); + if (sym_ZSTD_isError(res)) + return -EBADMSG; + + if (output.pos > 0) { + r = callback(output.dst, output.pos, userdata); + if (r < 0) + return r; + } + } + + break; + } +#endif + +#if HAVE_ZLIB + case COMPRESSION_GZIP: + if (size > UINT_MAX) + return -EFBIG; + + c->gzip.next_in = (void*) data; + c->gzip.avail_in = size; + + while (c->gzip.avail_in > 0) { + c->gzip.next_out = buffer; + c->gzip.avail_out = COMPRESS_PIPE_BUFFER_SIZE; + + int zr = sym_inflate(&c->gzip, Z_NO_FLUSH); + if (!IN_SET(zr, Z_OK, Z_STREAM_END)) + return -EBADMSG; + + if (c->gzip.avail_out < COMPRESS_PIPE_BUFFER_SIZE) { + r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->gzip.avail_out, userdata); + if (r < 0) + return r; } - if (left < output.pos) - return -EFBIG; - - wrote = maybe_sparse_write(fdt, output.dst, output.pos, sparse); - if (wrote < 0) - return wrote; - - left -= output.pos; + if (zr == Z_STREAM_END) + break; } - if (has_error) - break; - } - if (in_bytes == 0) - return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "ZSTD decoder failed: no data read"); - - if (last_result != 0) { - /* The last return value from ZSTD_decompressStream did not end - * on a frame, but we reached the end of the file! We assume - * this is an error, and the input was truncated. - */ - log_debug("ZSTD decoder failed: %s", sym_ZSTD_getErrorName(last_result)); - return zstd_ret_to_errno(last_result); - } - - if (in_bytes == 0) - log_debug("ZSTD decompression finished (no input data)"); - else - log_debug("ZSTD decompression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)", - in_bytes, - max_bytes - left, - (double) (max_bytes - left) / in_bytes * 100); - return sparse ? finalize_sparse(fdt) : 0; -#else - return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT), - "Cannot decompress file. Compiled without ZSTD support."); + break; #endif + +#if HAVE_BZIP2 + case COMPRESSION_BZIP2: + if (size > UINT_MAX) + return -EFBIG; + + c->bzip2.next_in = (char*) data; + c->bzip2.avail_in = size; + + while (c->bzip2.avail_in > 0) { + c->bzip2.next_out = (char*) buffer; + c->bzip2.avail_out = COMPRESS_PIPE_BUFFER_SIZE; + + int bzr = sym_BZ2_bzDecompress(&c->bzip2); + if (!IN_SET(bzr, BZ_OK, BZ_STREAM_END)) + return -EBADMSG; + + if (c->bzip2.avail_out < COMPRESS_PIPE_BUFFER_SIZE) { + r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->bzip2.avail_out, userdata); + if (r < 0) + return r; + } + + if (bzr == BZ_STREAM_END) + break; + } + + break; +#endif + + default: + assert_not_reached(); + } + + return 1; } -int decompress_stream(const char *filename, int fdf, int fdt, uint64_t max_bytes) { +int compressor_new(Compressor **ret, Compression type) { +#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2 + int r; +#endif - if (endswith(filename, ".lz4")) - return decompress_stream_lz4(fdf, fdt, max_bytes); - if (endswith(filename, ".xz")) - return decompress_stream_xz(fdf, fdt, max_bytes); - if (endswith(filename, ".zst")) - return decompress_stream_zstd(fdf, fdt, max_bytes); + assert(ret); - return -EPROTONOSUPPORT; + _cleanup_(compressor_freep) Compressor *c = new0(Compressor, 1); + if (!c) + return -ENOMEM; + + c->type = _COMPRESSION_INVALID; + /* Set encoding early so that compressor_freep calls the correct cleanup (compression vs + * decompression) if any operation in the switch fails after setting c->type. This is safe + * because _COMPRESSION_INVALID hits the default: break case regardless of the encoding flag. */ + c->encoding = true; + + switch (type) { + +#if HAVE_XZ + case COMPRESSION_XZ: { + r = dlopen_xz(); + if (r < 0) + return r; + + lzma_ret xzr = sym_lzma_easy_encoder(&c->xz, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64); + if (xzr != LZMA_OK) + return -EIO; + + c->type = COMPRESSION_XZ; + break; + } +#endif + +#if HAVE_LZ4 + case COMPRESSION_LZ4: { + r = dlopen_lz4(); + if (r < 0) + return r; + + size_t rc = sym_LZ4F_createCompressionContext(&c->c_lz4, LZ4F_VERSION); + if (sym_LZ4F_isError(rc)) + return -ENOMEM; + + c->type = COMPRESSION_LZ4; + + /* Generate the frame header and stash it for the first compressor_start call */ + size_t header_bound = sym_LZ4F_compressBound(0, &lz4_preferences); + c->lz4_header = malloc(header_bound); + if (!c->lz4_header) + return -ENOMEM; + + c->lz4_header_size = sym_LZ4F_compressBegin(c->c_lz4, c->lz4_header, header_bound, &lz4_preferences); + if (sym_LZ4F_isError(c->lz4_header_size)) + return -EINVAL; + + break; + } +#endif + +#if HAVE_ZSTD + case COMPRESSION_ZSTD: + r = dlopen_zstd(); + if (r < 0) + return r; + + c->c_zstd = sym_ZSTD_createCCtx(); + if (!c->c_zstd) + return -ENOMEM; + + c->type = COMPRESSION_ZSTD; + + size_t z = sym_ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_compressionLevel, ZSTD_CLEVEL_DEFAULT); + if (sym_ZSTD_isError(z)) + return -EIO; + + z = sym_ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_checksumFlag, /* enable= */ 1); + if (sym_ZSTD_isError(z)) + log_debug("Failed to enable ZSTD checksum, ignoring: %s", sym_ZSTD_getErrorName(z)); + + break; +#endif + +#if HAVE_ZLIB + case COMPRESSION_GZIP: + r = dlopen_zlib(); + if (r < 0) + return r; + + r = sym_deflateInit2_(&c->gzip, + Z_DEFAULT_COMPRESSION, + /* method= */ Z_DEFLATED, + /* windowBits= */ ZLIB_WBITS_GZIP, + /* memLevel= */ 8, + /* strategy= */ Z_DEFAULT_STRATEGY, + ZLIB_VERSION, (int) sizeof(c->gzip)); + if (r != Z_OK) + return -EIO; + + c->type = COMPRESSION_GZIP; + break; +#endif + +#if HAVE_BZIP2 + case COMPRESSION_BZIP2: + r = dlopen_bzip2(); + if (r < 0) + return r; + + r = sym_BZ2_bzCompressInit(&c->bzip2, /* blockSize100k= */ 9, /* verbosity= */ 0, /* workFactor= */ 0); + if (r != BZ_OK) + return -EIO; + + c->type = COMPRESSION_BZIP2; + break; +#endif + + case COMPRESSION_NONE: + c->type = COMPRESSION_NONE; + break; + + default: + return -EOPNOTSUPP; + } + + *ret = TAKE_PTR(c); + return 0; +} + +#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2 +static int enlarge_buffer(void **buffer, size_t *buffer_size, size_t *buffer_allocated, size_t need) { + assert(buffer); + assert(buffer_size); + assert(buffer_allocated); + + need = MAX3(need, *buffer_size + 1, (size_t) COMPRESS_PIPE_BUFFER_SIZE); + if (*buffer_allocated >= need) + return 0; + + if (!greedy_realloc(buffer, need, 1)) + return -ENOMEM; + + *buffer_allocated = MALLOC_SIZEOF_SAFE(*buffer); + return 1; +} +#endif + +int compressor_start( + Compressor *c, + const void *data, + size_t size, + void **buffer, + size_t *buffer_size, + size_t *buffer_allocated) { + +#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2 + int r; +#endif + + assert(c); + assert(buffer); + assert(buffer_size); + assert(buffer_allocated); + + if (!c->encoding) + return -EINVAL; + + if (size == 0) + return 0; + + assert(data); + + *buffer_size = 0; + + switch (c->type) { + +#if HAVE_XZ + case COMPRESSION_XZ: + + c->xz.next_in = data; + c->xz.avail_in = size; + + while (c->xz.avail_in > 0) { + lzma_ret lzr; + + r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0); + if (r < 0) + return r; + + c->xz.next_out = (uint8_t*) *buffer + *buffer_size; + c->xz.avail_out = *buffer_allocated - *buffer_size; + + lzr = sym_lzma_code(&c->xz, LZMA_RUN); + if (lzr != LZMA_OK) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out; + } + + break; +#endif + +#if HAVE_LZ4 + case COMPRESSION_LZ4: { + /* Prepend any stashed frame header from compressor_new */ + if (c->lz4_header_size > 0) { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated, c->lz4_header_size); + if (r < 0) + return r; + + memcpy(*buffer, c->lz4_header, c->lz4_header_size); + *buffer_size = c->lz4_header_size; + c->lz4_header = mfree(c->lz4_header); + c->lz4_header_size = 0; + } + + size_t bound = sym_LZ4F_compressBound(size, &lz4_preferences); + r = enlarge_buffer(buffer, buffer_size, buffer_allocated, *buffer_size + bound); + if (r < 0) + return r; + + size_t n = sym_LZ4F_compressUpdate(c->c_lz4, + (uint8_t*) *buffer + *buffer_size, + *buffer_allocated - *buffer_size, + data, size, NULL); + if (sym_LZ4F_isError(n)) + return -EIO; + + *buffer_size += n; + break; + } +#endif + +#if HAVE_ZSTD + case COMPRESSION_ZSTD: { + ZSTD_inBuffer input = { + .src = data, + .size = size, + }; + + while (input.pos < input.size) { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0); + if (r < 0) + return r; + + ZSTD_outBuffer output = { + .dst = ((uint8_t *) *buffer + *buffer_size), + .size = *buffer_allocated - *buffer_size, + }; + + size_t res = sym_ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_continue); + if (sym_ZSTD_isError(res)) + return -EIO; + + *buffer_size += output.pos; + } + + break; + } +#endif + +#if HAVE_ZLIB + case COMPRESSION_GZIP: + if (size > UINT_MAX) + return -EFBIG; + + c->gzip.next_in = (void*) data; + c->gzip.avail_in = size; + + while (c->gzip.avail_in > 0) { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0); + if (r < 0) + return r; + + size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX); + c->gzip.next_out = (uint8_t*) *buffer + *buffer_size; + c->gzip.avail_out = avail; + + r = sym_deflate(&c->gzip, Z_NO_FLUSH); + if (r != Z_OK) + return -EIO; + + *buffer_size += avail - c->gzip.avail_out; + } + + break; +#endif + +#if HAVE_BZIP2 + case COMPRESSION_BZIP2: + if (size > UINT_MAX) + return -EFBIG; + + c->bzip2.next_in = (void*) data; + c->bzip2.avail_in = size; + + while (c->bzip2.avail_in > 0) { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0); + if (r < 0) + return r; + + size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX); + c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size); + c->bzip2.avail_out = avail; + + r = sym_BZ2_bzCompress(&c->bzip2, BZ_RUN); + if (r != BZ_RUN_OK) + return -EIO; + + *buffer_size += avail - c->bzip2.avail_out; + } + + break; +#endif + + case COMPRESSION_NONE: + + if (*buffer_allocated < size) { + void *p; + + p = realloc(*buffer, size); + if (!p) + return -ENOMEM; + + *buffer = p; + *buffer_allocated = size; + } + + memcpy(*buffer, data, size); + *buffer_size = size; + break; + + default: + return -EOPNOTSUPP; + } + + return 0; +} + +int compressor_finish(Compressor *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated) { +#if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2 + int r; +#endif + + assert(c); + assert(buffer); + assert(buffer_size); + assert(buffer_allocated); + + if (!c->encoding) + return -EINVAL; + + *buffer_size = 0; + + switch (c->type) { + +#if HAVE_XZ + case COMPRESSION_XZ: { + lzma_ret lzr; + + c->xz.avail_in = 0; + + do { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0); + if (r < 0) + return r; + + c->xz.next_out = (uint8_t*) *buffer + *buffer_size; + c->xz.avail_out = *buffer_allocated - *buffer_size; + + lzr = sym_lzma_code(&c->xz, LZMA_FINISH); + if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END)) + return -EIO; + + *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out; + } while (lzr != LZMA_STREAM_END); + + break; + } +#endif + +#if HAVE_LZ4 + case COMPRESSION_LZ4: { + size_t bound = sym_LZ4F_compressBound(0, &lz4_preferences); + r = enlarge_buffer(buffer, buffer_size, buffer_allocated, bound); + if (r < 0) + return r; + + size_t n = sym_LZ4F_compressEnd(c->c_lz4, *buffer, *buffer_allocated, NULL); + if (sym_LZ4F_isError(n)) + return -EIO; + + *buffer_size = n; + break; + } +#endif + +#if HAVE_ZSTD + case COMPRESSION_ZSTD: { + ZSTD_inBuffer input = {}; + size_t res; + + do { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0); + if (r < 0) + return r; + + ZSTD_outBuffer output = { + .dst = ((uint8_t *) *buffer + *buffer_size), + .size = *buffer_allocated - *buffer_size, + }; + + res = sym_ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_end); + if (sym_ZSTD_isError(res)) + return -EIO; + + *buffer_size += output.pos; + } while (res != 0); + + break; + } +#endif + +#if HAVE_ZLIB + case COMPRESSION_GZIP: + c->gzip.avail_in = 0; + + do { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0); + if (r < 0) + return r; + + size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX); + c->gzip.next_out = (uint8_t*) *buffer + *buffer_size; + c->gzip.avail_out = avail; + + r = sym_deflate(&c->gzip, Z_FINISH); + if (!IN_SET(r, Z_OK, Z_STREAM_END)) + return -EIO; + + *buffer_size += avail - c->gzip.avail_out; + } while (r != Z_STREAM_END); + + break; +#endif + +#if HAVE_BZIP2 + case COMPRESSION_BZIP2: + c->bzip2.avail_in = 0; + + do { + r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0); + if (r < 0) + return r; + + size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX); + c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size); + c->bzip2.avail_out = avail; + + r = sym_BZ2_bzCompress(&c->bzip2, BZ_FINISH); + if (!IN_SET(r, BZ_FINISH_OK, BZ_STREAM_END)) + return -EIO; + + *buffer_size += avail - c->bzip2.avail_out; + } while (r != BZ_STREAM_END); + + break; +#endif + + case COMPRESSION_NONE: + break; + + default: + return -EOPNOTSUPP; + } + + return 0; } diff --git a/src/basic/compress.h b/src/basic/compress.h index 43885a7eed..a5d31b3fc2 100644 --- a/src/basic/compress.h +++ b/src/basic/compress.h @@ -8,103 +8,81 @@ typedef enum Compression { COMPRESSION_XZ, COMPRESSION_LZ4, COMPRESSION_ZSTD, + COMPRESSION_GZIP, + COMPRESSION_BZIP2, _COMPRESSION_MAX, _COMPRESSION_INVALID = -EINVAL, } Compression; DECLARE_STRING_TABLE_LOOKUP(compression, Compression); -DECLARE_STRING_TABLE_LOOKUP(compression_lowercase, Compression); +DECLARE_STRING_TABLE_LOOKUP(compression_uppercase, Compression); +DECLARE_STRING_TABLE_LOOKUP(compression_extension, Compression); + +/* Try the lowercase string table first, fall back to the uppercase one. Useful for parsing user input + * where both forms (e.g. "xz" and "XZ") have historically been accepted. */ +Compression compression_from_string_harder(const char *s); + +/* Derives the compression type from a filename's extension, defaulting to COMPRESSION_NONE if the + * filename does not carry a recognized compression suffix. */ +Compression compression_from_filename(const char *filename); bool compression_supported(Compression c); -int compress_blob_xz(const void *src, uint64_t src_size, - void *dst, size_t dst_alloc_size, size_t *dst_size, int level); -int compress_blob_lz4(const void *src, uint64_t src_size, - void *dst, size_t dst_alloc_size, size_t *dst_size, int level); -int compress_blob_zstd(const void *src, uint64_t src_size, - void *dst, size_t dst_alloc_size, size_t *dst_size, int level); +/* Buffer size used by streaming compression APIs and pipeline stages that feed into them. Sized to + * match the typical Linux pipe buffer so that pipeline stages don't lose throughput due to small + * intermediate buffers. */ +#define COMPRESS_PIPE_BUFFER_SIZE (128U*1024U) -int decompress_blob_xz(const void *src, uint64_t src_size, - void **dst, size_t* dst_size, size_t dst_max); -int decompress_blob_lz4(const void *src, uint64_t src_size, - void **dst, size_t* dst_size, size_t dst_max); -int decompress_blob_zstd(const void *src, uint64_t src_size, - void **dst, size_t* dst_size, size_t dst_max); +/* Compressor / Decompressor — opaque push-based streaming compression context */ + +typedef struct Compressor Compressor; +typedef Compressor Decompressor; + +typedef int (*DecompressorCallback)(const void *data, size_t size, void *userdata); + +Compressor* compressor_free(Compressor *c); +DEFINE_TRIVIAL_CLEANUP_FUNC(Compressor*, compressor_free); + +int compressor_new(Compressor **ret, Compression type); +int compressor_start(Compressor *c, const void *data, size_t size, void **buffer, size_t *buffer_size, size_t *buffer_allocated); +int compressor_finish(Compressor *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated); + +int decompressor_detect(Decompressor **ret, const void *data, size_t size); +int decompressor_force_off(Decompressor **ret); +int decompressor_push(Decompressor *c, const void *data, size_t size, DecompressorCallback callback, void *userdata); + +Compression compressor_type(const Compressor *c); + +/* Blob compression/decompression */ + +int compress_blob(Compression compression, + const void *src, uint64_t src_size, + void *dst, size_t dst_alloc_size, size_t *dst_size, int level); int decompress_blob(Compression compression, const void *src, uint64_t src_size, - void **dst, size_t* dst_size, size_t dst_max); + void **dst, size_t *dst_size, size_t dst_max); + +int decompress_zlib_raw(const void *src, uint64_t src_size, + void *dst, size_t dst_size, int wbits); -int decompress_startswith_xz(const void *src, uint64_t src_size, - void **buffer, - const void *prefix, size_t prefix_len, - uint8_t extra); -int decompress_startswith_lz4(const void *src, uint64_t src_size, - void **buffer, - const void *prefix, size_t prefix_len, - uint8_t extra); -int decompress_startswith_zstd(const void *src, uint64_t src_size, - void **buffer, - const void *prefix, size_t prefix_len, - uint8_t extra); int decompress_startswith(Compression compression, const void *src, uint64_t src_size, void **buffer, const void *prefix, size_t prefix_len, uint8_t extra); -int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size); -int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size); -int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size); +/* Stream compression/decompression (fd-to-fd) */ -int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes); -int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes); -int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes); +int compress_stream(Compression type, int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size); +int decompress_stream(Compression type, int fdf, int fdt, uint64_t max_bytes); +int decompress_stream_by_filename(const char *filename, int fdf, int fdt, uint64_t max_bytes); +int dlopen_xz(void); int dlopen_lz4(void); int dlopen_zstd(void); -int dlopen_lzma(void); - -static inline int compress_blob( - Compression compression, - const void *src, uint64_t src_size, - void *dst, size_t dst_alloc_size, size_t *dst_size, int level) { - - switch (compression) { - case COMPRESSION_ZSTD: - return compress_blob_zstd(src, src_size, dst, dst_alloc_size, dst_size, level); - case COMPRESSION_LZ4: - return compress_blob_lz4(src, src_size, dst, dst_alloc_size, dst_size, level); - case COMPRESSION_XZ: - return compress_blob_xz(src, src_size, dst, dst_alloc_size, dst_size, level); - default: - return -EOPNOTSUPP; - } -} - -static inline int compress_stream(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) { - switch (DEFAULT_COMPRESSION) { - case COMPRESSION_ZSTD: - return compress_stream_zstd(fdf, fdt, max_bytes, ret_uncompressed_size); - case COMPRESSION_LZ4: - return compress_stream_lz4(fdf, fdt, max_bytes, ret_uncompressed_size); - case COMPRESSION_XZ: - return compress_stream_xz(fdf, fdt, max_bytes, ret_uncompressed_size); - default: - return -EOPNOTSUPP; - } -} +int dlopen_zlib(void); +int dlopen_bzip2(void); static inline const char* default_compression_extension(void) { - switch (DEFAULT_COMPRESSION) { - case COMPRESSION_ZSTD: - return ".zst"; - case COMPRESSION_LZ4: - return ".lz4"; - case COMPRESSION_XZ: - return ".xz"; - default: - return ""; - } + return compression_extension_to_string(DEFAULT_COMPRESSION) ?: ""; } - -int decompress_stream(const char *filename, int fdf, int fdt, uint64_t max_bytes); diff --git a/src/basic/meson.build b/src/basic/meson.build index 775dc1fa3d..f847b175b6 100644 --- a/src/basic/meson.build +++ b/src/basic/meson.build @@ -211,12 +211,14 @@ libbasic_static = static_library( fundamental_sources, include_directories : basic_includes, implicit_include_directories : false, - dependencies : [libdl, + dependencies : [libbzip2_cflags, + libdl, libgcrypt_cflags, liblz4_cflags, libm, librt, libxz_cflags, + libz_cflags, libzstd_cflags, threads, userspace], diff --git a/src/boot/test-bcd.c b/src/boot/test-bcd.c index 0924c94fa0..27102c236b 100644 --- a/src/boot/test-bcd.c +++ b/src/boot/test-bcd.c @@ -17,7 +17,7 @@ static void load_bcd(const char *path, void **ret_bcd, size_t *ret_bcd_len) { assert_se(get_testdata_dir(path, &fn) >= 0); assert_se(read_full_file_full(AT_FDCWD, fn, UINT64_MAX, SIZE_MAX, 0, NULL, &compressed, &len) >= 0); - assert_se(decompress_blob_zstd(compressed, len, ret_bcd, ret_bcd_len, SIZE_MAX) >= 0); + assert_se(decompress_blob(COMPRESSION_ZSTD, compressed, len, ret_bcd, ret_bcd_len, SIZE_MAX) >= 0); } static void test_get_bcd_title_one( diff --git a/src/coredump/coredump-submit.c b/src/coredump/coredump-submit.c index 6cfbda0f46..6ce03cdec0 100644 --- a/src/coredump/coredump-submit.c +++ b/src/coredump/coredump-submit.c @@ -373,7 +373,7 @@ static int save_external_coredump( if (fd_compressed < 0) return log_error_errno(fd_compressed, "Failed to create temporary file for coredump %s: %m", fn_compressed); - r = compress_stream(fd, fd_compressed, max_size, &uncompressed_size); + r = compress_stream(DEFAULT_COMPRESSION, fd, fd_compressed, max_size, &uncompressed_size); if (r < 0) return log_error_errno(r, "Failed to compress %s: %m", coredump_tmpfile_name(tmp_compressed)); @@ -386,7 +386,7 @@ static int save_external_coredump( tmp = unlink_and_free(tmp); fd = safe_close(fd); - r = compress_stream(context->input_fd, fd_compressed, max_size, &partial_uncompressed_size); + r = compress_stream(DEFAULT_COMPRESSION, context->input_fd, fd_compressed, max_size, &partial_uncompressed_size); if (r < 0) return log_error_errno(r, "Failed to compress %s: %m", coredump_tmpfile_name(tmp_compressed)); uncompressed_size += partial_uncompressed_size; diff --git a/src/coredump/coredumpctl.c b/src/coredump/coredumpctl.c index fe0af59992..b6ca0f8b7d 100644 --- a/src/coredump/coredumpctl.c +++ b/src/coredump/coredumpctl.c @@ -1103,7 +1103,7 @@ static int save_core(sd_journal *j, FILE *file, char **path, bool *unlink_temp) goto error; } - r = decompress_stream(filename, fdf, fd, -1); + r = decompress_stream_by_filename(filename, fdf, fd, -1); if (r < 0) { log_error_errno(r, "Failed to decompress %s: %m", filename); goto error; diff --git a/src/fundamental/macro-fundamental.h b/src/fundamental/macro-fundamental.h index f595be2cc5..a5300d591a 100644 --- a/src/fundamental/macro-fundamental.h +++ b/src/fundamental/macro-fundamental.h @@ -227,6 +227,12 @@ assert_cc(sizeof(long long) == sizeof(intmax_t)); MAX(_d, a); \ }) +#define MAX5(x, y, z, a, b) \ + ({ \ + const typeof(x) _e = MAX4(x, y, z, a); \ + MAX(_e, b); \ + }) + #undef MIN #define MIN(a, b) __MIN(UNIQ, (a), UNIQ, (b)) #define __MIN(aq, a, bq, b) \ diff --git a/src/import/export-raw.c b/src/import/export-raw.c index 767e10f3ce..31524b1574 100644 --- a/src/import/export-raw.c +++ b/src/import/export-raw.c @@ -11,7 +11,6 @@ #include "fd-util.h" #include "format-util.h" #include "fs-util.h" -#include "import-common.h" #include "log.h" #include "pretty-print.h" #include "ratelimit.h" @@ -32,7 +31,7 @@ typedef struct RawExport { int input_fd; int output_fd; - ImportCompress compress; + Compressor *compress; sd_event_source *output_event_source; @@ -59,7 +58,7 @@ RawExport *raw_export_unref(RawExport *e) { sd_event_source_unref(e->output_event_source); - import_compress_free(&e->compress); + e->compress = compressor_free(e->compress); sd_event_unref(e->event); @@ -143,7 +142,7 @@ static int raw_export_process(RawExport *e) { assert(e); - if (!e->tried_reflink && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) { + if (!e->tried_reflink && compressor_type(e->compress) == COMPRESSION_NONE) { /* If we shall take an uncompressed snapshot we can * reflink source to destination directly. Let's see @@ -158,9 +157,9 @@ static int raw_export_process(RawExport *e) { e->tried_reflink = true; } - if (!e->tried_sendfile && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) { + if (!e->tried_sendfile && compressor_type(e->compress) == COMPRESSION_NONE) { - l = sendfile(e->output_fd, e->input_fd, NULL, IMPORT_BUFFER_SIZE); + l = sendfile(e->output_fd, e->input_fd, NULL, COMPRESS_PIPE_BUFFER_SIZE); if (l < 0) { if (errno == EAGAIN) return 0; @@ -180,7 +179,7 @@ static int raw_export_process(RawExport *e) { } while (e->buffer_size <= 0) { - uint8_t input[IMPORT_BUFFER_SIZE]; + uint8_t input[COMPRESS_PIPE_BUFFER_SIZE]; if (e->eof) { r = 0; @@ -195,10 +194,10 @@ static int raw_export_process(RawExport *e) { if (l == 0) { e->eof = true; - r = import_compress_finish(&e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated); + r = compressor_finish(e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated); } else { e->written_uncompressed += l; - r = import_compress(&e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated); + r = compressor_start(e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated); } if (r < 0) { r = log_error_errno(r, "Failed to encode: %m"); @@ -280,15 +279,15 @@ static int reflink_snapshot(int fd, const char *path) { return new_fd; } -int raw_export_start(RawExport *e, const char *path, int fd, ImportCompressType compress) { +int raw_export_start(RawExport *e, const char *path, int fd, Compression compress) { _cleanup_close_ int sfd = -EBADF, tfd = -EBADF; int r; assert(e); assert(path); assert(fd >= 0); - assert(compress < _IMPORT_COMPRESS_TYPE_MAX); - assert(compress != IMPORT_COMPRESS_UNKNOWN); + assert(compress >= 0); + assert(compress < _COMPRESSION_MAX); if (e->output_fd >= 0) return -EBUSY; @@ -318,7 +317,7 @@ int raw_export_start(RawExport *e, const char *path, int fd, ImportCompressType else e->input_fd = TAKE_FD(sfd); - r = import_compress_init(&e->compress, compress); + r = compressor_new(&e->compress, compress); if (r < 0) return r; diff --git a/src/import/export-raw.h b/src/import/export-raw.h index 664bdfc8e7..f1f17c2c6d 100644 --- a/src/import/export-raw.h +++ b/src/import/export-raw.h @@ -1,8 +1,8 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ #pragma once +#include "compress.h" #include "shared-forward.h" -#include "import-compress.h" typedef struct RawExport RawExport; @@ -13,4 +13,4 @@ RawExport* raw_export_unref(RawExport *e); DEFINE_TRIVIAL_CLEANUP_FUNC(RawExport*, raw_export_unref); -int raw_export_start(RawExport *e, const char *path, int fd, ImportCompressType compress); +int raw_export_start(RawExport *e, const char *path, int fd, Compression compress); diff --git a/src/import/export-tar.c b/src/import/export-tar.c index 22f731de57..93d163adda 100644 --- a/src/import/export-tar.c +++ b/src/import/export-tar.c @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ #include +#include #include "sd-daemon.h" #include "sd-event.h" @@ -38,7 +39,7 @@ typedef struct TarExport { int tree_fd; /* directory fd of the tree to set up */ int userns_fd; - ImportCompress compress; + Compressor *compress; sd_event_source *output_event_source; @@ -74,7 +75,7 @@ TarExport *tar_export_unref(TarExport *e) { free(e->temp_path); } - import_compress_free(&e->compress); + e->compress = compressor_free(e->compress); sd_event_unref(e->event); @@ -188,9 +189,9 @@ static int tar_export_process(TarExport *e) { assert(e); - if (!e->tried_splice && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) { + if (!e->tried_splice && compressor_type(e->compress) == COMPRESSION_NONE) { - l = splice(e->tar_fd, NULL, e->output_fd, NULL, IMPORT_BUFFER_SIZE, 0); + l = splice(e->tar_fd, NULL, e->output_fd, NULL, COMPRESS_PIPE_BUFFER_SIZE, 0); if (l < 0) { if (errno == EAGAIN) return 0; @@ -210,7 +211,7 @@ static int tar_export_process(TarExport *e) { } while (e->buffer_size <= 0) { - uint8_t input[IMPORT_BUFFER_SIZE]; + uint8_t input[COMPRESS_PIPE_BUFFER_SIZE]; if (e->eof) { r = tar_export_finish(e); @@ -225,10 +226,10 @@ static int tar_export_process(TarExport *e) { if (l == 0) { e->eof = true; - r = import_compress_finish(&e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated); + r = compressor_finish(e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated); } else { e->written_uncompressed += l; - r = import_compress(&e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated); + r = compressor_start(e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated); } if (r < 0) { r = log_error_errno(r, "Failed to encode: %m"); @@ -282,7 +283,7 @@ int tar_export_start( TarExport *e, const char *path, int fd, - ImportCompressType compress, + Compression compress, ImportFlags flags) { _cleanup_close_ int sfd = -EBADF; @@ -291,8 +292,8 @@ int tar_export_start( assert(e); assert(path); assert(fd >= 0); - assert(compress < _IMPORT_COMPRESS_TYPE_MAX); - assert(compress != IMPORT_COMPRESS_UNKNOWN); + assert(compress >= 0); + assert(compress < _COMPRESSION_MAX); if (e->output_fd >= 0) return -EBUSY; @@ -336,7 +337,7 @@ int tar_export_start( } } - r = import_compress_init(&e->compress, compress); + r = compressor_new(&e->compress, compress); if (r < 0) return r; diff --git a/src/import/export-tar.h b/src/import/export-tar.h index c5006d4231..be039b6b41 100644 --- a/src/import/export-tar.h +++ b/src/import/export-tar.h @@ -1,8 +1,8 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ #pragma once +#include "compress.h" #include "import-common.h" -#include "import-compress.h" #include "shared-forward.h" typedef struct TarExport TarExport; @@ -14,4 +14,4 @@ TarExport* tar_export_unref(TarExport *e); DEFINE_TRIVIAL_CLEANUP_FUNC(TarExport*, tar_export_unref); -int tar_export_start(TarExport *e, const char *path, int fd, ImportCompressType compress, ImportFlags flags); +int tar_export_start(TarExport *e, const char *path, int fd, Compression compress, ImportFlags flags); diff --git a/src/import/export.c b/src/import/export.c index 6612a1e70a..389d5428bf 100644 --- a/src/import/export.c +++ b/src/import/export.c @@ -2,6 +2,7 @@ #include #include +#include #include "sd-event.h" @@ -22,30 +23,15 @@ #include "verbs.h" static ImportFlags arg_import_flags = 0; -static ImportCompressType arg_compress = IMPORT_COMPRESS_UNKNOWN; +static Compression arg_compress = _COMPRESSION_INVALID; static ImageClass arg_class = IMAGE_MACHINE; static RuntimeScope arg_runtime_scope = _RUNTIME_SCOPE_INVALID; static void determine_compression_from_filename(const char *p) { - - if (arg_compress != IMPORT_COMPRESS_UNKNOWN) + if (arg_compress >= 0) return; - if (!p) { - arg_compress = IMPORT_COMPRESS_UNCOMPRESSED; - return; - } - - if (endswith(p, ".xz")) - arg_compress = IMPORT_COMPRESS_XZ; - else if (endswith(p, ".gz")) - arg_compress = IMPORT_COMPRESS_GZIP; - else if (endswith(p, ".bz2")) - arg_compress = IMPORT_COMPRESS_BZIP2; - else if (endswith(p, ".zst")) - arg_compress = IMPORT_COMPRESS_ZSTD; - else - arg_compress = IMPORT_COMPRESS_UNCOMPRESSED; + arg_compress = p ? compression_from_filename(p) : COMPRESSION_NONE; } static void on_tar_finished(TarExport *export, int error, void *userdata) { @@ -91,7 +77,7 @@ static int verb_export_tar(int argc, char *argv[], uintptr_t _data, void *userda fd = open_fd; - log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, path, import_compress_type_to_string(arg_compress)); + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, path, compression_to_string(arg_compress)); } else { _cleanup_free_ char *pretty = NULL; @@ -101,7 +87,7 @@ static int verb_export_tar(int argc, char *argv[], uintptr_t _data, void *userda fd = STDOUT_FILENO; (void) fd_get_path(fd, &pretty); - log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), import_compress_type_to_string(arg_compress)); + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), compression_to_string(arg_compress)); } r = import_allocate_event_with_signals(&event); @@ -172,14 +158,14 @@ static int verb_export_raw(int argc, char *argv[], uintptr_t _data, void *userda fd = open_fd; - log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, path, import_compress_type_to_string(arg_compress)); + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, path, compression_to_string(arg_compress)); } else { _cleanup_free_ char *pretty = NULL; fd = STDOUT_FILENO; (void) fd_get_path(fd, &pretty); - log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), import_compress_type_to_string(arg_compress)); + log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), compression_to_string(arg_compress)); } r = import_allocate_event_with_signals(&event); @@ -265,8 +251,8 @@ static int parse_argv(int argc, char *argv[]) { return version(); case ARG_FORMAT: - arg_compress = import_compress_type_from_string(optarg); - if (arg_compress < 0 || arg_compress == IMPORT_COMPRESS_UNKNOWN) + arg_compress = compression_from_string_harder(optarg); + if (arg_compress < 0) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Unknown format: %s", optarg); break; diff --git a/src/import/import-common.c b/src/import/import-common.c index 0a5144f94e..948cd82988 100644 --- a/src/import/import-common.c +++ b/src/import/import-common.c @@ -7,6 +7,7 @@ #include "sd-event.h" #include "capability-util.h" +#include "compress.h" #include "dirent-util.h" #include "dissect-image.h" #include "fd-util.h" @@ -43,7 +44,7 @@ int import_fork_tar_x(int tree_fd, int userns_fd, PidRef *ret_pid) { if (pipe2(pipefd, O_CLOEXEC) < 0) return log_error_errno(errno, "Failed to create pipe for tar: %m"); - (void) fcntl(pipefd[0], F_SETPIPE_SZ, IMPORT_BUFFER_SIZE); + (void) fcntl(pipefd[0], F_SETPIPE_SZ, COMPRESS_PIPE_BUFFER_SIZE); r = pidref_safe_fork_full( "tar-x", @@ -110,7 +111,7 @@ int import_fork_tar_c(int tree_fd, int userns_fd, PidRef *ret_pid) { if (pipe2(pipefd, O_CLOEXEC) < 0) return log_error_errno(errno, "Failed to create pipe for tar: %m"); - (void) fcntl(pipefd[0], F_SETPIPE_SZ, IMPORT_BUFFER_SIZE); + (void) fcntl(pipefd[0], F_SETPIPE_SZ, COMPRESS_PIPE_BUFFER_SIZE); r = pidref_safe_fork_full( "tar-c", diff --git a/src/import/import-common.h b/src/import/import-common.h index 6b10f8c29d..69bdb33528 100644 --- a/src/import/import-common.h +++ b/src/import/import-common.h @@ -49,5 +49,3 @@ int import_allocate_event_with_signals(sd_event **ret); int import_make_foreign_userns(int *userns_fd); int import_remove_tree(const char *path, int *userns_fd, ImportFlags flags); - -#define IMPORT_BUFFER_SIZE (128U*1024U) diff --git a/src/import/import-compress.c b/src/import/import-compress.c deleted file mode 100644 index aca4041f2a..0000000000 --- a/src/import/import-compress.c +++ /dev/null @@ -1,611 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1-or-later */ - -#include -#include - -#include "import-common.h" -#include "import-compress.h" -#include "log.h" -#include "string-table.h" - -void import_compress_free(ImportCompress *c) { - assert(c); - - if (c->type == IMPORT_COMPRESS_XZ) - lzma_end(&c->xz); - else if (c->type == IMPORT_COMPRESS_GZIP) { - if (c->encoding) - deflateEnd(&c->gzip); - else - inflateEnd(&c->gzip); -#if HAVE_BZIP2 - } else if (c->type == IMPORT_COMPRESS_BZIP2) { - if (c->encoding) - BZ2_bzCompressEnd(&c->bzip2); - else - BZ2_bzDecompressEnd(&c->bzip2); -#endif -#if HAVE_ZSTD - } else if (c->type == IMPORT_COMPRESS_ZSTD) { - if (c->encoding) { - ZSTD_freeCCtx(c->c_zstd); - c->c_zstd = NULL; - } else { - ZSTD_freeDCtx(c->d_zstd); - c->d_zstd = NULL; - } -#endif - } - - c->type = IMPORT_COMPRESS_UNKNOWN; -} - -int import_uncompress_detect(ImportCompress *c, const void *data, size_t size) { - static const uint8_t xz_signature[] = { - 0xfd, '7', 'z', 'X', 'Z', 0x00 - }; - static const uint8_t gzip_signature[] = { - 0x1f, 0x8b - }; - static const uint8_t bzip2_signature[] = { - 'B', 'Z', 'h' - }; - static const uint8_t zstd_signature[] = { - 0x28, 0xb5, 0x2f, 0xfd - }; - - int r; - - assert(c); - - if (c->type != IMPORT_COMPRESS_UNKNOWN) - return 1; - - if (size < MAX4(sizeof(xz_signature), - sizeof(gzip_signature), - sizeof(zstd_signature), - sizeof(bzip2_signature))) - return 0; - - assert(data); - - if (memcmp(data, xz_signature, sizeof(xz_signature)) == 0) { - lzma_ret xzr; - - xzr = lzma_stream_decoder(&c->xz, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED); - if (xzr != LZMA_OK) - return -EIO; - - c->type = IMPORT_COMPRESS_XZ; - - } else if (memcmp(data, gzip_signature, sizeof(gzip_signature)) == 0) { - r = inflateInit2(&c->gzip, 15+16); - if (r != Z_OK) - return -EIO; - - c->type = IMPORT_COMPRESS_GZIP; - -#if HAVE_BZIP2 - } else if (memcmp(data, bzip2_signature, sizeof(bzip2_signature)) == 0) { - r = BZ2_bzDecompressInit(&c->bzip2, 0, 0); - if (r != BZ_OK) - return -EIO; - - c->type = IMPORT_COMPRESS_BZIP2; -#endif -#if HAVE_ZSTD - } else if (memcmp(data, zstd_signature, sizeof(zstd_signature)) == 0) { - c->d_zstd = ZSTD_createDCtx(); - if (!c->d_zstd) - return -ENOMEM; - - c->type = IMPORT_COMPRESS_ZSTD; -#endif - } else - c->type = IMPORT_COMPRESS_UNCOMPRESSED; - - c->encoding = false; - - log_debug("Detected compression type: %s", import_compress_type_to_string(c->type)); - return 1; -} - -void import_uncompress_force_off(ImportCompress *c) { - assert(c); - - c->type = IMPORT_COMPRESS_UNCOMPRESSED; - c->encoding = false; -} - -int import_uncompress(ImportCompress *c, const void *data, size_t size, ImportCompressCallback callback, void *userdata) { - int r; - - assert(c); - assert(callback); - - r = import_uncompress_detect(c, data, size); - if (r <= 0) - return r; - - if (c->encoding) - return -EINVAL; - - if (size <= 0) - return 1; - - assert(data); - - switch (c->type) { - - case IMPORT_COMPRESS_UNCOMPRESSED: - r = callback(data, size, userdata); - if (r < 0) - return r; - - break; - - case IMPORT_COMPRESS_XZ: - c->xz.next_in = data; - c->xz.avail_in = size; - - while (c->xz.avail_in > 0) { - uint8_t buffer[IMPORT_BUFFER_SIZE]; - lzma_ret lzr; - - c->xz.next_out = buffer; - c->xz.avail_out = sizeof(buffer); - - lzr = lzma_code(&c->xz, LZMA_RUN); - if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END)) - return -EIO; - - if (c->xz.avail_out < sizeof(buffer)) { - r = callback(buffer, sizeof(buffer) - c->xz.avail_out, userdata); - if (r < 0) - return r; - } - } - - break; - - case IMPORT_COMPRESS_GZIP: - c->gzip.next_in = (void*) data; - c->gzip.avail_in = size; - - while (c->gzip.avail_in > 0) { - uint8_t buffer[IMPORT_BUFFER_SIZE]; - - c->gzip.next_out = buffer; - c->gzip.avail_out = sizeof(buffer); - - r = inflate(&c->gzip, Z_NO_FLUSH); - if (!IN_SET(r, Z_OK, Z_STREAM_END)) - return -EIO; - - if (c->gzip.avail_out < sizeof(buffer)) { - r = callback(buffer, sizeof(buffer) - c->gzip.avail_out, userdata); - if (r < 0) - return r; - } - } - - break; - -#if HAVE_BZIP2 - case IMPORT_COMPRESS_BZIP2: - c->bzip2.next_in = (void*) data; - c->bzip2.avail_in = size; - - while (c->bzip2.avail_in > 0) { - uint8_t buffer[IMPORT_BUFFER_SIZE]; - - c->bzip2.next_out = (char*) buffer; - c->bzip2.avail_out = sizeof(buffer); - - r = BZ2_bzDecompress(&c->bzip2); - if (!IN_SET(r, BZ_OK, BZ_STREAM_END)) - return -EIO; - - if (c->bzip2.avail_out < sizeof(buffer)) { - r = callback(buffer, sizeof(buffer) - c->bzip2.avail_out, userdata); - if (r < 0) - return r; - } - } - - break; -#endif -#if HAVE_ZSTD - case IMPORT_COMPRESS_ZSTD: { - ZSTD_inBuffer input = { - .src = (void*) data, - .size = size, - }; - - while (input.pos < input.size) { - uint8_t buffer[IMPORT_BUFFER_SIZE]; - ZSTD_outBuffer output = { - .dst = buffer, - .size = sizeof(buffer), - }; - size_t res; - - res = ZSTD_decompressStream(c->d_zstd, &output, &input); - if (ZSTD_isError(res)) - return -EIO; - - if (output.pos > 0) { - r = callback(output.dst, output.pos, userdata); - if (r < 0) - return r; - } - } - - break; - } -#endif - - default: - assert_not_reached(); - } - - return 1; -} - -int import_compress_init(ImportCompress *c, ImportCompressType t) { - int r; - - assert(c); - - switch (t) { - - case IMPORT_COMPRESS_XZ: { - lzma_ret xzr; - - xzr = lzma_easy_encoder(&c->xz, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64); - if (xzr != LZMA_OK) - return -EIO; - - c->type = IMPORT_COMPRESS_XZ; - break; - } - - case IMPORT_COMPRESS_GZIP: - r = deflateInit2(&c->gzip, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY); - if (r != Z_OK) - return -EIO; - - c->type = IMPORT_COMPRESS_GZIP; - break; - -#if HAVE_BZIP2 - case IMPORT_COMPRESS_BZIP2: - r = BZ2_bzCompressInit(&c->bzip2, 9, 0, 0); - if (r != BZ_OK) - return -EIO; - - c->type = IMPORT_COMPRESS_BZIP2; - break; -#endif - -#if HAVE_ZSTD - case IMPORT_COMPRESS_ZSTD: - c->c_zstd = ZSTD_createCCtx(); - if (!c->c_zstd) - return -ENOMEM; - - r = ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_compressionLevel, ZSTD_CLEVEL_DEFAULT); - if (ZSTD_isError(r)) - return -EIO; - - c->type = IMPORT_COMPRESS_ZSTD; - break; -#endif - - case IMPORT_COMPRESS_UNCOMPRESSED: - c->type = IMPORT_COMPRESS_UNCOMPRESSED; - break; - - default: - return -EOPNOTSUPP; - } - - c->encoding = true; - return 0; -} - -static int enlarge_buffer(void **buffer, size_t *buffer_size, size_t *buffer_allocated) { - size_t l; - void *p; - - assert(buffer); - assert(buffer_size); - assert(buffer_allocated); - - if (*buffer_allocated > *buffer_size) - return 0; - - l = MAX(IMPORT_BUFFER_SIZE, (*buffer_size * 2)); - p = realloc(*buffer, l); - if (!p) - return -ENOMEM; - - *buffer = p; - *buffer_allocated = l; - - return 1; -} - -int import_compress(ImportCompress *c, const void *data, size_t size, void **buffer, size_t *buffer_size, size_t *buffer_allocated) { - int r; - - assert(c); - assert(buffer); - assert(buffer_size); - assert(buffer_allocated); - - if (!c->encoding) - return -EINVAL; - - if (size <= 0) - return 0; - - assert(data); - - *buffer_size = 0; - - switch (c->type) { - - case IMPORT_COMPRESS_XZ: - - c->xz.next_in = data; - c->xz.avail_in = size; - - while (c->xz.avail_in > 0) { - lzma_ret lzr; - - r = enlarge_buffer(buffer, buffer_size, buffer_allocated); - if (r < 0) - return r; - - c->xz.next_out = (uint8_t*) *buffer + *buffer_size; - c->xz.avail_out = *buffer_allocated - *buffer_size; - - lzr = lzma_code(&c->xz, LZMA_RUN); - if (lzr != LZMA_OK) - return -EIO; - - *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out; - } - - break; - - case IMPORT_COMPRESS_GZIP: - - c->gzip.next_in = (void*) data; - c->gzip.avail_in = size; - - while (c->gzip.avail_in > 0) { - r = enlarge_buffer(buffer, buffer_size, buffer_allocated); - if (r < 0) - return r; - - c->gzip.next_out = (uint8_t*) *buffer + *buffer_size; - c->gzip.avail_out = *buffer_allocated - *buffer_size; - - r = deflate(&c->gzip, Z_NO_FLUSH); - if (r != Z_OK) - return -EIO; - - *buffer_size += (*buffer_allocated - *buffer_size) - c->gzip.avail_out; - } - - break; - -#if HAVE_BZIP2 - case IMPORT_COMPRESS_BZIP2: - - c->bzip2.next_in = (void*) data; - c->bzip2.avail_in = size; - - while (c->bzip2.avail_in > 0) { - r = enlarge_buffer(buffer, buffer_size, buffer_allocated); - if (r < 0) - return r; - - c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size); - c->bzip2.avail_out = *buffer_allocated - *buffer_size; - - r = BZ2_bzCompress(&c->bzip2, BZ_RUN); - if (r != BZ_RUN_OK) - return -EIO; - - *buffer_size += (*buffer_allocated - *buffer_size) - c->bzip2.avail_out; - } - - break; -#endif - -#if HAVE_ZSTD - case IMPORT_COMPRESS_ZSTD: { - ZSTD_inBuffer input = { - .src = data, - .size = size, - }; - - while (input.pos < input.size) { - r = enlarge_buffer(buffer, buffer_size, buffer_allocated); - if (r < 0) - return r; - - ZSTD_outBuffer output = { - .dst = ((uint8_t *) *buffer + *buffer_size), - .size = *buffer_allocated - *buffer_size, - }; - size_t res; - - res = ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_continue); - if (ZSTD_isError(res)) - return -EIO; - - *buffer_size += output.pos; - } - - break; - } -#endif - - case IMPORT_COMPRESS_UNCOMPRESSED: - - if (*buffer_allocated < size) { - void *p; - - p = realloc(*buffer, size); - if (!p) - return -ENOMEM; - - *buffer = p; - *buffer_allocated = size; - } - - memcpy(*buffer, data, size); - *buffer_size = size; - break; - - default: - return -EOPNOTSUPP; - } - - return 0; -} - -int import_compress_finish(ImportCompress *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated) { - int r; - - assert(c); - assert(buffer); - assert(buffer_size); - assert(buffer_allocated); - - if (!c->encoding) - return -EINVAL; - - *buffer_size = 0; - - switch (c->type) { - - case IMPORT_COMPRESS_XZ: { - lzma_ret lzr; - - c->xz.avail_in = 0; - - do { - r = enlarge_buffer(buffer, buffer_size, buffer_allocated); - if (r < 0) - return r; - - c->xz.next_out = (uint8_t*) *buffer + *buffer_size; - c->xz.avail_out = *buffer_allocated - *buffer_size; - - lzr = lzma_code(&c->xz, LZMA_FINISH); - if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END)) - return -EIO; - - *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out; - } while (lzr != LZMA_STREAM_END); - - break; - } - - case IMPORT_COMPRESS_GZIP: - c->gzip.avail_in = 0; - - do { - r = enlarge_buffer(buffer, buffer_size, buffer_allocated); - if (r < 0) - return r; - - c->gzip.next_out = (uint8_t*) *buffer + *buffer_size; - c->gzip.avail_out = *buffer_allocated - *buffer_size; - - r = deflate(&c->gzip, Z_FINISH); - if (!IN_SET(r, Z_OK, Z_STREAM_END)) - return -EIO; - - *buffer_size += (*buffer_allocated - *buffer_size) - c->gzip.avail_out; - } while (r != Z_STREAM_END); - - break; - -#if HAVE_BZIP2 - case IMPORT_COMPRESS_BZIP2: - c->bzip2.avail_in = 0; - - do { - r = enlarge_buffer(buffer, buffer_size, buffer_allocated); - if (r < 0) - return r; - - c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size); - c->bzip2.avail_out = *buffer_allocated - *buffer_size; - - r = BZ2_bzCompress(&c->bzip2, BZ_FINISH); - if (!IN_SET(r, BZ_FINISH_OK, BZ_STREAM_END)) - return -EIO; - - *buffer_size += (*buffer_allocated - *buffer_size) - c->bzip2.avail_out; - } while (r != BZ_STREAM_END); - - break; -#endif - -#if HAVE_ZSTD - case IMPORT_COMPRESS_ZSTD: { - ZSTD_inBuffer input = {}; - size_t res; - - do { - r = enlarge_buffer(buffer, buffer_size, buffer_allocated); - if (r < 0) - return r; - - ZSTD_outBuffer output = { - .dst = ((uint8_t *) *buffer + *buffer_size), - .size = *buffer_allocated - *buffer_size, - }; - - res = ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_end); - if (ZSTD_isError(res)) - return -EIO; - - *buffer_size += output.pos; - } while (res != 0); - - break; - } -#endif - - case IMPORT_COMPRESS_UNCOMPRESSED: - break; - - default: - return -EOPNOTSUPP; - } - - return 0; -} - -static const char* const import_compress_type_table[_IMPORT_COMPRESS_TYPE_MAX] = { - [IMPORT_COMPRESS_UNKNOWN] = "unknown", - [IMPORT_COMPRESS_UNCOMPRESSED] = "uncompressed", - [IMPORT_COMPRESS_XZ] = "xz", - [IMPORT_COMPRESS_GZIP] = "gzip", -#if HAVE_BZIP2 - [IMPORT_COMPRESS_BZIP2] = "bzip2", -#endif -#if HAVE_ZSTD - [IMPORT_COMPRESS_ZSTD] = "zstd", -#endif -}; - -DEFINE_STRING_TABLE_LOOKUP(import_compress_type, ImportCompressType); diff --git a/src/import/import-compress.h b/src/import/import-compress.h deleted file mode 100644 index 647e623266..0000000000 --- a/src/import/import-compress.h +++ /dev/null @@ -1,54 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1-or-later */ -#pragma once - -#if HAVE_BZIP2 -#include -#endif -#include -#include -#if HAVE_ZSTD -#include -#endif - -#include "shared-forward.h" - -typedef enum ImportCompressType { - IMPORT_COMPRESS_UNKNOWN, - IMPORT_COMPRESS_UNCOMPRESSED, - IMPORT_COMPRESS_XZ, - IMPORT_COMPRESS_GZIP, - IMPORT_COMPRESS_BZIP2, - IMPORT_COMPRESS_ZSTD, - _IMPORT_COMPRESS_TYPE_MAX, - _IMPORT_COMPRESS_TYPE_INVALID = -EINVAL, -} ImportCompressType; - -typedef struct ImportCompress { - ImportCompressType type; - bool encoding; - union { - lzma_stream xz; - z_stream gzip; -#if HAVE_BZIP2 - bz_stream bzip2; -#endif -#if HAVE_ZSTD - ZSTD_CCtx *c_zstd; - ZSTD_DCtx *d_zstd; -#endif - }; -} ImportCompress; - -typedef int (*ImportCompressCallback)(const void *data, size_t size, void *userdata); - -void import_compress_free(ImportCompress *c); - -int import_uncompress_detect(ImportCompress *c, const void *data, size_t size); -void import_uncompress_force_off(ImportCompress *c); -int import_uncompress(ImportCompress *c, const void *data, size_t size, ImportCompressCallback callback, void *userdata); - -int import_compress_init(ImportCompress *c, ImportCompressType t); -int import_compress(ImportCompress *c, const void *data, size_t size, void **buffer, size_t *buffer_size, size_t *buffer_allocated); -int import_compress_finish(ImportCompress *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated); - -DECLARE_STRING_TABLE_LOOKUP(import_compress_type, ImportCompressType); diff --git a/src/import/import-raw.c b/src/import/import-raw.c index 1d7302cd88..05d4bc9c9f 100644 --- a/src/import/import-raw.c +++ b/src/import/import-raw.c @@ -1,17 +1,18 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ #include +#include #include "sd-daemon.h" #include "sd-event.h" #include "alloc-util.h" +#include "compress.h" #include "copy.h" #include "fd-util.h" #include "format-util.h" #include "fs-util.h" #include "import-common.h" -#include "import-compress.h" #include "import-raw.h" #include "import-util.h" #include "install-file.h" @@ -43,11 +44,11 @@ typedef struct RawImport { int input_fd; int output_fd; - ImportCompress compress; + Compressor *compress; sd_event_source *input_event_source; - uint8_t buffer[IMPORT_BUFFER_SIZE]; + uint8_t buffer[COMPRESS_PIPE_BUFFER_SIZE]; size_t buffer_size; uint64_t written_compressed; @@ -71,7 +72,7 @@ RawImport* raw_import_unref(RawImport *i) { unlink_and_free(i->temp_path); - import_compress_free(&i->compress); + i->compress = compressor_free(i->compress); sd_event_unref(i->event); @@ -328,7 +329,7 @@ static int raw_import_try_reflink(RawImport *i) { assert(i->input_fd >= 0); assert(i->output_fd >= 0); - if (i->compress.type != IMPORT_COMPRESS_UNCOMPRESSED) + if (compressor_type(i->compress) != COMPRESSION_NONE) return 0; if (i->offset != UINT64_MAX || i->size_max != UINT64_MAX) @@ -425,13 +426,13 @@ static int raw_import_process(RawImport *i) { i->buffer_size += l; - if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) { + if (!i->compress) { if (l == 0) { /* EOF */ log_debug("File too short to be compressed, as no compression signature fits in, thus assuming uncompressed."); - import_uncompress_force_off(&i->compress); + decompressor_force_off(&i->compress); } else { - r = import_uncompress_detect(&i->compress, i->buffer, i->buffer_size); + r = decompressor_detect(&i->compress, i->buffer, i->buffer_size); if (r < 0) { log_error_errno(r, "Failed to detect file compression: %m"); goto finish; @@ -451,7 +452,7 @@ static int raw_import_process(RawImport *i) { goto complete; } - r = import_uncompress(&i->compress, i->buffer, i->buffer_size, raw_import_write, i); + r = decompressor_push(i->compress, i->buffer, i->buffer_size, raw_import_write, i); if (r < 0) { log_error_errno(r, "Failed to decode and write: %m"); goto finish; diff --git a/src/import/import-tar.c b/src/import/import-tar.c index 5e74de896e..4bd5978800 100644 --- a/src/import/import-tar.c +++ b/src/import/import-tar.c @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ #include +#include #include "sd-daemon.h" #include "sd-event.h" @@ -8,12 +9,12 @@ #include "alloc-util.h" #include "btrfs-util.h" +#include "compress.h" #include "dissect-image.h" #include "errno-util.h" #include "fd-util.h" #include "format-util.h" #include "import-common.h" -#include "import-compress.h" #include "import-tar.h" #include "import-util.h" #include "install-file.h" @@ -50,11 +51,11 @@ typedef struct TarImport { int tree_fd; int userns_fd; - ImportCompress compress; + Compressor *compress; sd_event_source *input_event_source; - uint8_t buffer[IMPORT_BUFFER_SIZE]; + uint8_t buffer[COMPRESS_PIPE_BUFFER_SIZE]; size_t buffer_size; uint64_t written_compressed; @@ -81,7 +82,7 @@ TarImport* tar_import_unref(TarImport *i) { free(i->temp_path); } - import_compress_free(&i->compress); + i->compress = compressor_free(i->compress); sd_event_unref(i->event); @@ -344,13 +345,13 @@ static int tar_import_process(TarImport *i) { i->buffer_size += l; - if (i->compress.type == IMPORT_COMPRESS_UNKNOWN) { + if (!i->compress) { if (l == 0) { /* EOF */ log_debug("File too short to be compressed, as no compression signature fits in, thus assuming uncompressed."); - import_uncompress_force_off(&i->compress); + decompressor_force_off(&i->compress); } else { - r = import_uncompress_detect(&i->compress, i->buffer, i->buffer_size); + r = decompressor_detect(&i->compress, i->buffer, i->buffer_size); if (r < 0) { log_error_errno(r, "Failed to detect file compression: %m"); goto finish; @@ -364,7 +365,7 @@ static int tar_import_process(TarImport *i) { goto finish; } - r = import_uncompress(&i->compress, i->buffer, i->buffer_size, tar_import_write, i); + r = decompressor_push(i->compress, i->buffer, i->buffer_size, tar_import_write, i); if (r < 0) { log_error_errno(r, "Failed to decode and write: %m"); goto finish; diff --git a/src/import/meson.build b/src/import/meson.build index 30751058f1..a98604d491 100644 --- a/src/import/meson.build +++ b/src/import/meson.build @@ -7,11 +7,7 @@ if conf.get('ENABLE_IMPORTD') != 1 endif common_deps = [ - libbzip2, libcurl, - libxz, - libz, - libzstd, ] executables += [ @@ -25,7 +21,6 @@ executables += [ 'extract' : files( 'oci-util.c', 'import-common.c', - 'import-compress.c', 'qcow2-util.c', ), 'dependencies' : [common_deps, threads], diff --git a/src/import/pull-common.c b/src/import/pull-common.c index c0e0e9907e..b234331945 100644 --- a/src/import/pull-common.c +++ b/src/import/pull-common.c @@ -1,5 +1,7 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ +#include + #include "sd-id128.h" #include "alloc-util.h" diff --git a/src/import/pull-job.c b/src/import/pull-job.c index dcdb1fe8fa..385043dda8 100644 --- a/src/import/pull-job.c +++ b/src/import/pull-job.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "alloc-util.h" #include "curl-util.h" @@ -53,7 +54,7 @@ PullJob* pull_job_unref(PullJob *j) { curl_glue_remove_and_free(j->glue, j->curl); curl_slist_free_all(j->request_header); - import_compress_free(&j->compress); + j->compress = compressor_free(j->compress); if (j->checksum_ctx) EVP_MD_CTX_free(j->checksum_ctx); @@ -134,7 +135,7 @@ int pull_job_restart(PullJob *j, const char *new_url) { curl_glue_remove_and_free(j->glue, j->curl); j->curl = NULL; - import_compress_free(&j->compress); + j->compress = compressor_free(j->compress); if (j->checksum_ctx) { EVP_MD_CTX_free(j->checksum_ctx); @@ -453,7 +454,7 @@ static int pull_job_write_compressed(PullJob *j, const struct iovec *data) { "Could not hash chunk."); } - r = import_uncompress(&j->compress, data->iov_base, data->iov_len, pull_job_write_uncompressed, j); + r = decompressor_push(j->compress, data->iov_base, data->iov_len, pull_job_write_uncompressed, j); if (r < 0) return r; @@ -502,13 +503,13 @@ static int pull_job_detect_compression(PullJob *j) { assert(j); - r = import_uncompress_detect(&j->compress, j->payload.iov_base, j->payload.iov_len); + r = decompressor_detect(&j->compress, j->payload.iov_base, j->payload.iov_len); if (r < 0) return log_error_errno(r, "Failed to initialize compressor: %m"); if (r == 0) return 0; - log_debug("Stream is compressed: %s", import_compress_type_to_string(j->compress.type)); + log_debug("Stream is compressed: %s", compression_to_string(compressor_type(j->compress))); r = pull_job_open_disk(j); if (r < 0) diff --git a/src/import/pull-job.h b/src/import/pull-job.h index ea58b62f3b..1daa006c1c 100644 --- a/src/import/pull-job.h +++ b/src/import/pull-job.h @@ -4,9 +4,9 @@ #include #include -#include "shared-forward.h" -#include "import-compress.h" +#include "compress.h" #include "openssl-util.h" +#include "shared-forward.h" typedef struct CurlGlue CurlGlue; typedef struct PullJob PullJob; @@ -73,7 +73,7 @@ typedef struct PullJob { usec_t mtime; char *content_type; - ImportCompress compress; + Compressor *compress; unsigned progress_percent; usec_t start_usec; diff --git a/src/import/pull-oci.c b/src/import/pull-oci.c index 0f16c65630..4ae933928f 100644 --- a/src/import/pull-oci.c +++ b/src/import/pull-oci.c @@ -1,5 +1,7 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ +#include + #include "sd-event.h" #include "sd-json.h" #include "sd-varlink.h" diff --git a/src/import/pull-raw.c b/src/import/pull-raw.c index 31a08eb24a..6fde8c5f8b 100644 --- a/src/import/pull-raw.c +++ b/src/import/pull-raw.c @@ -1,5 +1,7 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ +#include + #include "sd-daemon.h" #include "sd-event.h" diff --git a/src/import/pull-tar.c b/src/import/pull-tar.c index f4a8bfca62..a235c2fba3 100644 --- a/src/import/pull-tar.c +++ b/src/import/pull-tar.c @@ -1,5 +1,7 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ +#include + #include "sd-daemon.h" #include "sd-event.h" #include "sd-varlink.h" diff --git a/src/import/qcow2-util.c b/src/import/qcow2-util.c index fe0c8a2620..2b219b04a1 100644 --- a/src/import/qcow2-util.c +++ b/src/import/qcow2-util.c @@ -1,8 +1,9 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ -#include +#include #include "alloc-util.h" +#include "compress.h" #include "copy.h" #include "qcow2-util.h" #include "sparse-endian.h" @@ -97,8 +98,6 @@ static int decompress_cluster( void *buffer2) { _cleanup_free_ void *large_buffer = NULL; - z_stream s = {}; - uint64_t sz; ssize_t l; int r; @@ -119,20 +118,9 @@ static int decompress_cluster( if ((uint64_t) l != compressed_size) return -EIO; - s.next_in = buffer1; - s.avail_in = compressed_size; - s.next_out = buffer2; - s.avail_out = cluster_size; - - r = inflateInit2(&s, -12); - if (r != Z_OK) - return -EIO; - - r = inflate(&s, Z_FINISH); - sz = (uint8_t*) s.next_out - (uint8_t*) buffer2; - inflateEnd(&s); - if (r != Z_STREAM_END || sz != cluster_size) - return -EIO; + r = decompress_zlib_raw(buffer1, compressed_size, buffer2, cluster_size, /* wbits= */ -12); + if (r < 0) + return r; l = pwrite(dfd, buffer2, cluster_size, doffset); if (l < 0) diff --git a/src/journal-remote/journal-compression-util.c b/src/journal-remote/journal-compression-util.c index 00d3935895..8e415878b9 100644 --- a/src/journal-remote/journal-compression-util.c +++ b/src/journal-remote/journal-compression-util.c @@ -130,7 +130,7 @@ int config_parse_compression( } } - Compression c = compression_lowercase_from_string(word); + Compression c = compression_from_string_harder(word); if (c <= 0 || !compression_supported(c)) { log_syntax(unit, LOG_WARNING, filename, line, c, "Compression algorithm '%s' is not supported on the system, ignoring.", word); diff --git a/src/journal-remote/journal-remote-main.c b/src/journal-remote/journal-remote-main.c index 0ff44ede6f..fbb53cc42f 100644 --- a/src/journal-remote/journal-remote-main.c +++ b/src/journal-remote/journal-remote-main.c @@ -209,7 +209,7 @@ static int build_accept_encoding(char **ret) { const CompressionConfig *cc; ORDERED_HASHMAP_FOREACH(cc, arg_compression) { - const char *c = compression_lowercase_to_string(cc->algorithm); + const char *c = compression_to_string(cc->algorithm); if (strextendf_with_separator(&buf, ",", "%s;q=%.1f", c, q) < 0) return -ENOMEM; q -= step; @@ -361,7 +361,7 @@ static mhd_result request_handler( RemoteSource *source = *connection_cls; header = MHD_lookup_connection_value(connection, MHD_HEADER_KIND, "Content-Encoding"); if (header) { - Compression c = compression_lowercase_from_string(header); + Compression c = compression_from_string_harder(header); if (c <= 0 || !compression_supported(c)) return mhd_respondf(connection, 0, MHD_HTTP_UNSUPPORTED_MEDIA_TYPE, "Unsupported Content-Encoding type: %s", header); diff --git a/src/journal-remote/journal-upload-journal.c b/src/journal-remote/journal-upload-journal.c index 054451aafc..66cc4114f4 100644 --- a/src/journal-remote/journal-upload-journal.c +++ b/src/journal-remote/journal-upload-journal.c @@ -354,7 +354,7 @@ static size_t journal_input_callback(void *buf, size_t size, size_t nmemb, void r = compress_blob(u->compression->algorithm, compression_buffer, filled, buf, size * nmemb, &compressed_size, u->compression->level); if (r < 0) { log_error_errno(r, "Failed to compress %zu bytes by %s with level %i: %m", - filled, compression_lowercase_to_string(u->compression->algorithm), u->compression->level); + filled, compression_to_string(u->compression->algorithm), u->compression->level); return CURL_READFUNC_ABORT; } diff --git a/src/journal-remote/journal-upload.c b/src/journal-remote/journal-upload.c index c6123146a5..c4eab80a1f 100644 --- a/src/journal-remote/journal-upload.c +++ b/src/journal-remote/journal-upload.c @@ -218,7 +218,7 @@ int start_upload(Uploader *u, h = l; if (u->compression) { - _cleanup_free_ char *header = strjoin("Content-Encoding: ", compression_lowercase_to_string(u->compression->algorithm)); + _cleanup_free_ char *header = strjoin("Content-Encoding: ", compression_to_string(u->compression->algorithm)); if (!header) return log_oom(); @@ -369,7 +369,7 @@ static size_t fd_input_callback(void *buf, size_t size, size_t nmemb, void *user r = compress_blob(u->compression->algorithm, compression_buffer, n, buf, size * nmemb, &compressed_size, u->compression->level); if (r < 0) { log_error_errno(r, "Failed to compress %zd bytes by %s with level %i: %m", - n, compression_lowercase_to_string(u->compression->algorithm), u->compression->level); + n, compression_to_string(u->compression->algorithm), u->compression->level); return CURL_READFUNC_ABORT; } assert(compressed_size <= size * nmemb); @@ -528,7 +528,7 @@ static int update_content_encoding_header(Uploader *u, const CompressionConfig * return 0; /* Already picked the algorithm. Let's shortcut. */ if (cc) { - _cleanup_free_ char *header = strjoin("Content-Encoding: ", compression_lowercase_to_string(cc->algorithm)); + _cleanup_free_ char *header = strjoin("Content-Encoding: ", compression_to_string(cc->algorithm)); if (!header) return log_oom(); @@ -572,7 +572,7 @@ static int update_content_encoding_header(Uploader *u, const CompressionConfig * u->compression = cc; if (cc) - log_debug("Using compression algorithm %s with compression level %i.", compression_lowercase_to_string(cc->algorithm), cc->level); + log_debug("Using compression algorithm %s with compression level %i.", compression_to_string(cc->algorithm), cc->level); else log_debug("Disabled compression algorithm."); return 0; @@ -610,7 +610,7 @@ static int parse_accept_encoding_header(Uploader *u) { if (streq(word, "*")) return update_content_encoding_header(u, ordered_hashmap_first(arg_compression)); - Compression c = compression_lowercase_from_string(word); + Compression c = compression_from_string_harder(word); if (c <= 0 || !compression_supported(c)) continue; /* unsupported or invalid algorithm. */ diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index 235f471224..54c647d75b 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -370,7 +370,7 @@ static Compression getenv_compression(void) { if (r >= 0) return r ? DEFAULT_COMPRESSION : COMPRESSION_NONE; - c = compression_from_string(e); + c = compression_from_string_harder(e); if (c < 0) { log_debug_errno(c, "Failed to parse SYSTEMD_JOURNAL_COMPRESS value, ignoring: %s", e); return DEFAULT_COMPRESSION; diff --git a/src/test/test-compress-benchmark.c b/src/test/test-compress-benchmark.c index da68c6cb7b..8b00a00a06 100644 --- a/src/test/test-compress-benchmark.c +++ b/src/test/test-compress-benchmark.c @@ -1,27 +1,36 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ #include "alloc-util.h" +#include "argv-util.h" #include "compress.h" -#include "nulstr-util.h" #include "parse-util.h" #include "process-util.h" #include "random-util.h" +#include "string-table.h" #include "tests.h" #include "time-util.h" -typedef int (compress_t)(const void *src, uint64_t src_size, void *dst, - size_t dst_alloc_size, size_t *dst_size, int level); -typedef int (decompress_t)(const void *src, uint64_t src_size, - void **dst, size_t* dst_size, size_t dst_max); - -#if HAVE_COMPRESSION - static usec_t arg_duration; static size_t arg_start; #define MAX_SIZE (1024*1024LU) #define PRIME 1048571 /* A prime close enough to one megabyte that mod 4 == 3 */ +typedef enum BenchmarkDataType { + BENCHMARK_DATA_ZEROS, + BENCHMARK_DATA_SIMPLE, + BENCHMARK_DATA_RANDOM, + _BENCHMARK_DATA_TYPE_MAX, +} BenchmarkDataType; + +static const char* const benchmark_data_type_table[_BENCHMARK_DATA_TYPE_MAX] = { + [BENCHMARK_DATA_ZEROS] = "zeros", + [BENCHMARK_DATA_SIMPLE] = "simple", + [BENCHMARK_DATA_RANDOM] = "random", +}; + +DEFINE_PRIVATE_STRING_TABLE_LOOKUP_TO_STRING(benchmark_data_type, BenchmarkDataType); + static size_t _permute(size_t x) { size_t residue; @@ -39,19 +48,24 @@ static size_t permute(size_t x) { return _permute((_permute(x) + arg_start) % MAX_SIZE ^ 0xFF345); } -static char* make_buf(size_t count, const char *type) { +static char* make_buf(size_t count, BenchmarkDataType type) { char *buf; - size_t i; buf = malloc(count); - assert_se(buf); + ASSERT_NOT_NULL(buf); - if (streq(type, "zeros")) + switch (type) { + + case BENCHMARK_DATA_ZEROS: memzero(buf, count); - else if (streq(type, "simple")) - for (i = 0; i < count; i++) + break; + + case BENCHMARK_DATA_SIMPLE: + for (size_t i = 0; i < count; i++) buf[i] = 'a' + i % ('z' - 'a' + 1); - else if (streq(type, "random")) { + break; + + case BENCHMARK_DATA_RANDOM: { size_t step = count / 10; random_bytes(buf, step); @@ -64,110 +78,103 @@ static char* make_buf(size_t count, const char *type) { memzero(buf + 7*step, step); random_bytes(buf + 8*step, step); memzero(buf + 9*step, step); - } else + break; + } + + default: assert_not_reached(); + } return buf; } -static void test_compress_decompress(const char* label, const char* type, - compress_t compress, decompress_t decompress) { - usec_t n, n2 = 0; - float dt; +TEST(benchmark) { + for (BenchmarkDataType dt = 0; dt < _BENCHMARK_DATA_TYPE_MAX; dt++) + for (Compression c = 0; c < _COMPRESSION_MAX; c++) { + if (c == COMPRESSION_NONE || !compression_supported(c)) + continue; - _cleanup_free_ char *text = NULL, *buf = NULL; - _cleanup_free_ void *buf2 = NULL; - size_t skipped = 0, compressed = 0, total = 0; + const char *label = compression_to_string(c); + const char *type = benchmark_data_type_to_string(dt); + usec_t n, n2 = 0; - text = make_buf(MAX_SIZE, type); - buf = calloc(MAX_SIZE + 1, 1); - assert_se(text && buf); + _cleanup_free_ char *text = NULL, *buf = NULL; + _cleanup_free_ void *buf2 = NULL; + size_t skipped = 0, compressed = 0, total = 0; - n = now(CLOCK_MONOTONIC); + text = make_buf(MAX_SIZE, dt); + buf = calloc(MAX_SIZE + 1, 1); + ASSERT_NOT_NULL(text); + ASSERT_NOT_NULL(buf); - for (size_t i = 0; i <= MAX_SIZE; i++) { - size_t j = 0, k = 0, size; - int r; + n = now(CLOCK_MONOTONIC); - size = permute(i); - if (size == 0) - continue; + for (size_t i = 0; i <= MAX_SIZE; i++) { + size_t j = 0, k = 0, size; + int r; - log_debug("%s %zu %zu", type, i, size); + size = permute(i); + if (size == 0) + continue; - memzero(buf, MIN(size + 1000, MAX_SIZE)); + log_debug("%s %zu %zu", type, i, size); - r = compress(text, size, buf, size, &j, /* level= */ -1); - /* assume compression must be successful except for small or random inputs */ - assert_se(r >= 0 || (size < 2048 && r == -ENOBUFS) || streq(type, "random")); + memzero(buf, MIN(size + 1000, MAX_SIZE)); - /* check for overwrites */ - assert_se(buf[size] == 0); - if (r < 0) { - skipped += size; - continue; + r = compress_blob(c, text, size, buf, size, &j, /* level= */ -1); + /* assume compression must be successful except for small or random inputs */ + ASSERT_TRUE(r >= 0 || (size < 2048 && r == -ENOBUFS) || dt == BENCHMARK_DATA_RANDOM); + + /* check for overwrites */ + ASSERT_EQ(buf[size], 0); + if (r < 0) { + skipped += size; + continue; + } + + ASSERT_TRUE(j > 0); + if (j >= size) + log_error("%s \"compressed\" %zu -> %zu", label, size, j); + + ASSERT_OK_ZERO(decompress_blob(c, buf, j, &buf2, &k, 0)); + ASSERT_EQ(k, size); + ASSERT_EQ(memcmp(text, buf2, size), 0); + + total += size; + compressed += j; + + n2 = now(CLOCK_MONOTONIC); + if (n2 - n > arg_duration) + break; + } + + float elapsed = (n2-n) / 1e6; + + log_info("%s/%s: compressed & decompressed %zu bytes in %.2fs (%.2fMiB/s), " + "mean compression %.2f%%, skipped %zu bytes", + label, type, total, elapsed, + total / 1024. / 1024 / elapsed, + 100 - compressed * 100. / total, + skipped); } - - assert_se(j > 0); - if (j >= size) - log_error("%s \"compressed\" %zu -> %zu", label, size, j); - - r = decompress(buf, j, &buf2, &k, 0); - assert_se(r == 0); - assert_se(k == size); - - assert_se(memcmp(text, buf2, size) == 0); - - total += size; - compressed += j; - - n2 = now(CLOCK_MONOTONIC); - if (n2 - n > arg_duration) - break; - } - - dt = (n2-n) / 1e6; - - log_info("%s/%s: compressed & decompressed %zu bytes in %.2fs (%.2fMiB/s), " - "mean compression %.2f%%, skipped %zu bytes", - label, type, total, dt, - total / 1024. / 1024 / dt, - 100 - compressed * 100. / total, - skipped); } -#endif -int main(int argc, char *argv[]) { -#if HAVE_COMPRESSION - test_setup_logging(LOG_INFO); - - if (argc >= 2) { +static int intro(void) { + if (saved_argc >= 2) { unsigned x; - assert_se(safe_atou(argv[1], &x) >= 0); + ASSERT_OK(safe_atou(saved_argv[1], &x)); arg_duration = x * USEC_PER_SEC; } else arg_duration = slow_tests_enabled() ? 2 * USEC_PER_SEC : USEC_PER_SEC / 50; - if (argc == 3) - (void) safe_atozu(argv[2], &arg_start); + if (saved_argc == 3) + (void) safe_atozu(saved_argv[2], &arg_start); else arg_start = getpid_cached(); - NULSTR_FOREACH(i, "zeros\0simple\0random\0") { -#if HAVE_XZ - test_compress_decompress("XZ", i, compress_blob_xz, decompress_blob_xz); -#endif -#if HAVE_LZ4 - test_compress_decompress("LZ4", i, compress_blob_lz4, decompress_blob_lz4); -#endif -#if HAVE_ZSTD - test_compress_decompress("ZSTD", i, compress_blob_zstd, decompress_blob_zstd); -#endif - } return 0; -#else - return log_tests_skipped("No compression feature is enabled"); -#endif } + +DEFINE_TEST_MAIN_WITH_INTRO(LOG_INFO, intro); diff --git a/src/test/test-compress.c b/src/test/test-compress.c index 80f2923dd6..0c90443a87 100644 --- a/src/test/test-compress.c +++ b/src/test/test-compress.c @@ -4,13 +4,9 @@ #include #include -#if HAVE_LZ4 -#include -#endif - #include "alloc-util.h" +#include "argv-util.h" #include "compress.h" -#include "dlfcn-util.h" #include "fd-util.h" #include "io-util.h" #include "path-util.h" @@ -18,516 +14,455 @@ #include "tests.h" #include "tmpfile-util.h" -#if HAVE_XZ -# define XZ_OK 0 -#else -# define XZ_OK -EPROTONOSUPPORT -#endif - -#if HAVE_LZ4 -# define LZ4_OK 0 -#else -# define LZ4_OK -EPROTONOSUPPORT -#endif - #define HUGE_SIZE (4096*1024) -typedef int (compress_blob_t)(const void *src, uint64_t src_size, - void *dst, size_t dst_alloc_size, size_t *dst_size, int level); -typedef int (decompress_blob_t)(const void *src, uint64_t src_size, - void **dst, - size_t* dst_size, size_t dst_max); -typedef int (decompress_sw_t)(const void *src, uint64_t src_size, - void **buffer, - const void *prefix, size_t prefix_len, - uint8_t extra); +static const char text[] = + "text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF" + "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"; +static char data[512] = "random\0"; +static char *huge = NULL; +static const char *srcfile; -typedef int (compress_stream_t)(int fdf, int fdt, uint64_t max_bytes, uint64_t *uncompressed_size); -typedef int (decompress_stream_t)(int fdf, int fdt, uint64_t max_size); - -#if HAVE_COMPRESSION -_unused_ static void test_compress_decompress( - const char *compression, - compress_blob_t compress, - decompress_blob_t decompress, - const char *data, - size_t data_len, - bool may_fail) { - - char compressed[512]; - size_t csize; - _cleanup_free_ char *decompressed = NULL; - int r; - - log_info("/* testing %s %s blob compression/decompression */", - compression, data); - - r = compress(data, data_len, compressed, sizeof(compressed), &csize, /* level= */ -1); - if (r == -ENOBUFS) { - log_info_errno(r, "compression failed: %m"); - assert_se(may_fail); - } else { - assert_se(r >= 0); - r = decompress(compressed, csize, - (void **) &decompressed, &csize, 0); - assert_se(r == 0); - assert_se(decompressed); - assert_se(memcmp(decompressed, data, data_len) == 0); +static const char* cat_for_compression(Compression c) { + switch (c) { + case COMPRESSION_XZ: return "xzcat"; + case COMPRESSION_LZ4: return "lz4cat"; + case COMPRESSION_ZSTD: return "zstdcat"; + case COMPRESSION_GZIP: return "zcat"; + case COMPRESSION_BZIP2: return "bzcat"; + default: return NULL; } - - r = decompress("garbage", 7, - (void **) &decompressed, &csize, 0); - assert_se(r < 0); - - /* make sure to have the minimal lz4 compressed size */ - r = decompress("00000000\1g", 9, - (void **) &decompressed, &csize, 0); - assert_se(r < 0); - - r = decompress("\100000000g", 9, - (void **) &decompressed, &csize, 0); - assert_se(r < 0); - - explicit_bzero_safe(decompressed, MALLOC_SIZEOF_SAFE(decompressed)); } -_unused_ static void test_decompress_startswith(const char *compression, - compress_blob_t compress, - decompress_sw_t decompress_sw, - const char *data, - size_t data_len, - bool may_fail) { +TEST(compress_decompress_blob) { + for (Compression c = 0; c < _COMPRESSION_MAX; c++) { + if (c == COMPRESSION_NONE || !compression_supported(c)) + continue; - char *compressed; - _cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL; - size_t csize, len; - int r; + const char *label = compression_to_string(c); - log_info("/* testing decompress_startswith with %s on %.20s text */", - compression, data); + for (size_t t = 0; t < 2; t++) { + const char *input = t == 0 ? text : data; + size_t input_len = t == 0 ? sizeof(text) : sizeof(data); + bool may_fail = t == 1; -#define BUFSIZE_1 512 -#define BUFSIZE_2 20000 + char compressed[512]; + size_t csize; + _cleanup_free_ char *decompressed = NULL; + int r; - compressed = compressed1 = malloc(BUFSIZE_1); - assert_se(compressed1); - r = compress(data, data_len, compressed, BUFSIZE_1, &csize, /* level= */ -1); - if (r == -ENOBUFS) { - log_info_errno(r, "compression failed: %m"); - assert_se(may_fail); + log_info("/* testing %s %s blob compression/decompression */", label, input); - compressed = compressed2 = malloc(BUFSIZE_2); - assert_se(compressed2); - r = compress(data, data_len, compressed, BUFSIZE_2, &csize, /* level= */ -1); + r = compress_blob(c, input, input_len, compressed, sizeof(compressed), &csize, -1); + if (r == -ENOBUFS) { + log_info_errno(r, "compression failed: %m"); + ASSERT_TRUE(may_fail); + } else { + ASSERT_OK(r); + ASSERT_OK_ZERO(decompress_blob(c, compressed, csize, (void **) &decompressed, &csize, 0)); + ASSERT_NOT_NULL(decompressed); + ASSERT_EQ(memcmp(decompressed, input, input_len), 0); + } + + ASSERT_FAIL(decompress_blob(c, "garbage", 7, (void **) &decompressed, &csize, 0)); + } } - assert_se(r >= 0); - - len = strlen(data); - - r = decompress_sw(compressed, csize, (void **) &decompressed, data, len, '\0'); - assert_se(r > 0); - r = decompress_sw(compressed, csize, (void **) &decompressed, data, len, 'w'); - assert_se(r == 0); - r = decompress_sw(compressed, csize, (void **) &decompressed, "barbarbar", 9, ' '); - assert_se(r == 0); - r = decompress_sw(compressed, csize, (void **) &decompressed, data, len - 1, data[len-1]); - assert_se(r > 0); - r = decompress_sw(compressed, csize, (void **) &decompressed, data, len - 1, 'w'); - assert_se(r == 0); - r = decompress_sw(compressed, csize, (void **) &decompressed, data, len, '\0'); - assert_se(r > 0); } -_unused_ static void test_decompress_startswith_short(const char *compression, - compress_blob_t compress, - decompress_sw_t decompress_sw) { +TEST(decompress_startswith) { + for (Compression c = 0; c < _COMPRESSION_MAX; c++) { + if (c == COMPRESSION_NONE || !compression_supported(c)) + continue; + const char *label = compression_to_string(c); + + struct { const char *buf; size_t len; bool may_fail; } inputs[] = { + { text, sizeof(text), false }, + { data, sizeof(data), true }, + { huge, HUGE_SIZE, true }, + }; + + for (size_t t = 0; t < ELEMENTSOF(inputs); t++) { + char *compressed; + _cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL; + size_t csize, len; + int r; + + log_info("/* testing decompress_startswith with %s on %.20s */", label, inputs[t].buf); + + compressed = compressed1 = malloc(512); + ASSERT_NOT_NULL(compressed1); + r = compress_blob(c, inputs[t].buf, inputs[t].len, compressed, 512, &csize, -1); + if (r == -ENOBUFS) { + log_info_errno(r, "compression failed: %m"); + ASSERT_TRUE(inputs[t].may_fail); + + compressed = compressed2 = malloc(20000); + ASSERT_NOT_NULL(compressed2); + r = compress_blob(c, inputs[t].buf, inputs[t].len, compressed, 20000, &csize, -1); + } + if (r == -ENOBUFS) { + log_info_errno(r, "compression failed again: %m"); + ASSERT_TRUE(inputs[t].may_fail); + continue; + } + ASSERT_OK(r); + + len = strlen(inputs[t].buf); + + ASSERT_OK_POSITIVE(decompress_startswith(c, compressed, csize, (void **) &decompressed, inputs[t].buf, len, '\0')); + ASSERT_OK_ZERO(decompress_startswith(c, compressed, csize, (void **) &decompressed, inputs[t].buf, len, 'w')); + ASSERT_OK_POSITIVE(decompress_startswith(c, compressed, csize, (void **) &decompressed, inputs[t].buf, len - 1, inputs[t].buf[len-1])); + ASSERT_OK_ZERO(decompress_startswith(c, compressed, csize, (void **) &decompressed, inputs[t].buf, len - 1, 'w')); + } + } +} + +TEST(decompress_startswith_large) { + /* Test decompress_startswith with large data to exercise the buffer growth path. */ + + _cleanup_free_ char *large = NULL; + size_t large_size = 8 * 1024; + + ASSERT_NOT_NULL(large = malloc(large_size)); + for (size_t i = 0; i < large_size; i++) + large[i] = 'A' + (i % 26); + + for (Compression c = 0; c < _COMPRESSION_MAX; c++) { + if (c == COMPRESSION_NONE || !compression_supported(c)) + continue; + + _cleanup_free_ char *compressed = NULL; + size_t csize; + + log_info("/* decompress_startswith_large with %s */", compression_to_string(c)); + + ASSERT_NOT_NULL(compressed = malloc(large_size)); + int r = compress_blob(c, large, large_size, compressed, large_size, &csize, -1); + if (r == -ENOBUFS) { + log_info_errno(r, "compression failed: %m"); + continue; + } + ASSERT_OK(r); + + _cleanup_free_ void *buf = NULL; + + ASSERT_OK_POSITIVE(decompress_startswith(c, compressed, csize, &buf, large, 1, large[1])); + ASSERT_OK_ZERO(decompress_startswith(c, compressed, csize, &buf, large, 1, 0xff)); + ASSERT_OK_POSITIVE(decompress_startswith(c, compressed, csize, &buf, large, 512, large[512])); + ASSERT_OK_ZERO(decompress_startswith(c, compressed, csize, &buf, large, 512, 0xff)); + ASSERT_OK_POSITIVE(decompress_startswith(c, compressed, csize, &buf, large, 4096, large[4096])); + ASSERT_OK_ZERO(decompress_startswith(c, compressed, csize, &buf, large, 4096, 0xff)); + } +} + +TEST(decompress_startswith_short) { #define TEXT "HUGE=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - char buf[1024]; - size_t csize; - int r; + for (Compression c = 0; c < _COMPRESSION_MAX; c++) { + if (c == COMPRESSION_NONE || !compression_supported(c)) + continue; - log_info("/* %s with %s */", __func__, compression); + char buf[1024]; + size_t csize; - r = compress(TEXT, sizeof TEXT, buf, sizeof buf, &csize, /* level= */ -1); - assert_se(r >= 0); + log_info("/* decompress_startswith_short with %s */", compression_to_string(c)); - for (size_t i = 1; i < strlen(TEXT); i++) { - _cleanup_free_ void *buf2 = NULL; + ASSERT_OK(compress_blob(c, TEXT, sizeof TEXT, buf, sizeof buf, &csize, -1)); - assert_se(buf2 = malloc(i)); + for (size_t i = 1; i < strlen(TEXT); i++) { + _cleanup_free_ void *buf2 = NULL; - assert_se(decompress_sw(buf, csize, &buf2, TEXT, i, TEXT[i]) == 1); - assert_se(decompress_sw(buf, csize, &buf2, TEXT, i, 'y') == 0); + ASSERT_NOT_NULL(buf2 = malloc(i)); + + ASSERT_OK_POSITIVE(decompress_startswith(c, buf, csize, &buf2, TEXT, i, TEXT[i])); + ASSERT_OK_ZERO(decompress_startswith(c, buf, csize, &buf2, TEXT, i, 'y')); + } } +#undef TEXT } -_unused_ static void test_compress_stream(const char *compression, - const char *cat, - compress_stream_t compress, - decompress_stream_t decompress, - const char *srcfile) { +TEST(compress_decompress_stream) { + for (Compression c = 0; c < _COMPRESSION_MAX; c++) { + if (c == COMPRESSION_NONE || !compression_supported(c)) + continue; - _cleanup_close_ int src = -EBADF, dst = -EBADF, dst2 = -EBADF; - _cleanup_(unlink_tempfilep) char - pattern[] = "/tmp/systemd-test.compressed.XXXXXX", - pattern2[] = "/tmp/systemd-test.compressed.XXXXXX"; - int r; - _cleanup_free_ char *cmd = NULL, *cmd2 = NULL; - struct stat st = {}; - uint64_t uncompressed_size; + const char *cat = cat_for_compression(c); + if (!cat) + continue; - r = find_executable(cat, NULL); - if (r < 0) { - log_error_errno(r, "Skipping %s, could not find %s binary: %m", __func__, cat); - return; - } + int r = find_executable(cat, NULL); + if (r < 0) { + log_error_errno(r, "Skipping %s, could not find %s binary: %m", + compression_to_string(c), cat); + continue; + } - log_debug("/* testing %s compression */", compression); - - log_debug("/* create source from %s */", srcfile); - - ASSERT_OK(src = open(srcfile, O_RDONLY|O_CLOEXEC)); - - log_debug("/* test compression */"); - - assert_se((dst = mkostemp_safe(pattern)) >= 0); - - ASSERT_OK(compress(src, dst, -1, &uncompressed_size)); - - if (cat) { - assert_se(asprintf(&cmd, "%s %s | diff '%s' -", cat, pattern, srcfile) > 0); - assert_se(system(cmd) == 0); - } - - log_debug("/* test decompression */"); - - assert_se((dst2 = mkostemp_safe(pattern2)) >= 0); - - assert_se(stat(srcfile, &st) == 0); - assert_se((uint64_t)st.st_size == uncompressed_size); - - assert_se(lseek(dst, 0, SEEK_SET) == 0); - r = decompress(dst, dst2, st.st_size); - assert_se(r == 0); - - assert_se(asprintf(&cmd2, "diff '%s' %s", srcfile, pattern2) > 0); - assert_se(system(cmd2) == 0); - - log_debug("/* test faulty decompression */"); - - assert_se(lseek(dst, 1, SEEK_SET) == 1); - r = decompress(dst, dst2, st.st_size); - assert_se(IN_SET(r, 0, -EBADMSG)); - - assert_se(lseek(dst, 0, SEEK_SET) == 0); - assert_se(lseek(dst2, 0, SEEK_SET) == 0); - r = decompress(dst, dst2, st.st_size - 1); - assert_se(r == -EFBIG); -} - -_unused_ static void test_decompress_stream_sparse(const char *compression, - compress_stream_t compress, - decompress_stream_t decompress) { - - _cleanup_close_ int src = -EBADF, compressed = -EBADF, decompressed = -EBADF; - _cleanup_(unlink_tempfilep) char - pattern_src[] = "/tmp/systemd-test.sparse-src.XXXXXX", - pattern_compressed[] = "/tmp/systemd-test.sparse-compressed.XXXXXX", - pattern_decompressed[] = "/tmp/systemd-test.sparse-decompressed.XXXXXX"; - /* Create a sparse-like input: 4K of data, 64K of zeros, 4K of data, 64K trailing zeros. - * Total apparent size: 136K, but most of it is zeros. */ - uint8_t data_block[4096]; - struct stat st_src, st_decompressed; - uint64_t uncompressed_size; - int r; - - assert(compression); - - log_debug("/* testing %s sparse decompression */", compression); - - random_bytes(data_block, sizeof(data_block)); - - assert_se((src = mkostemp_safe(pattern_src)) >= 0); - - /* Write: 4K data, 64K zeros, 4K data, 64K zeros */ - assert_se(loop_write(src, data_block, sizeof(data_block)) >= 0); - assert_se(ftruncate(src, sizeof(data_block) + 65536) >= 0); - assert_se(lseek(src, sizeof(data_block) + 65536, SEEK_SET) >= 0); - assert_se(loop_write(src, data_block, sizeof(data_block)) >= 0); - assert_se(ftruncate(src, 2 * sizeof(data_block) + 2 * 65536) >= 0); - assert_se(lseek(src, 0, SEEK_SET) == 0); - - assert_se(fstat(src, &st_src) >= 0); - assert_se(st_src.st_size == 2 * (off_t) sizeof(data_block) + 2 * 65536); - - /* Compress */ - assert_se((compressed = mkostemp_safe(pattern_compressed)) >= 0); - ASSERT_OK(compress(src, compressed, -1, &uncompressed_size)); - assert_se((uint64_t) st_src.st_size == uncompressed_size); - - /* Decompress to a regular file (sparse writes auto-detected) */ - assert_se((decompressed = mkostemp_safe(pattern_decompressed)) >= 0); - assert_se(lseek(compressed, 0, SEEK_SET) == 0); - r = decompress(compressed, decompressed, st_src.st_size); - assert_se(r == 0); - - /* Verify apparent size matches */ - assert_se(fstat(decompressed, &st_decompressed) >= 0); - assert_se(st_decompressed.st_size == st_src.st_size); - - /* Verify content matches by comparing bytes */ - assert_se(lseek(src, 0, SEEK_SET) == 0); - assert_se(lseek(decompressed, 0, SEEK_SET) == 0); - - for (off_t offset = 0; offset < st_src.st_size;) { - uint8_t buf_src[4096], buf_dst[4096]; - size_t to_read = MIN((size_t) (st_src.st_size - offset), sizeof(buf_src)); - ssize_t n; - - n = loop_read(src, buf_src, to_read, true); - assert_se(n == (ssize_t) to_read); - n = loop_read(decompressed, buf_dst, to_read, true); - assert_se(n == (ssize_t) to_read); - assert_se(memcmp(buf_src, buf_dst, to_read) == 0); - offset += to_read; - } - - /* Verify the decompressed file is actually sparse (uses less disk than apparent size). - * st_blocks is in 512-byte units. The file has 128K of zeros, so disk usage should be - * noticeably less than the apparent size if sparse writes worked. - * Only assert if the filesystem supports holes (SEEK_HOLE). */ - log_debug("%s sparse decompression: apparent=%jd disk=%jd", - compression, - (intmax_t) st_decompressed.st_size, - (intmax_t) st_decompressed.st_blocks * 512); - if (lseek(decompressed, 0, SEEK_HOLE) < st_decompressed.st_size) - assert_se(st_decompressed.st_blocks * 512 < st_decompressed.st_size); - else - log_debug("Filesystem does not support holes, skipping sparsity check"); - - /* Test all-zeros input: entire output should be a hole */ - log_debug("/* testing %s sparse decompression of all-zeros */", compression); - { - _cleanup_close_ int zsrc = -EBADF, zcompressed = -EBADF, zdecompressed = -EBADF; + _cleanup_close_ int src = -EBADF, dst = -EBADF, dst2 = -EBADF; _cleanup_(unlink_tempfilep) char - zp_src[] = "/tmp/systemd-test.sparse-zero-src.XXXXXX", - zp_compressed[] = "/tmp/systemd-test.sparse-zero-compressed.XXXXXX", - zp_decompressed[] = "/tmp/systemd-test.sparse-zero-decompressed.XXXXXX"; - struct stat zst; - uint64_t zsize; - uint8_t zeros[65536] = {}; + pattern[] = "/tmp/systemd-test.compressed.XXXXXX", + pattern2[] = "/tmp/systemd-test.compressed.XXXXXX"; + _cleanup_free_ char *cmd = NULL, *cmd2 = NULL; + struct stat st = {}; + uint64_t uncompressed_size; - assert_se((zsrc = mkostemp_safe(zp_src)) >= 0); - assert_se(loop_write(zsrc, zeros, sizeof(zeros)) >= 0); - assert_se(lseek(zsrc, 0, SEEK_SET) == 0); + log_debug("/* testing %s stream compression */", compression_to_string(c)); - assert_se((zcompressed = mkostemp_safe(zp_compressed)) >= 0); - ASSERT_OK(compress(zsrc, zcompressed, -1, &zsize)); - assert_se(zsize == sizeof(zeros)); + ASSERT_OK(src = open(srcfile, O_RDONLY|O_CLOEXEC)); + ASSERT_OK(dst = mkostemp_safe(pattern)); - assert_se((zdecompressed = mkostemp_safe(zp_decompressed)) >= 0); - assert_se(lseek(zcompressed, 0, SEEK_SET) == 0); - assert_se(decompress(zcompressed, zdecompressed, sizeof(zeros)) == 0); + ASSERT_OK(compress_stream(c, src, dst, -1, &uncompressed_size)); - assert_se(fstat(zdecompressed, &zst) >= 0); - assert_se(zst.st_size == (off_t) sizeof(zeros)); - /* All zeros — disk usage should be minimal */ - log_debug("%s all-zeros sparse: apparent=%jd disk=%jd", - compression, (intmax_t) zst.st_size, (intmax_t) zst.st_blocks * 512); - if (lseek(zdecompressed, 0, SEEK_HOLE) < zst.st_size) - assert_se(zst.st_blocks * 512 < zst.st_size); + ASSERT_OK_POSITIVE(asprintf(&cmd, "%s %s | diff '%s' -", cat, pattern, srcfile)); + ASSERT_OK_ZERO(system(cmd)); + + ASSERT_OK(dst2 = mkostemp_safe(pattern2)); + + ASSERT_OK_ZERO_ERRNO(stat(srcfile, &st)); + ASSERT_EQ((uint64_t) st.st_size, uncompressed_size); + + ASSERT_OK_ERRNO(lseek(dst, 0, SEEK_SET)); + ASSERT_OK_ZERO(decompress_stream(c, dst, dst2, st.st_size)); + + ASSERT_OK_POSITIVE(asprintf(&cmd2, "diff '%s' %s", srcfile, pattern2)); + ASSERT_OK_ZERO(system(cmd2)); + + log_debug("/* test faulty decompression */"); + + ASSERT_OK_ERRNO(lseek(dst, 1, SEEK_SET)); + r = decompress_stream(c, dst, dst2, st.st_size); + ASSERT_TRUE(IN_SET(r, 0, -EBADMSG)); + + ASSERT_OK_ERRNO(lseek(dst, 0, SEEK_SET)); + ASSERT_OK_ERRNO(lseek(dst2, 0, SEEK_SET)); + ASSERT_ERROR(decompress_stream(c, dst, dst2, st.st_size - 1), EFBIG); + } +} + +struct decompressor_test_data { + uint8_t *buf; + size_t size; +}; + +static int test_decompressor_callback(const void *p, size_t size, void *userdata) { + struct decompressor_test_data *d = ASSERT_PTR(userdata); + + if (!GREEDY_REALLOC(d->buf, d->size + size)) + return -ENOMEM; + + memcpy(d->buf + d->size, p, size); + d->size += size; + return 0; +} + +TEST(decompress_stream_sparse) { + for (Compression c = 0; c < _COMPRESSION_MAX; c++) { + if (c == COMPRESSION_NONE || !compression_supported(c)) + continue; + + _cleanup_close_ int src = -EBADF, compressed = -EBADF, decompressed = -EBADF; + _cleanup_(unlink_tempfilep) char + pattern_src[] = "/tmp/systemd-test.sparse-src.XXXXXX", + pattern_compressed[] = "/tmp/systemd-test.sparse-compressed.XXXXXX", + pattern_decompressed[] = "/tmp/systemd-test.sparse-decompressed.XXXXXX"; + /* Create a sparse-like input: 4K of data, 64K of zeros, 4K of data, 64K trailing zeros. + * Total apparent size: 136K, but most of it is zeros. */ + uint8_t data_block[4096]; + struct stat st_src, st_decompressed; + uint64_t uncompressed_size; + + log_debug("/* testing %s sparse decompression */", compression_to_string(c)); + + random_bytes(data_block, sizeof(data_block)); + + ASSERT_OK(src = mkostemp_safe(pattern_src)); + + /* Write: 4K data, 64K zeros, 4K data, 64K zeros */ + ASSERT_OK(loop_write(src, data_block, sizeof(data_block))); + ASSERT_OK_ERRNO(ftruncate(src, sizeof(data_block) + 65536)); + ASSERT_OK_ERRNO(lseek(src, sizeof(data_block) + 65536, SEEK_SET)); + ASSERT_OK(loop_write(src, data_block, sizeof(data_block))); + ASSERT_OK_ERRNO(ftruncate(src, 2 * sizeof(data_block) + 2 * 65536)); + ASSERT_EQ(lseek(src, 0, SEEK_SET), (off_t) 0); + + ASSERT_OK_ERRNO(fstat(src, &st_src)); + ASSERT_EQ(st_src.st_size, 2 * (off_t) sizeof(data_block) + 2 * 65536); + + /* Compress */ + ASSERT_OK(compressed = mkostemp_safe(pattern_compressed)); + ASSERT_OK(compress_stream(c, src, compressed, -1, &uncompressed_size)); + ASSERT_EQ((uint64_t) st_src.st_size, uncompressed_size); + + /* Decompress to a regular file (sparse writes auto-detected) */ + ASSERT_OK(decompressed = mkostemp_safe(pattern_decompressed)); + ASSERT_EQ(lseek(compressed, 0, SEEK_SET), (off_t) 0); + ASSERT_OK_ZERO(decompress_stream(c, compressed, decompressed, st_src.st_size)); + + /* Verify apparent size matches */ + ASSERT_OK_ERRNO(fstat(decompressed, &st_decompressed)); + ASSERT_EQ(st_decompressed.st_size, st_src.st_size); + + /* Verify content matches by comparing bytes */ + ASSERT_EQ(lseek(src, 0, SEEK_SET), (off_t) 0); + ASSERT_EQ(lseek(decompressed, 0, SEEK_SET), (off_t) 0); + + for (off_t offset = 0; offset < st_src.st_size;) { + uint8_t buf_src[4096], buf_dst[4096]; + size_t to_read = MIN((size_t) (st_src.st_size - offset), sizeof(buf_src)); + + ASSERT_EQ(loop_read(src, buf_src, to_read, true), (ssize_t) to_read); + ASSERT_EQ(loop_read(decompressed, buf_dst, to_read, true), (ssize_t) to_read); + ASSERT_EQ(memcmp(buf_src, buf_dst, to_read), 0); + offset += to_read; + } + + /* Verify the decompressed file is actually sparse (uses less disk than apparent size). + * st_blocks is in 512-byte units. The file has 128K of zeros, so disk usage should be + * noticeably less than the apparent size if sparse writes worked. + * Only assert if the filesystem supports holes (SEEK_HOLE). */ + log_debug("%s sparse decompression: apparent=%jd disk=%jd", + compression_to_string(c), + (intmax_t) st_decompressed.st_size, + (intmax_t) st_decompressed.st_blocks * 512); + if (lseek(decompressed, 0, SEEK_HOLE) < st_decompressed.st_size) + ASSERT_LT(st_decompressed.st_blocks * 512, st_decompressed.st_size); else log_debug("Filesystem does not support holes, skipping sparsity check"); - } - /* Test data ending with non-zero bytes: ftruncate should be a no-op */ - log_debug("/* testing %s sparse decompression ending with data */", compression); - { - _cleanup_close_ int dsrc = -EBADF, dcompressed = -EBADF, ddecompressed = -EBADF; - _cleanup_(unlink_tempfilep) char - dp_src[] = "/tmp/systemd-test.sparse-end-src.XXXXXX", - dp_compressed[] = "/tmp/systemd-test.sparse-end-compressed.XXXXXX", - dp_decompressed[] = "/tmp/systemd-test.sparse-end-decompressed.XXXXXX"; - struct stat dst; - uint64_t dsize; - uint8_t zeros[65536] = {}; + /* Test all-zeros input: entire output should be a hole */ + log_debug("/* testing %s sparse decompression of all-zeros */", compression_to_string(c)); + { + _cleanup_close_ int zsrc = -EBADF, zcompressed = -EBADF, zdecompressed = -EBADF; + _cleanup_(unlink_tempfilep) char + zp_src[] = "/tmp/systemd-test.sparse-zero-src.XXXXXX", + zp_compressed[] = "/tmp/systemd-test.sparse-zero-compressed.XXXXXX", + zp_decompressed[] = "/tmp/systemd-test.sparse-zero-decompressed.XXXXXX"; + struct stat zst; + uint64_t zsize; + uint8_t zeros[65536] = {}; - /* 64K zeros followed by 4K random data */ - assert_se((dsrc = mkostemp_safe(dp_src)) >= 0); - assert_se(loop_write(dsrc, zeros, sizeof(zeros)) >= 0); - assert_se(loop_write(dsrc, data_block, sizeof(data_block)) >= 0); - assert_se(lseek(dsrc, 0, SEEK_SET) == 0); + ASSERT_OK(zsrc = mkostemp_safe(zp_src)); + ASSERT_OK(loop_write(zsrc, zeros, sizeof(zeros))); + ASSERT_EQ(lseek(zsrc, 0, SEEK_SET), (off_t) 0); - assert_se((dcompressed = mkostemp_safe(dp_compressed)) >= 0); - ASSERT_OK(compress(dsrc, dcompressed, -1, &dsize)); - assert_se(dsize == sizeof(zeros) + sizeof(data_block)); + ASSERT_OK(zcompressed = mkostemp_safe(zp_compressed)); + ASSERT_OK(compress_stream(c, zsrc, zcompressed, -1, &zsize)); + ASSERT_EQ(zsize, (uint64_t) sizeof(zeros)); - assert_se((ddecompressed = mkostemp_safe(dp_decompressed)) >= 0); - assert_se(lseek(dcompressed, 0, SEEK_SET) == 0); - assert_se(decompress(dcompressed, ddecompressed, dsize) == 0); + ASSERT_OK(zdecompressed = mkostemp_safe(zp_decompressed)); + ASSERT_EQ(lseek(zcompressed, 0, SEEK_SET), (off_t) 0); + ASSERT_OK_ZERO(decompress_stream(c, zcompressed, zdecompressed, sizeof(zeros))); - assert_se(fstat(ddecompressed, &dst) >= 0); - assert_se(dst.st_size == (off_t)(sizeof(zeros) + sizeof(data_block))); + ASSERT_OK_ERRNO(fstat(zdecompressed, &zst)); + ASSERT_EQ(zst.st_size, (off_t) sizeof(zeros)); + /* All zeros — disk usage should be minimal */ + log_debug("%s all-zeros sparse: apparent=%jd disk=%jd", + compression_to_string(c), (intmax_t) zst.st_size, (intmax_t) zst.st_blocks * 512); + if (lseek(zdecompressed, 0, SEEK_HOLE) < zst.st_size) + ASSERT_LT(zst.st_blocks * 512, zst.st_size); + else + log_debug("Filesystem does not support holes, skipping sparsity check"); + } + + /* Test data ending with non-zero bytes: ftruncate should be a no-op */ + log_debug("/* testing %s sparse decompression ending with data */", compression_to_string(c)); + { + _cleanup_close_ int dsrc = -EBADF, dcompressed = -EBADF, ddecompressed = -EBADF; + _cleanup_(unlink_tempfilep) char + dp_src[] = "/tmp/systemd-test.sparse-end-src.XXXXXX", + dp_compressed[] = "/tmp/systemd-test.sparse-end-compressed.XXXXXX", + dp_decompressed[] = "/tmp/systemd-test.sparse-end-decompressed.XXXXXX"; + struct stat dst; + uint64_t dsize; + uint8_t zeros[65536] = {}; + + /* 64K zeros followed by 4K random data */ + ASSERT_OK(dsrc = mkostemp_safe(dp_src)); + ASSERT_OK(loop_write(dsrc, zeros, sizeof(zeros))); + ASSERT_OK(loop_write(dsrc, data_block, sizeof(data_block))); + ASSERT_EQ(lseek(dsrc, 0, SEEK_SET), (off_t) 0); + + ASSERT_OK(dcompressed = mkostemp_safe(dp_compressed)); + ASSERT_OK(compress_stream(c, dsrc, dcompressed, -1, &dsize)); + ASSERT_EQ(dsize, (uint64_t)(sizeof(zeros) + sizeof(data_block))); + + ASSERT_OK(ddecompressed = mkostemp_safe(dp_decompressed)); + ASSERT_EQ(lseek(dcompressed, 0, SEEK_SET), (off_t) 0); + ASSERT_OK_ZERO(decompress_stream(c, dcompressed, ddecompressed, dsize)); + + ASSERT_OK_ERRNO(fstat(ddecompressed, &dst)); + ASSERT_EQ(dst.st_size, (off_t)(sizeof(zeros) + sizeof(data_block))); + } } } -#endif -#if HAVE_LZ4 -extern DLSYM_PROTOTYPE(LZ4_compress_default); -extern DLSYM_PROTOTYPE(LZ4_decompress_safe); -extern DLSYM_PROTOTYPE(LZ4_decompress_safe_partial); -extern DLSYM_PROTOTYPE(LZ4_versionNumber); +TEST(compressor_decompressor_push_api) { + for (Compression c = 0; c < _COMPRESSION_MAX; c++) { + if (c == COMPRESSION_NONE || !compression_supported(c)) + continue; -static void test_lz4_decompress_partial(void) { - char buf[20000], buf2[100]; - size_t buf_size = sizeof(buf), compressed; - int r; - _cleanup_free_ char *huge = NULL; + log_info("/* testing %s Compressor/Decompressor push API */", compression_to_string(c)); - log_debug("/* %s */", __func__); + _cleanup_(compressor_freep) Compressor *compressor = NULL; + _cleanup_(compressor_freep) Decompressor *decompressor = NULL; + _cleanup_free_ void *compressed = NULL, *finish_buf = NULL; + size_t compressed_size = 0, compressed_alloc = 0; + size_t finish_size = 0, finish_alloc = 0; - assert_se(huge = malloc(HUGE_SIZE)); + /* Compress */ + ASSERT_OK(compressor_new(&compressor, c)); + ASSERT_EQ(compressor_type(compressor), c); + + ASSERT_OK(compressor_start(compressor, text, sizeof(text), &compressed, &compressed_size, &compressed_alloc)); + ASSERT_OK(compressor_finish(compressor, &finish_buf, &finish_size, &finish_alloc)); + + size_t total_compressed = compressed_size + finish_size; + _cleanup_free_ void *full_compressed = malloc(total_compressed); + ASSERT_NOT_NULL(full_compressed); + memcpy(full_compressed, compressed, compressed_size); + if (finish_size > 0) + memcpy((uint8_t*) full_compressed + compressed_size, finish_buf, finish_size); + + compressor = compressor_free(compressor); + + /* Decompress via detect + push and verify content */ + ASSERT_OK_POSITIVE(decompressor_detect(&decompressor, full_compressed, total_compressed)); + ASSERT_EQ(compressor_type(decompressor), c); + + struct decompressor_test_data result = {}; + ASSERT_OK(decompressor_push(decompressor, full_compressed, total_compressed, test_decompressor_callback, &result)); + ASSERT_EQ(result.size, sizeof(text)); + ASSERT_EQ(memcmp(result.buf, text, sizeof(text)), 0); + free(result.buf); + + decompressor = compressor_free(decompressor); + } + + /* Test compressor_type on NULL */ + ASSERT_EQ(compressor_type(NULL), _COMPRESSION_INVALID); + + /* Test decompressor_force_off */ + _cleanup_(compressor_freep) Decompressor *d = NULL; + ASSERT_OK(decompressor_force_off(&d)); + ASSERT_EQ(compressor_type(d), COMPRESSION_NONE); + d = compressor_free(d); + + /* Test decompressor_detect returning 0 on too-small input */ + ASSERT_OK_ZERO(decompressor_detect(&d, "x", 1)); + ASSERT_NULL(d); +} + +static int intro(void) { + srcfile = saved_argc > 1 ? saved_argv[1] : saved_argv[0]; + + ASSERT_NOT_NULL(huge = malloc(HUGE_SIZE)); memcpy(huge, "HUGE=", STRLEN("HUGE=")); memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1); huge[HUGE_SIZE - 1] = '\0'; - r = sym_LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size); - assert_se(r >= 0); - compressed = r; - log_info("Compressed %i → %zu", HUGE_SIZE, compressed); - - r = sym_LZ4_decompress_safe(buf, huge, r, HUGE_SIZE); - assert_se(r >= 0); - log_info("Decompressed → %i", r); - - r = sym_LZ4_decompress_safe_partial(buf, huge, - compressed, - 12, HUGE_SIZE); - assert_se(r >= 0); - log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE, r); - - for (size_t size = 1; size < sizeof(buf2); size++) { - /* This failed in older lz4s but works in newer ones. */ - r = sym_LZ4_decompress_safe_partial(buf, buf2, compressed, size, size); - log_info("Decompressed partial %zu/%zu → %i (%s)", size, size, r, - r < 0 ? "bad" : "good"); - if (r >= 0 && sym_LZ4_versionNumber() >= 10803) - /* lz4 <= 1.8.2 should fail that test, let's only check for newer ones */ - assert_se(memcmp(buf2, huge, r) == 0); - } -} -#endif - -int main(int argc, char *argv[]) { -#if HAVE_COMPRESSION - _unused_ const char text[] = - "text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF" - "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"; - - /* The file to test compression on can be specified as the first argument */ - const char *srcfile = argc > 1 ? argv[1] : argv[0]; - - char data[512] = "random\0"; - - _cleanup_free_ char *huge = NULL; - - assert_se(huge = malloc(HUGE_SIZE)); - memcpy(huge, "HUGE=", STRLEN("HUGE=")); - memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1); - huge[HUGE_SIZE - 1] = '\0'; - - test_setup_logging(LOG_DEBUG); - random_bytes(data + 7, sizeof(data) - 7); -#if HAVE_XZ - test_compress_decompress("XZ", compress_blob_xz, decompress_blob_xz, - text, sizeof(text), false); - test_compress_decompress("XZ", compress_blob_xz, decompress_blob_xz, - data, sizeof(data), true); - - test_decompress_startswith("XZ", - compress_blob_xz, decompress_startswith_xz, - text, sizeof(text), false); - test_decompress_startswith("XZ", - compress_blob_xz, decompress_startswith_xz, - data, sizeof(data), true); - test_decompress_startswith("XZ", - compress_blob_xz, decompress_startswith_xz, - huge, HUGE_SIZE, true); - - test_compress_stream("XZ", "xzcat", - compress_stream_xz, decompress_stream_xz, srcfile); - - test_decompress_stream_sparse("XZ", compress_stream_xz, decompress_stream_xz); - - test_decompress_startswith_short("XZ", compress_blob_xz, decompress_startswith_xz); - -#else - log_info("/* XZ test skipped */"); -#endif - -#if HAVE_LZ4 - if (dlopen_lz4() >= 0) { - test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4, - text, sizeof(text), false); - test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4, - data, sizeof(data), true); - - test_decompress_startswith("LZ4", - compress_blob_lz4, decompress_startswith_lz4, - text, sizeof(text), false); - test_decompress_startswith("LZ4", - compress_blob_lz4, decompress_startswith_lz4, - data, sizeof(data), true); - test_decompress_startswith("LZ4", - compress_blob_lz4, decompress_startswith_lz4, - huge, HUGE_SIZE, true); - - test_compress_stream("LZ4", "lz4cat", - compress_stream_lz4, decompress_stream_lz4, srcfile); - - test_decompress_stream_sparse("LZ4", compress_stream_lz4, decompress_stream_lz4); - - test_lz4_decompress_partial(); - - test_decompress_startswith_short("LZ4", compress_blob_lz4, decompress_startswith_lz4); - } else - log_error("/* Can't load liblz4 */"); -#else - log_info("/* LZ4 test skipped */"); -#endif - -#if HAVE_ZSTD - test_compress_decompress("ZSTD", compress_blob_zstd, decompress_blob_zstd, - text, sizeof(text), false); - test_compress_decompress("ZSTD", compress_blob_zstd, decompress_blob_zstd, - data, sizeof(data), true); - - test_decompress_startswith("ZSTD", - compress_blob_zstd, decompress_startswith_zstd, - text, sizeof(text), false); - test_decompress_startswith("ZSTD", - compress_blob_zstd, decompress_startswith_zstd, - data, sizeof(data), true); - test_decompress_startswith("ZSTD", - compress_blob_zstd, decompress_startswith_zstd, - huge, HUGE_SIZE, true); - - test_compress_stream("ZSTD", "zstdcat", - compress_stream_zstd, decompress_stream_zstd, srcfile); - - test_decompress_stream_sparse("ZSTD", compress_stream_zstd, decompress_stream_zstd); - - test_decompress_startswith_short("ZSTD", compress_blob_zstd, decompress_startswith_zstd); -#else - log_info("/* ZSTD test skipped */"); -#endif - return 0; -#else - return log_tests_skipped("no compression algorithm supported"); -#endif } + +DEFINE_TEST_MAIN_WITH_INTRO(LOG_DEBUG, intro); diff --git a/src/test/test-dlopen-so.c b/src/test/test-dlopen-so.c index 4b80532698..89d2112630 100644 --- a/src/test/test-dlopen-so.c +++ b/src/test/test-dlopen-so.c @@ -42,6 +42,7 @@ static int run(int argc, char **argv) { * where .so versions change and distributions update, but systemd doesn't have the new so names * around yet. */ + ASSERT_DLOPEN(dlopen_bzip2, HAVE_BZIP2); ASSERT_DLOPEN(dlopen_bpf, HAVE_LIBBPF); ASSERT_DLOPEN(dlopen_cryptsetup, HAVE_LIBCRYPTSETUP); ASSERT_DLOPEN(dlopen_dw, HAVE_ELFUTILS); @@ -60,14 +61,15 @@ static int run(int argc, char **argv) { ASSERT_DLOPEN(dlopen_libpam, HAVE_PAM); ASSERT_DLOPEN(dlopen_libseccomp, HAVE_SECCOMP); ASSERT_DLOPEN(dlopen_libselinux, HAVE_SELINUX); + ASSERT_DLOPEN(dlopen_xz, HAVE_XZ); ASSERT_DLOPEN(dlopen_lz4, HAVE_LZ4); - ASSERT_DLOPEN(dlopen_lzma, HAVE_XZ); ASSERT_DLOPEN(dlopen_p11kit, HAVE_P11KIT); ASSERT_DLOPEN(dlopen_passwdqc, HAVE_PASSWDQC); ASSERT_DLOPEN(dlopen_pcre2, HAVE_PCRE2); ASSERT_DLOPEN(dlopen_pwquality, HAVE_PWQUALITY); ASSERT_DLOPEN(dlopen_qrencode, HAVE_QRENCODE); ASSERT_DLOPEN(dlopen_tpm2, HAVE_TPM2); + ASSERT_DLOPEN(dlopen_zlib, HAVE_ZLIB); ASSERT_DLOPEN(dlopen_zstd, HAVE_ZSTD); return 0; diff --git a/test/units/TEST-04-JOURNAL.SYSTEMD_JOURNAL_COMPRESS.sh b/test/units/TEST-04-JOURNAL.SYSTEMD_JOURNAL_COMPRESS.sh index 13ca3751cb..97782f9634 100755 --- a/test/units/TEST-04-JOURNAL.SYSTEMD_JOURNAL_COMPRESS.sh +++ b/test/units/TEST-04-JOURNAL.SYSTEMD_JOURNAL_COMPRESS.sh @@ -17,6 +17,13 @@ EOF systemctl reset-failed systemd-journald.service for c in NONE XZ LZ4 ZSTD; do + # compression_to_string() returns "uncompressed" for COMPRESSION_NONE + if [[ "${c}" == NONE ]]; then + log_name="uncompressed" + else + log_name="${c,,}" + fi + cat >/run/systemd/system/systemd-journald.service.d/compress.conf <&1 | grep -F 'compress=${c}' >/dev/null; do sleep .5; done" + timeout 10 bash -c "until SYSTEMD_LOG_LEVEL=debug journalctl --verify --quiet --file /var/log/journal/$MACHINE_ID/system.journal 2>&1 | grep -F 'compress=${log_name}' >/dev/null; do sleep .5; done" # $SYSTEMD_JOURNAL_COMPRESS= also works for journal-remote if [[ -x /usr/lib/systemd/systemd-journal-remote ]]; then for cc in NONE XZ LZ4 ZSTD; do + if [[ "${cc}" == NONE ]]; then + cc_log_name="uncompressed" + else + cc_log_name="${cc,,}" + fi + rm -f /tmp/foo.journal SYSTEMD_JOURNAL_COMPRESS="${cc}" /usr/lib/systemd/systemd-journal-remote --split-mode=none -o /tmp/foo.journal --getter="journalctl -b -o export -t $ID" - SYSTEMD_LOG_LEVEL=debug journalctl --verify --quiet --file /tmp/foo.journal 2>&1 | grep -F "compress=${cc}" >/dev/null + SYSTEMD_LOG_LEVEL=debug journalctl --verify --quiet --file /tmp/foo.journal 2>&1 | grep -F "compress=${cc_log_name}" >/dev/null journalctl -t "$ID" -o cat --file /tmp/foo.journal | grep -F "hoge with ${c}" >/dev/null done fi