From 9bd72b612b76fab62ff6275c48ec19ced918e662 Mon Sep 17 00:00:00 2001 From: noxiouz Date: Tue, 7 Apr 2026 21:32:52 +0100 Subject: [PATCH] compress: write sparse files when decompressing to regular files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Core dumps are often very sparse, containing large zero-filled regions whose actual disk usage can be significantly reduced by preserving holes. Previously, decompress_stream() always wrote dense output, expanding all zero regions into allocated disk blocks. Each decompression backend (xz, lz4, zstd) now auto-detects whether the output fd is suitable for sparse writes via a shared should_sparse() helper. The check requires both S_ISREG (regular file) and !O_APPEND, since O_APPEND causes write() to ignore the file position set by lseek(), which would collapse the holes and corrupt the output. For pipes, sockets, and append-mode files, dense writes are preserved via loop_write_full() with USEC_INFINITY timeout, matching the original behavior. After sparse decompression, finalize_sparse() sets the final file size to account for any trailing holes. This is transparent to callers — all public signatures are unchanged. coredumpctl benefits automatically: - coredumpctl debug: temp file in /var/tmp is now sparse - coredumpctl dump -o file: output file is now sparse - coredumpctl dump > file: redirected stdout is now sparse - coredumpctl dump | ...: pipe output unchanged (dense) - coredumpctl dump >> file: append mode, falls back to dense Co-developed-by: Claude Opus 4.6 Co-developed-by: Codex (GPT-5) --- src/basic/compress.c | 73 +++++++++++++++++-- src/test/test-compress.c | 153 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 220 insertions(+), 6 deletions(-) diff --git a/src/basic/compress.c b/src/basic/compress.c index d9759ad417..5f00f968a2 100644 --- a/src/basic/compress.c +++ b/src/basic/compress.c @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ +#include #include #include #include @@ -951,11 +952,69 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco #endif } +#if HAVE_COMPRESSION +/* Determine whether sparse writes should be used for this fd. Sparse writes are only safe on + * regular files without O_APPEND (O_APPEND ignores lseek position, which would collapse holes). */ +static int should_sparse(int fd) { + struct stat st; + + assert(fd >= 0); + + if (fstat(fd, &st) < 0) + return -errno; + + int flags = fcntl(fd, F_GETFL); + if (flags < 0) + return -errno; + + return S_ISREG(st.st_mode) && !FLAGS_SET(flags, O_APPEND); +} + +/* After sparse decompression, set the file size to the current position to account for + * trailing holes that sparse_write() created via lseek but never extended the file size for. */ +static int finalize_sparse(int fd) { + off_t pos; + + assert(fd >= 0); + + pos = lseek(fd, 0, SEEK_CUR); + if (pos < 0) + return -errno; + + if (ftruncate(fd, pos) < 0) + return -errno; + + return 0; +} + +static int maybe_sparse_write(int fd, const void *buf, size_t nbytes, bool sparse) { + int r; + + if (sparse) { + ssize_t k; + + /* Note: sparse_write() does not retry on EINTR and converts short writes to -EIO. + * This is fine here since sparse mode is only used on regular files, where short + * writes and EINTR are not expected in practice. */ + k = sparse_write(fd, buf, nbytes, 64); + if (k < 0) + return (int) k; + } else { + r = loop_write_full(fd, buf, nbytes, USEC_INFINITY); + if (r < 0) + return r; + } + + return 0; +} +#endif + int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) { assert(fdf >= 0); assert(fdt >= 0); #if HAVE_XZ + bool sparse = should_sparse(fdt) > 0; int r; r = dlopen_lzma(); @@ -1009,7 +1068,7 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) { max_bytes -= n; } - k = loop_write(fdt, out, n); + k = maybe_sparse_write(fdt, out, n, sparse); if (k < 0) return k; @@ -1021,7 +1080,7 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) { s.total_in, s.total_out, (double) s.total_out / s.total_in * 100); - return 0; + return sparse ? finalize_sparse(fdt) : 0; } } } @@ -1038,6 +1097,7 @@ int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) { _cleanup_free_ char *buf = NULL; char *src; struct stat st; + bool sparse = should_sparse(fdt) > 0; int r; size_t total_in = 0, total_out = 0; @@ -1082,7 +1142,7 @@ int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) { goto cleanup; } - r = loop_write(fdt, buf, produced); + r = maybe_sparse_write(fdt, buf, produced, sparse); if (r < 0) goto cleanup; } @@ -1093,7 +1153,7 @@ int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) { log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)", total_in, total_out, (double) total_out / total_in * 100); - r = 0; + r = sparse ? finalize_sparse(fdt) : 0; cleanup: munmap(src, st.st_size); return r; @@ -1219,6 +1279,7 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) { #if HAVE_ZSTD _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = NULL; _cleanup_free_ void *in_buff = NULL, *out_buff = NULL; + bool sparse = should_sparse(fdt) > 0; size_t in_allocsize, out_allocsize; size_t last_result = 0; uint64_t left = max_bytes, in_bytes = 0; @@ -1290,7 +1351,7 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) { if (left < output.pos) return -EFBIG; - wrote = loop_write_full(fdt, output.dst, output.pos, USEC_INFINITY); + wrote = maybe_sparse_write(fdt, output.dst, output.pos, sparse); if (wrote < 0) return wrote; @@ -1319,7 +1380,7 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) { in_bytes, max_bytes - left, (double) (max_bytes - left) / in_bytes * 100); - return 0; + return sparse ? finalize_sparse(fdt) : 0; #else return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT), "Cannot decompress file. Compiled without ZSTD support."); diff --git a/src/test/test-compress.c b/src/test/test-compress.c index 7b12e88adf..80f2923dd6 100644 --- a/src/test/test-compress.c +++ b/src/test/test-compress.c @@ -12,6 +12,7 @@ #include "compress.h" #include "dlfcn-util.h" #include "fd-util.h" +#include "io-util.h" #include "path-util.h" #include "random-util.h" #include "tests.h" @@ -225,6 +226,152 @@ _unused_ static void test_compress_stream(const char *compression, r = decompress(dst, dst2, st.st_size - 1); assert_se(r == -EFBIG); } + +_unused_ static void test_decompress_stream_sparse(const char *compression, + compress_stream_t compress, + decompress_stream_t decompress) { + + _cleanup_close_ int src = -EBADF, compressed = -EBADF, decompressed = -EBADF; + _cleanup_(unlink_tempfilep) char + pattern_src[] = "/tmp/systemd-test.sparse-src.XXXXXX", + pattern_compressed[] = "/tmp/systemd-test.sparse-compressed.XXXXXX", + pattern_decompressed[] = "/tmp/systemd-test.sparse-decompressed.XXXXXX"; + /* Create a sparse-like input: 4K of data, 64K of zeros, 4K of data, 64K trailing zeros. + * Total apparent size: 136K, but most of it is zeros. */ + uint8_t data_block[4096]; + struct stat st_src, st_decompressed; + uint64_t uncompressed_size; + int r; + + assert(compression); + + log_debug("/* testing %s sparse decompression */", compression); + + random_bytes(data_block, sizeof(data_block)); + + assert_se((src = mkostemp_safe(pattern_src)) >= 0); + + /* Write: 4K data, 64K zeros, 4K data, 64K zeros */ + assert_se(loop_write(src, data_block, sizeof(data_block)) >= 0); + assert_se(ftruncate(src, sizeof(data_block) + 65536) >= 0); + assert_se(lseek(src, sizeof(data_block) + 65536, SEEK_SET) >= 0); + assert_se(loop_write(src, data_block, sizeof(data_block)) >= 0); + assert_se(ftruncate(src, 2 * sizeof(data_block) + 2 * 65536) >= 0); + assert_se(lseek(src, 0, SEEK_SET) == 0); + + assert_se(fstat(src, &st_src) >= 0); + assert_se(st_src.st_size == 2 * (off_t) sizeof(data_block) + 2 * 65536); + + /* Compress */ + assert_se((compressed = mkostemp_safe(pattern_compressed)) >= 0); + ASSERT_OK(compress(src, compressed, -1, &uncompressed_size)); + assert_se((uint64_t) st_src.st_size == uncompressed_size); + + /* Decompress to a regular file (sparse writes auto-detected) */ + assert_se((decompressed = mkostemp_safe(pattern_decompressed)) >= 0); + assert_se(lseek(compressed, 0, SEEK_SET) == 0); + r = decompress(compressed, decompressed, st_src.st_size); + assert_se(r == 0); + + /* Verify apparent size matches */ + assert_se(fstat(decompressed, &st_decompressed) >= 0); + assert_se(st_decompressed.st_size == st_src.st_size); + + /* Verify content matches by comparing bytes */ + assert_se(lseek(src, 0, SEEK_SET) == 0); + assert_se(lseek(decompressed, 0, SEEK_SET) == 0); + + for (off_t offset = 0; offset < st_src.st_size;) { + uint8_t buf_src[4096], buf_dst[4096]; + size_t to_read = MIN((size_t) (st_src.st_size - offset), sizeof(buf_src)); + ssize_t n; + + n = loop_read(src, buf_src, to_read, true); + assert_se(n == (ssize_t) to_read); + n = loop_read(decompressed, buf_dst, to_read, true); + assert_se(n == (ssize_t) to_read); + assert_se(memcmp(buf_src, buf_dst, to_read) == 0); + offset += to_read; + } + + /* Verify the decompressed file is actually sparse (uses less disk than apparent size). + * st_blocks is in 512-byte units. The file has 128K of zeros, so disk usage should be + * noticeably less than the apparent size if sparse writes worked. + * Only assert if the filesystem supports holes (SEEK_HOLE). */ + log_debug("%s sparse decompression: apparent=%jd disk=%jd", + compression, + (intmax_t) st_decompressed.st_size, + (intmax_t) st_decompressed.st_blocks * 512); + if (lseek(decompressed, 0, SEEK_HOLE) < st_decompressed.st_size) + assert_se(st_decompressed.st_blocks * 512 < st_decompressed.st_size); + else + log_debug("Filesystem does not support holes, skipping sparsity check"); + + /* Test all-zeros input: entire output should be a hole */ + log_debug("/* testing %s sparse decompression of all-zeros */", compression); + { + _cleanup_close_ int zsrc = -EBADF, zcompressed = -EBADF, zdecompressed = -EBADF; + _cleanup_(unlink_tempfilep) char + zp_src[] = "/tmp/systemd-test.sparse-zero-src.XXXXXX", + zp_compressed[] = "/tmp/systemd-test.sparse-zero-compressed.XXXXXX", + zp_decompressed[] = "/tmp/systemd-test.sparse-zero-decompressed.XXXXXX"; + struct stat zst; + uint64_t zsize; + uint8_t zeros[65536] = {}; + + assert_se((zsrc = mkostemp_safe(zp_src)) >= 0); + assert_se(loop_write(zsrc, zeros, sizeof(zeros)) >= 0); + assert_se(lseek(zsrc, 0, SEEK_SET) == 0); + + assert_se((zcompressed = mkostemp_safe(zp_compressed)) >= 0); + ASSERT_OK(compress(zsrc, zcompressed, -1, &zsize)); + assert_se(zsize == sizeof(zeros)); + + assert_se((zdecompressed = mkostemp_safe(zp_decompressed)) >= 0); + assert_se(lseek(zcompressed, 0, SEEK_SET) == 0); + assert_se(decompress(zcompressed, zdecompressed, sizeof(zeros)) == 0); + + assert_se(fstat(zdecompressed, &zst) >= 0); + assert_se(zst.st_size == (off_t) sizeof(zeros)); + /* All zeros — disk usage should be minimal */ + log_debug("%s all-zeros sparse: apparent=%jd disk=%jd", + compression, (intmax_t) zst.st_size, (intmax_t) zst.st_blocks * 512); + if (lseek(zdecompressed, 0, SEEK_HOLE) < zst.st_size) + assert_se(zst.st_blocks * 512 < zst.st_size); + else + log_debug("Filesystem does not support holes, skipping sparsity check"); + } + + /* Test data ending with non-zero bytes: ftruncate should be a no-op */ + log_debug("/* testing %s sparse decompression ending with data */", compression); + { + _cleanup_close_ int dsrc = -EBADF, dcompressed = -EBADF, ddecompressed = -EBADF; + _cleanup_(unlink_tempfilep) char + dp_src[] = "/tmp/systemd-test.sparse-end-src.XXXXXX", + dp_compressed[] = "/tmp/systemd-test.sparse-end-compressed.XXXXXX", + dp_decompressed[] = "/tmp/systemd-test.sparse-end-decompressed.XXXXXX"; + struct stat dst; + uint64_t dsize; + uint8_t zeros[65536] = {}; + + /* 64K zeros followed by 4K random data */ + assert_se((dsrc = mkostemp_safe(dp_src)) >= 0); + assert_se(loop_write(dsrc, zeros, sizeof(zeros)) >= 0); + assert_se(loop_write(dsrc, data_block, sizeof(data_block)) >= 0); + assert_se(lseek(dsrc, 0, SEEK_SET) == 0); + + assert_se((dcompressed = mkostemp_safe(dp_compressed)) >= 0); + ASSERT_OK(compress(dsrc, dcompressed, -1, &dsize)); + assert_se(dsize == sizeof(zeros) + sizeof(data_block)); + + assert_se((ddecompressed = mkostemp_safe(dp_decompressed)) >= 0); + assert_se(lseek(dcompressed, 0, SEEK_SET) == 0); + assert_se(decompress(dcompressed, ddecompressed, dsize) == 0); + + assert_se(fstat(ddecompressed, &dst) >= 0); + assert_se(dst.st_size == (off_t)(sizeof(zeros) + sizeof(data_block))); + } +} #endif #if HAVE_LZ4 @@ -314,6 +461,8 @@ int main(int argc, char *argv[]) { test_compress_stream("XZ", "xzcat", compress_stream_xz, decompress_stream_xz, srcfile); + test_decompress_stream_sparse("XZ", compress_stream_xz, decompress_stream_xz); + test_decompress_startswith_short("XZ", compress_blob_xz, decompress_startswith_xz); #else @@ -340,6 +489,8 @@ int main(int argc, char *argv[]) { test_compress_stream("LZ4", "lz4cat", compress_stream_lz4, decompress_stream_lz4, srcfile); + test_decompress_stream_sparse("LZ4", compress_stream_lz4, decompress_stream_lz4); + test_lz4_decompress_partial(); test_decompress_startswith_short("LZ4", compress_blob_lz4, decompress_startswith_lz4); @@ -368,6 +519,8 @@ int main(int argc, char *argv[]) { test_compress_stream("ZSTD", "zstdcat", compress_stream_zstd, decompress_stream_zstd, srcfile); + test_decompress_stream_sparse("ZSTD", compress_stream_zstd, decompress_stream_zstd); + test_decompress_startswith_short("ZSTD", compress_blob_zstd, decompress_startswith_zstd); #else log_info("/* ZSTD test skipped */");