SymbX/systemd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

compress: write sparse files when decompressing to regular files

Browse Source 
Core dumps are often very sparse, containing large zero-filled regions
whose actual disk usage can be significantly reduced by preserving
holes. Previously, decompress_stream() always wrote dense output,
expanding all zero regions into allocated disk blocks.

Each decompression backend (xz, lz4, zstd) now auto-detects whether the
output fd is suitable for sparse writes via a shared should_sparse()
helper. The check requires both S_ISREG (regular file) and !O_APPEND,
since O_APPEND causes write() to ignore the file position set by
lseek(), which would collapse the holes and corrupt the output. For
pipes, sockets, and append-mode files, dense writes are preserved via
loop_write_full() with USEC_INFINITY timeout, matching the original
behavior. After sparse decompression, finalize_sparse() sets the final
file size to account for any trailing holes.

This is transparent to callers — all public signatures are unchanged.
coredumpctl benefits automatically:
- coredumpctl debug: temp file in /var/tmp is now sparse
- coredumpctl dump -o file: output file is now sparse
- coredumpctl dump > file: redirected stdout is now sparse
- coredumpctl dump | ...: pipe output unchanged (dense)
- coredumpctl dump >> file: append mode, falls back to dense

Co-developed-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-developed-by: Codex (GPT-5) <noreply@openai.com>
This commit is contained in:
noxiouz
2026-04-08 23:44:23 +01:00
committed by Luca Boccassi
parent bb93fdaaf3
commit 9bd72b612b
2 changed files with 220 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/basic/compress.c
+67 -6
View File
@@ -1,5 +1,6 @@
/* SPDX-License-Identifier: LGPL-2.1-or-later */
#include <fcntl.h>
#include <stdio.h>
#include <sys/mman.h>
#include <sys/stat.h>
@@ -951,11 +952,69 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco
#endif
}
#if HAVE_COMPRESSION
/* Determine whether sparse writes should be used for this fd. Sparse writes are only safe on
* regular files without O_APPEND (O_APPEND ignores lseek position, which would collapse holes). */
static int should_sparse(int fd) {
struct stat st;
assert(fd >= 0);
if (fstat(fd, &st) < 0)
return -errno;
int flags = fcntl(fd, F_GETFL);
if (flags < 0)
return -errno;
return S_ISREG(st.st_mode) && !FLAGS_SET(flags, O_APPEND);
}
/* After sparse decompression, set the file size to the current position to account for
* trailing holes that sparse_write() created via lseek but never extended the file size for. */
static int finalize_sparse(int fd) {
off_t pos;
assert(fd >= 0);
pos = lseek(fd, 0, SEEK_CUR);
if (pos < 0)
return -errno;
if (ftruncate(fd, pos) < 0)
return -errno;
return 0;
}
static int maybe_sparse_write(int fd, const void *buf, size_t nbytes, bool sparse) {
int r;
if (sparse) {
ssize_t k;
/* Note: sparse_write() does not retry on EINTR and converts short writes to -EIO.
* This is fine here since sparse mode is only used on regular files, where short
* writes and EINTR are not expected in practice. */
k = sparse_write(fd, buf, nbytes, 64);
if (k < 0)
return (int) k;
} else {
r = loop_write_full(fd, buf, nbytes, USEC_INFINITY);
if (r < 0)
return r;
}
return 0;
}
#endif
int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
assert(fdf >= 0);
assert(fdt >= 0);
#if HAVE_XZ
bool sparse = should_sparse(fdt) > 0;
int r;
r = dlopen_lzma();
@@ -1009,7 +1068,7 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
max_bytes -= n;
}
k = loop_write(fdt, out, n);
k = maybe_sparse_write(fdt, out, n, sparse);
if (k < 0)
return k;
@@ -1021,7 +1080,7 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
s.total_in, s.total_out,
(double) s.total_out / s.total_in * 100);
return 0;
return sparse ? finalize_sparse(fdt) : 0;
}
}
}
@@ -1038,6 +1097,7 @@ int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
_cleanup_free_ char *buf = NULL;
char *src;
struct stat st;
bool sparse = should_sparse(fdt) > 0;
int r;
size_t total_in = 0, total_out = 0;
@@ -1082,7 +1142,7 @@ int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
goto cleanup;
}
r = loop_write(fdt, buf, produced);
r = maybe_sparse_write(fdt, buf, produced, sparse);
if (r < 0)
goto cleanup;
}
@@ -1093,7 +1153,7 @@ int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
total_in, total_out,
(double) total_out / total_in * 100);
r = 0;
r = sparse ? finalize_sparse(fdt) : 0;
cleanup:
munmap(src, st.st_size);
return r;
@@ -1219,6 +1279,7 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
#if HAVE_ZSTD
_cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = NULL;
_cleanup_free_ void *in_buff = NULL, *out_buff = NULL;
bool sparse = should_sparse(fdt) > 0;
size_t in_allocsize, out_allocsize;
size_t last_result = 0;
uint64_t left = max_bytes, in_bytes = 0;
@@ -1290,7 +1351,7 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
if (left < output.pos)
return -EFBIG;
wrote = loop_write_full(fdt, output.dst, output.pos, USEC_INFINITY);
wrote = maybe_sparse_write(fdt, output.dst, output.pos, sparse);
if (wrote < 0)
return wrote;
@@ -1319,7 +1380,7 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
in_bytes,
max_bytes - left,
(double) (max_bytes - left) / in_bytes * 100);
return 0;
return sparse ? finalize_sparse(fdt) : 0;
#else
return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
"Cannot decompress file. Compiled without ZSTD support.");
src/test/test-compress.c
+153
View File
@@ -12,6 +12,7 @@
#include "compress.h"
#include "dlfcn-util.h"
#include "fd-util.h"
#include "io-util.h"
#include "path-util.h"
#include "random-util.h"
#include "tests.h"
@@ -225,6 +226,152 @@ _unused_ static void test_compress_stream(const char *compression,
r = decompress(dst, dst2, st.st_size - 1);
assert_se(r == -EFBIG);
}
_unused_ static void test_decompress_stream_sparse(const char *compression,
compress_stream_t compress,
decompress_stream_t decompress) {
_cleanup_close_ int src = -EBADF, compressed = -EBADF, decompressed = -EBADF;
_cleanup_(unlink_tempfilep) char
pattern_src[] = "/tmp/systemd-test.sparse-src.XXXXXX",
pattern_compressed[] = "/tmp/systemd-test.sparse-compressed.XXXXXX",
pattern_decompressed[] = "/tmp/systemd-test.sparse-decompressed.XXXXXX";
/* Create a sparse-like input: 4K of data, 64K of zeros, 4K of data, 64K trailing zeros.
* Total apparent size: 136K, but most of it is zeros. */
uint8_t data_block[4096];
struct stat st_src, st_decompressed;
uint64_t uncompressed_size;
int r;
assert(compression);
log_debug("/* testing %s sparse decompression */", compression);
random_bytes(data_block, sizeof(data_block));
assert_se((src = mkostemp_safe(pattern_src)) >= 0);
/* Write: 4K data, 64K zeros, 4K data, 64K zeros */
assert_se(loop_write(src, data_block, sizeof(data_block)) >= 0);
assert_se(ftruncate(src, sizeof(data_block) + 65536) >= 0);
assert_se(lseek(src, sizeof(data_block) + 65536, SEEK_SET) >= 0);
assert_se(loop_write(src, data_block, sizeof(data_block)) >= 0);
assert_se(ftruncate(src, 2 * sizeof(data_block) + 2 * 65536) >= 0);
assert_se(lseek(src, 0, SEEK_SET) == 0);
assert_se(fstat(src, &st_src) >= 0);
assert_se(st_src.st_size == 2 * (off_t) sizeof(data_block) + 2 * 65536);
/* Compress */
assert_se((compressed = mkostemp_safe(pattern_compressed)) >= 0);
ASSERT_OK(compress(src, compressed, -1, &uncompressed_size));
assert_se((uint64_t) st_src.st_size == uncompressed_size);
/* Decompress to a regular file (sparse writes auto-detected) */
assert_se((decompressed = mkostemp_safe(pattern_decompressed)) >= 0);
assert_se(lseek(compressed, 0, SEEK_SET) == 0);
r = decompress(compressed, decompressed, st_src.st_size);
assert_se(r == 0);
/* Verify apparent size matches */
assert_se(fstat(decompressed, &st_decompressed) >= 0);
assert_se(st_decompressed.st_size == st_src.st_size);
/* Verify content matches by comparing bytes */
assert_se(lseek(src, 0, SEEK_SET) == 0);
assert_se(lseek(decompressed, 0, SEEK_SET) == 0);
for (off_t offset = 0; offset < st_src.st_size;) {
uint8_t buf_src[4096], buf_dst[4096];
size_t to_read = MIN((size_t) (st_src.st_size - offset), sizeof(buf_src));
ssize_t n;
n = loop_read(src, buf_src, to_read, true);
assert_se(n == (ssize_t) to_read);
n = loop_read(decompressed, buf_dst, to_read, true);
assert_se(n == (ssize_t) to_read);
assert_se(memcmp(buf_src, buf_dst, to_read) == 0);
offset += to_read;
}
/* Verify the decompressed file is actually sparse (uses less disk than apparent size).
* st_blocks is in 512-byte units. The file has 128K of zeros, so disk usage should be
* noticeably less than the apparent size if sparse writes worked.
* Only assert if the filesystem supports holes (SEEK_HOLE). */
log_debug("%s sparse decompression: apparent=%jd disk=%jd",
compression,
(intmax_t) st_decompressed.st_size,
(intmax_t) st_decompressed.st_blocks * 512);
if (lseek(decompressed, 0, SEEK_HOLE) < st_decompressed.st_size)
assert_se(st_decompressed.st_blocks * 512 < st_decompressed.st_size);
else
log_debug("Filesystem does not support holes, skipping sparsity check");
/* Test all-zeros input: entire output should be a hole */
log_debug("/* testing %s sparse decompression of all-zeros */", compression);
{
_cleanup_close_ int zsrc = -EBADF, zcompressed = -EBADF, zdecompressed = -EBADF;
_cleanup_(unlink_tempfilep) char
zp_src[] = "/tmp/systemd-test.sparse-zero-src.XXXXXX",
zp_compressed[] = "/tmp/systemd-test.sparse-zero-compressed.XXXXXX",
zp_decompressed[] = "/tmp/systemd-test.sparse-zero-decompressed.XXXXXX";
struct stat zst;
uint64_t zsize;
uint8_t zeros[65536] = {};
assert_se((zsrc = mkostemp_safe(zp_src)) >= 0);
assert_se(loop_write(zsrc, zeros, sizeof(zeros)) >= 0);
assert_se(lseek(zsrc, 0, SEEK_SET) == 0);
assert_se((zcompressed = mkostemp_safe(zp_compressed)) >= 0);
ASSERT_OK(compress(zsrc, zcompressed, -1, &zsize));
assert_se(zsize == sizeof(zeros));
assert_se((zdecompressed = mkostemp_safe(zp_decompressed)) >= 0);
assert_se(lseek(zcompressed, 0, SEEK_SET) == 0);
assert_se(decompress(zcompressed, zdecompressed, sizeof(zeros)) == 0);
assert_se(fstat(zdecompressed, &zst) >= 0);
assert_se(zst.st_size == (off_t) sizeof(zeros));
/* All zeros — disk usage should be minimal */
log_debug("%s all-zeros sparse: apparent=%jd disk=%jd",
compression, (intmax_t) zst.st_size, (intmax_t) zst.st_blocks * 512);
if (lseek(zdecompressed, 0, SEEK_HOLE) < zst.st_size)
assert_se(zst.st_blocks * 512 < zst.st_size);
else
log_debug("Filesystem does not support holes, skipping sparsity check");
}
/* Test data ending with non-zero bytes: ftruncate should be a no-op */
log_debug("/* testing %s sparse decompression ending with data */", compression);
{
_cleanup_close_ int dsrc = -EBADF, dcompressed = -EBADF, ddecompressed = -EBADF;
_cleanup_(unlink_tempfilep) char
dp_src[] = "/tmp/systemd-test.sparse-end-src.XXXXXX",
dp_compressed[] = "/tmp/systemd-test.sparse-end-compressed.XXXXXX",
dp_decompressed[] = "/tmp/systemd-test.sparse-end-decompressed.XXXXXX";
struct stat dst;
uint64_t dsize;
uint8_t zeros[65536] = {};
/* 64K zeros followed by 4K random data */
assert_se((dsrc = mkostemp_safe(dp_src)) >= 0);
assert_se(loop_write(dsrc, zeros, sizeof(zeros)) >= 0);
assert_se(loop_write(dsrc, data_block, sizeof(data_block)) >= 0);
assert_se(lseek(dsrc, 0, SEEK_SET) == 0);
assert_se((dcompressed = mkostemp_safe(dp_compressed)) >= 0);
ASSERT_OK(compress(dsrc, dcompressed, -1, &dsize));
assert_se(dsize == sizeof(zeros) + sizeof(data_block));
assert_se((ddecompressed = mkostemp_safe(dp_decompressed)) >= 0);
assert_se(lseek(dcompressed, 0, SEEK_SET) == 0);
assert_se(decompress(dcompressed, ddecompressed, dsize) == 0);
assert_se(fstat(ddecompressed, &dst) >= 0);
assert_se(dst.st_size == (off_t)(sizeof(zeros) + sizeof(data_block)));
}
}
#endif
#if HAVE_LZ4
@@ -314,6 +461,8 @@ int main(int argc, char *argv[]) {
test_compress_stream("XZ", "xzcat",
compress_stream_xz, decompress_stream_xz, srcfile);
test_decompress_stream_sparse("XZ", compress_stream_xz, decompress_stream_xz);
test_decompress_startswith_short("XZ", compress_blob_xz, decompress_startswith_xz);
#else
@@ -340,6 +489,8 @@ int main(int argc, char *argv[]) {
test_compress_stream("LZ4", "lz4cat",
compress_stream_lz4, decompress_stream_lz4, srcfile);
test_decompress_stream_sparse("LZ4", compress_stream_lz4, decompress_stream_lz4);
test_lz4_decompress_partial();
test_decompress_startswith_short("LZ4", compress_blob_lz4, decompress_startswith_lz4);
@@ -368,6 +519,8 @@ int main(int argc, char *argv[]) {
test_compress_stream("ZSTD", "zstdcat",
compress_stream_zstd, decompress_stream_zstd, srcfile);
test_decompress_stream_sparse("ZSTD", compress_stream_zstd, decompress_stream_zstd);
test_decompress_startswith_short("ZSTD", compress_blob_zstd, decompress_startswith_zstd);
#else
log_info("/* ZSTD test skipped */");