compress: write sparse files when decompressing to regular files

Core dumps are often very sparse, containing large zero-filled regions
whose actual disk usage can be significantly reduced by preserving
holes. Previously, decompress_stream() always wrote dense output,
expanding all zero regions into allocated disk blocks.

Each decompression backend (xz, lz4, zstd) now auto-detects whether the
output fd is suitable for sparse writes via a shared should_sparse()
helper. The check requires both S_ISREG (regular file) and !O_APPEND,
since O_APPEND causes write() to ignore the file position set by
lseek(), which would collapse the holes and corrupt the output. For
pipes, sockets, and append-mode files, dense writes are preserved via
loop_write_full() with USEC_INFINITY timeout, matching the original
behavior. After sparse decompression, finalize_sparse() sets the final
file size to account for any trailing holes.

This is transparent to callers — all public signatures are unchanged.
coredumpctl benefits automatically:
- coredumpctl debug: temp file in /var/tmp is now sparse
- coredumpctl dump -o file: output file is now sparse
- coredumpctl dump > file: redirected stdout is now sparse
- coredumpctl dump | ...: pipe output unchanged (dense)
- coredumpctl dump >> file: append mode, falls back to dense

Co-developed-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-developed-by: Codex (GPT-5) <noreply@openai.com>
This commit is contained in:
noxiouz
2026-04-08 23:44:23 +01:00
committed by Luca Boccassi
parent bb93fdaaf3
commit 9bd72b612b
2 changed files with 220 additions and 6 deletions
+67 -6
View File
@@ -1,5 +1,6 @@
/* SPDX-License-Identifier: LGPL-2.1-or-later */
#include <fcntl.h>
#include <stdio.h>
#include <sys/mman.h>
#include <sys/stat.h>
@@ -951,11 +952,69 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco
#endif
}
#if HAVE_COMPRESSION
/* Determine whether sparse writes should be used for this fd. Sparse writes are only safe on
* regular files without O_APPEND (O_APPEND ignores lseek position, which would collapse holes). */
static int should_sparse(int fd) {
struct stat st;
assert(fd >= 0);
if (fstat(fd, &st) < 0)
return -errno;
int flags = fcntl(fd, F_GETFL);
if (flags < 0)
return -errno;
return S_ISREG(st.st_mode) && !FLAGS_SET(flags, O_APPEND);
}
/* After sparse decompression, set the file size to the current position to account for
* trailing holes that sparse_write() created via lseek but never extended the file size for. */
static int finalize_sparse(int fd) {
off_t pos;
assert(fd >= 0);
pos = lseek(fd, 0, SEEK_CUR);
if (pos < 0)
return -errno;
if (ftruncate(fd, pos) < 0)
return -errno;
return 0;
}
static int maybe_sparse_write(int fd, const void *buf, size_t nbytes, bool sparse) {
int r;
if (sparse) {
ssize_t k;
/* Note: sparse_write() does not retry on EINTR and converts short writes to -EIO.
* This is fine here since sparse mode is only used on regular files, where short
* writes and EINTR are not expected in practice. */
k = sparse_write(fd, buf, nbytes, 64);
if (k < 0)
return (int) k;
} else {
r = loop_write_full(fd, buf, nbytes, USEC_INFINITY);
if (r < 0)
return r;
}
return 0;
}
#endif
int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
assert(fdf >= 0);
assert(fdt >= 0);
#if HAVE_XZ
bool sparse = should_sparse(fdt) > 0;
int r;
r = dlopen_lzma();
@@ -1009,7 +1068,7 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
max_bytes -= n;
}
k = loop_write(fdt, out, n);
k = maybe_sparse_write(fdt, out, n, sparse);
if (k < 0)
return k;
@@ -1021,7 +1080,7 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
s.total_in, s.total_out,
(double) s.total_out / s.total_in * 100);
return 0;
return sparse ? finalize_sparse(fdt) : 0;
}
}
}
@@ -1038,6 +1097,7 @@ int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
_cleanup_free_ char *buf = NULL;
char *src;
struct stat st;
bool sparse = should_sparse(fdt) > 0;
int r;
size_t total_in = 0, total_out = 0;
@@ -1082,7 +1142,7 @@ int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
goto cleanup;
}
r = loop_write(fdt, buf, produced);
r = maybe_sparse_write(fdt, buf, produced, sparse);
if (r < 0)
goto cleanup;
}
@@ -1093,7 +1153,7 @@ int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
total_in, total_out,
(double) total_out / total_in * 100);
r = 0;
r = sparse ? finalize_sparse(fdt) : 0;
cleanup:
munmap(src, st.st_size);
return r;
@@ -1219,6 +1279,7 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
#if HAVE_ZSTD
_cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = NULL;
_cleanup_free_ void *in_buff = NULL, *out_buff = NULL;
bool sparse = should_sparse(fdt) > 0;
size_t in_allocsize, out_allocsize;
size_t last_result = 0;
uint64_t left = max_bytes, in_bytes = 0;
@@ -1290,7 +1351,7 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
if (left < output.pos)
return -EFBIG;
wrote = loop_write_full(fdt, output.dst, output.pos, USEC_INFINITY);
wrote = maybe_sparse_write(fdt, output.dst, output.pos, sparse);
if (wrote < 0)
return wrote;
@@ -1319,7 +1380,7 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
in_bytes,
max_bytes - left,
(double) (max_bytes - left) / in_bytes * 100);
return 0;
return sparse ? finalize_sparse(fdt) : 0;
#else
return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
"Cannot decompress file. Compiled without ZSTD support.");