compress: write sparse files when decompressing to regular files
Core dumps are often very sparse, containing large zero-filled regions whose actual disk usage can be significantly reduced by preserving holes. Previously, decompress_stream() always wrote dense output, expanding all zero regions into allocated disk blocks. Each decompression backend (xz, lz4, zstd) now auto-detects whether the output fd is suitable for sparse writes via a shared should_sparse() helper. The check requires both S_ISREG (regular file) and !O_APPEND, since O_APPEND causes write() to ignore the file position set by lseek(), which would collapse the holes and corrupt the output. For pipes, sockets, and append-mode files, dense writes are preserved via loop_write_full() with USEC_INFINITY timeout, matching the original behavior. After sparse decompression, finalize_sparse() sets the final file size to account for any trailing holes. This is transparent to callers — all public signatures are unchanged. coredumpctl benefits automatically: - coredumpctl debug: temp file in /var/tmp is now sparse - coredumpctl dump -o file: output file is now sparse - coredumpctl dump > file: redirected stdout is now sparse - coredumpctl dump | ...: pipe output unchanged (dense) - coredumpctl dump >> file: append mode, falls back to dense Co-developed-by: Claude Opus 4.6 <noreply@anthropic.com> Co-developed-by: Codex (GPT-5) <noreply@openai.com>
This commit is contained in:
+67
-6
@@ -1,5 +1,6 @@
|
||||
/* SPDX-License-Identifier: LGPL-2.1-or-later */
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <stdio.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/stat.h>
|
||||
@@ -951,11 +952,69 @@ int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_unco
|
||||
#endif
|
||||
}
|
||||
|
||||
#if HAVE_COMPRESSION
|
||||
/* Determine whether sparse writes should be used for this fd. Sparse writes are only safe on
|
||||
* regular files without O_APPEND (O_APPEND ignores lseek position, which would collapse holes). */
|
||||
static int should_sparse(int fd) {
|
||||
struct stat st;
|
||||
|
||||
assert(fd >= 0);
|
||||
|
||||
if (fstat(fd, &st) < 0)
|
||||
return -errno;
|
||||
|
||||
int flags = fcntl(fd, F_GETFL);
|
||||
if (flags < 0)
|
||||
return -errno;
|
||||
|
||||
return S_ISREG(st.st_mode) && !FLAGS_SET(flags, O_APPEND);
|
||||
}
|
||||
|
||||
/* After sparse decompression, set the file size to the current position to account for
|
||||
* trailing holes that sparse_write() created via lseek but never extended the file size for. */
|
||||
static int finalize_sparse(int fd) {
|
||||
off_t pos;
|
||||
|
||||
assert(fd >= 0);
|
||||
|
||||
pos = lseek(fd, 0, SEEK_CUR);
|
||||
if (pos < 0)
|
||||
return -errno;
|
||||
|
||||
if (ftruncate(fd, pos) < 0)
|
||||
return -errno;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int maybe_sparse_write(int fd, const void *buf, size_t nbytes, bool sparse) {
|
||||
int r;
|
||||
|
||||
if (sparse) {
|
||||
ssize_t k;
|
||||
|
||||
/* Note: sparse_write() does not retry on EINTR and converts short writes to -EIO.
|
||||
* This is fine here since sparse mode is only used on regular files, where short
|
||||
* writes and EINTR are not expected in practice. */
|
||||
k = sparse_write(fd, buf, nbytes, 64);
|
||||
if (k < 0)
|
||||
return (int) k;
|
||||
} else {
|
||||
r = loop_write_full(fd, buf, nbytes, USEC_INFINITY);
|
||||
if (r < 0)
|
||||
return r;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
|
||||
assert(fdf >= 0);
|
||||
assert(fdt >= 0);
|
||||
|
||||
#if HAVE_XZ
|
||||
bool sparse = should_sparse(fdt) > 0;
|
||||
int r;
|
||||
|
||||
r = dlopen_lzma();
|
||||
@@ -1009,7 +1068,7 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
|
||||
max_bytes -= n;
|
||||
}
|
||||
|
||||
k = loop_write(fdt, out, n);
|
||||
k = maybe_sparse_write(fdt, out, n, sparse);
|
||||
if (k < 0)
|
||||
return k;
|
||||
|
||||
@@ -1021,7 +1080,7 @@ int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
|
||||
s.total_in, s.total_out,
|
||||
(double) s.total_out / s.total_in * 100);
|
||||
|
||||
return 0;
|
||||
return sparse ? finalize_sparse(fdt) : 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1038,6 +1097,7 @@ int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
|
||||
_cleanup_free_ char *buf = NULL;
|
||||
char *src;
|
||||
struct stat st;
|
||||
bool sparse = should_sparse(fdt) > 0;
|
||||
int r;
|
||||
size_t total_in = 0, total_out = 0;
|
||||
|
||||
@@ -1082,7 +1142,7 @@ int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
r = loop_write(fdt, buf, produced);
|
||||
r = maybe_sparse_write(fdt, buf, produced, sparse);
|
||||
if (r < 0)
|
||||
goto cleanup;
|
||||
}
|
||||
@@ -1093,7 +1153,7 @@ int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
|
||||
log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
|
||||
total_in, total_out,
|
||||
(double) total_out / total_in * 100);
|
||||
r = 0;
|
||||
r = sparse ? finalize_sparse(fdt) : 0;
|
||||
cleanup:
|
||||
munmap(src, st.st_size);
|
||||
return r;
|
||||
@@ -1219,6 +1279,7 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
|
||||
#if HAVE_ZSTD
|
||||
_cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = NULL;
|
||||
_cleanup_free_ void *in_buff = NULL, *out_buff = NULL;
|
||||
bool sparse = should_sparse(fdt) > 0;
|
||||
size_t in_allocsize, out_allocsize;
|
||||
size_t last_result = 0;
|
||||
uint64_t left = max_bytes, in_bytes = 0;
|
||||
@@ -1290,7 +1351,7 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
|
||||
if (left < output.pos)
|
||||
return -EFBIG;
|
||||
|
||||
wrote = loop_write_full(fdt, output.dst, output.pos, USEC_INFINITY);
|
||||
wrote = maybe_sparse_write(fdt, output.dst, output.pos, sparse);
|
||||
if (wrote < 0)
|
||||
return wrote;
|
||||
|
||||
@@ -1319,7 +1380,7 @@ int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
|
||||
in_bytes,
|
||||
max_bytes - left,
|
||||
(double) (max_bytes - left) / in_bytes * 100);
|
||||
return 0;
|
||||
return sparse ? finalize_sparse(fdt) : 0;
|
||||
#else
|
||||
return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
|
||||
"Cannot decompress file. Compiled without ZSTD support.");
|
||||
|
||||
Reference in New Issue
Block a user