compress: consolidate all compression into compress.c with dlopen

Move the push-based streaming compression API from import-compress.c
into compress.c and delete import-compress.c/h. This consolidates all
compression code in one place and makes all compression libraries
(liblzma, liblz4, libzstd, libz, libbz2) runtime-loaded via dlopen
instead of directly linked.

Introduce opaque Compressor/Decompressor types backed by a heap-
allocated struct defined only in compress.c, keeping all third-party
library headers out of compress.h.

Rewrite the per-codec fd-to-fd stream functions as thin wrappers around
the push API via generic compress_stream()/decompress_stream() taking a
Compression type parameter. Integrate LZ4 into this framework using the
LZ4 Frame API, eliminating all LZ4 special-casing.

Extend the Compression enum with COMPRESSION_GZIP and COMPRESSION_BZIP2
and add the corresponding blob, startswith, and stream functions for
both.

Rename the ImportCompress types and functions: ImportCompressType becomes
the existing Compression enum, ImportCompress becomes Compressor (with
Decompressor typedef), and all import_compress_*/import_uncompress_*
become compressor_*/decompressor_*. Rename dlopen_lzma() to dlopen_xz()
for consistency. Make compression_to_string() return lowercase by
default.

Add INT_MAX/UINT_MAX overflow checks for LZ4, zlib, and bzip2 blob
functions where the codec API uses narrower integer types than our
uint64_t parameters.

Migrate test-compress.c and test-compress-benchmark.c to the TEST()
macro framework, new assertion macros, and codec-generic loops instead
of per-codec duplication.

Co-developed-by: Claude Opus 4.6 <noreply@anthropic.com>

src/basic/compress.h

@@ -8,103 +8,81 @@ typedef enum Compression {
         COMPRESSION_XZ,
         COMPRESSION_LZ4,
         COMPRESSION_ZSTD,
+        COMPRESSION_GZIP,
+        COMPRESSION_BZIP2,
         _COMPRESSION_MAX,
         _COMPRESSION_INVALID = -EINVAL,
 } Compression;
 
 DECLARE_STRING_TABLE_LOOKUP(compression, Compression);
-DECLARE_STRING_TABLE_LOOKUP(compression_lowercase, Compression);
+DECLARE_STRING_TABLE_LOOKUP(compression_uppercase, Compression);
+DECLARE_STRING_TABLE_LOOKUP(compression_extension, Compression);
+
+/* Try the lowercase string table first, fall back to the uppercase one. Useful for parsing user input
+ * where both forms (e.g. "xz" and "XZ") have historically been accepted. */
+Compression compression_from_string_harder(const char *s);
+
+/* Derives the compression type from a filename's extension, defaulting to COMPRESSION_NONE if the
+ * filename does not carry a recognized compression suffix. */
+Compression compression_from_filename(const char *filename);
 
 bool compression_supported(Compression c);
 
-int compress_blob_xz(const void *src, uint64_t src_size,
-                     void *dst, size_t dst_alloc_size, size_t *dst_size, int level);
-int compress_blob_lz4(const void *src, uint64_t src_size,
-                      void *dst, size_t dst_alloc_size, size_t *dst_size, int level);
-int compress_blob_zstd(const void *src, uint64_t src_size,
-                       void *dst, size_t dst_alloc_size, size_t *dst_size, int level);
+/* Buffer size used by streaming compression APIs and pipeline stages that feed into them. Sized to
+ * match the typical Linux pipe buffer so that pipeline stages don't lose throughput due to small
+ * intermediate buffers. */
+#define COMPRESS_PIPE_BUFFER_SIZE (128U*1024U)
 
-int decompress_blob_xz(const void *src, uint64_t src_size,
-                       void **dst, size_t* dst_size, size_t dst_max);
-int decompress_blob_lz4(const void *src, uint64_t src_size,
-                        void **dst, size_t* dst_size, size_t dst_max);
-int decompress_blob_zstd(const void *src, uint64_t src_size,
-                        void **dst, size_t* dst_size, size_t dst_max);
+/* Compressor / Decompressor — opaque push-based streaming compression context */
+
+typedef struct Compressor Compressor;
+typedef Compressor Decompressor;
+
+typedef int (*DecompressorCallback)(const void *data, size_t size, void *userdata);
+
+Compressor* compressor_free(Compressor *c);
+DEFINE_TRIVIAL_CLEANUP_FUNC(Compressor*, compressor_free);
+
+int compressor_new(Compressor **ret, Compression type);
+int compressor_start(Compressor *c, const void *data, size_t size, void **buffer, size_t *buffer_size, size_t *buffer_allocated);
+int compressor_finish(Compressor *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated);
+
+int decompressor_detect(Decompressor **ret, const void *data, size_t size);
+int decompressor_force_off(Decompressor **ret);
+int decompressor_push(Decompressor *c, const void *data, size_t size, DecompressorCallback callback, void *userdata);
+
+Compression compressor_type(const Compressor *c);
+
+/* Blob compression/decompression */
+
+int compress_blob(Compression compression,
+                  const void *src, uint64_t src_size,
+                  void *dst, size_t dst_alloc_size, size_t *dst_size, int level);
 int decompress_blob(Compression compression,
                     const void *src, uint64_t src_size,
-                    void **dst, size_t* dst_size, size_t dst_max);
+                    void **dst, size_t *dst_size, size_t dst_max);
+
+int decompress_zlib_raw(const void *src, uint64_t src_size,
+                        void *dst, size_t dst_size, int wbits);
 
-int decompress_startswith_xz(const void *src, uint64_t src_size,
-                             void **buffer,
-                             const void *prefix, size_t prefix_len,
-                             uint8_t extra);
-int decompress_startswith_lz4(const void *src, uint64_t src_size,
-                              void **buffer,
-                              const void *prefix, size_t prefix_len,
-                              uint8_t extra);
-int decompress_startswith_zstd(const void *src, uint64_t src_size,
-                               void **buffer,
-                               const void *prefix, size_t prefix_len,
-                               uint8_t extra);
 int decompress_startswith(Compression compression,
                           const void *src, uint64_t src_size,
                           void **buffer,
                           const void *prefix, size_t prefix_len,
                           uint8_t extra);
 
-int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size);
-int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size);
-int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size);
+/* Stream compression/decompression (fd-to-fd) */
 
-int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes);
-int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes);
-int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes);
+int compress_stream(Compression type, int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size);
+int decompress_stream(Compression type, int fdf, int fdt, uint64_t max_bytes);
+int decompress_stream_by_filename(const char *filename, int fdf, int fdt, uint64_t max_bytes);
 
+int dlopen_xz(void);
 int dlopen_lz4(void);
 int dlopen_zstd(void);
-int dlopen_lzma(void);
-
-static inline int compress_blob(
-                Compression compression,
-                const void *src, uint64_t src_size,
-                void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
-
-        switch (compression) {
-        case COMPRESSION_ZSTD:
-                return compress_blob_zstd(src, src_size, dst, dst_alloc_size, dst_size, level);
-        case COMPRESSION_LZ4:
-                return compress_blob_lz4(src, src_size, dst, dst_alloc_size, dst_size, level);
-        case COMPRESSION_XZ:
-                return compress_blob_xz(src, src_size, dst, dst_alloc_size, dst_size, level);
-        default:
-                return -EOPNOTSUPP;
-        }
-}
-
-static inline int compress_stream(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) {
-        switch (DEFAULT_COMPRESSION) {
-        case COMPRESSION_ZSTD:
-                return compress_stream_zstd(fdf, fdt, max_bytes, ret_uncompressed_size);
-        case COMPRESSION_LZ4:
-                return compress_stream_lz4(fdf, fdt, max_bytes, ret_uncompressed_size);
-        case COMPRESSION_XZ:
-                return compress_stream_xz(fdf, fdt, max_bytes, ret_uncompressed_size);
-        default:
-                return -EOPNOTSUPP;
-        }
-}
+int dlopen_zlib(void);
+int dlopen_bzip2(void);
 
 static inline const char* default_compression_extension(void) {
-        switch (DEFAULT_COMPRESSION) {
-        case COMPRESSION_ZSTD:
-                return ".zst";
-        case COMPRESSION_LZ4:
-                return ".lz4";
-        case COMPRESSION_XZ:
-                return ".xz";
-        default:
-                return "";
-        }
+        return compression_extension_to_string(DEFAULT_COMPRESSION) ?: "";
 }
-
-int decompress_stream(const char *filename, int fdf, int fdt, uint64_t max_bytes);

src/basic/meson.build

@@ -211,12 +211,14 @@ libbasic_static = static_library(
         fundamental_sources,
         include_directories : basic_includes,
         implicit_include_directories : false,
-        dependencies : [libdl,
+        dependencies : [libbzip2_cflags,
+                        libdl,
                         libgcrypt_cflags,
                         liblz4_cflags,
                         libm,
                         librt,
                         libxz_cflags,
+                        libz_cflags,
                         libzstd_cflags,
                         threads,
                         userspace],