From 326ab6d43029b6aaf46ab4a246aa0e43022a276d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20L=C3=BCke?= Date: Fri, 15 May 2026 23:49:44 +0900 Subject: [PATCH 1/2] import: Move pull_job_curl_on_finished after pull_job_open_disk To call into pull_job_write_(un)compressed and pull_job_open_disk from pull_job_curl_on_finished it has to be defined after them. This is in preparation for a bug fix for small files where we need to leave the compression analysis state to finish the download successfully. --- src/import/pull-job.c | 272 +++++++++++++++++++++--------------------- 1 file changed, 136 insertions(+), 136 deletions(-) diff --git a/src/import/pull-job.c b/src/import/pull-job.c index 5b8aa6da26..2dc960c392 100644 --- a/src/import/pull-job.c +++ b/src/import/pull-job.c @@ -163,6 +163,142 @@ static uint64_t pull_job_content_length_effective(PullJob *j) { return j->content_length; } +static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata) { + PullJob *j = ASSERT_PTR(userdata); + bool too_much = false; + int r; + + assert(p); + assert(sz > 0); + + if (j->written_uncompressed > UINT64_MAX - sz) + return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow"); + + if (j->written_uncompressed >= j->uncompressed_max) { + too_much = true; + goto finish; + } + + if (j->written_uncompressed + sz > j->uncompressed_max) { + too_much = true; + sz = j->uncompressed_max - j->written_uncompressed; /* since we have the data in memory + * already, we might as well write it to + * disk to the max */ + } + + if (j->disk_fd >= 0) { + + if (S_ISREG(j->disk_stat.st_mode) && j->offset == UINT64_MAX) { + ssize_t n; + + n = sparse_write(j->disk_fd, p, sz, 64); + if (n < 0) + return log_error_errno((int) n, "Failed to write file: %m"); + if ((size_t) n < sz) + return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write"); + } else { + r = loop_write(j->disk_fd, p, sz); + if (r < 0) + return log_error_errno(r, "Failed to write file: %m"); + } + } + + if (j->disk_fd < 0 || j->force_memory) { + uint8_t *a = j->payload.iov_base; + + if (!GREEDY_REALLOC(a, j->payload.iov_len + sz + 1)) + return log_oom(); + + *((uint8_t*) mempcpy(a + j->payload.iov_len, p, sz)) = 0; + j->payload.iov_base = a; + j->payload.iov_len += sz; + } + + j->written_uncompressed += sz; + +finish: + if (too_much) + return log_error_errno(SYNTHETIC_ERRNO(EFBIG), "File overly large, refusing."); + + return 0; +} + +static int pull_job_write_compressed(PullJob *j, const struct iovec *data) { + int r; + + assert(j); + assert(iovec_is_valid(data)); + + if (!iovec_is_set(data)) + return 0; + + if (j->written_compressed + data->iov_len < j->written_compressed) + return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow"); + + if (j->written_compressed + data->iov_len > j->compressed_max) + return log_error_errno(SYNTHETIC_ERRNO(EFBIG), "File overly large, refusing."); + + uint64_t cl = pull_job_content_length_effective(j); + if (cl != UINT64_MAX && + j->written_compressed + data->iov_len > cl) + return log_error_errno(SYNTHETIC_ERRNO(EFBIG), + "Content length incorrect."); + + if (j->checksum_ctx) { + r = sym_EVP_DigestUpdate(j->checksum_ctx, data->iov_base, data->iov_len); + if (r == 0) + return log_error_errno(SYNTHETIC_ERRNO(EIO), + "Could not hash chunk."); + } + + r = decompressor_push(j->compress, data->iov_base, data->iov_len, pull_job_write_uncompressed, j); + if (r < 0) + return r; + + j->written_compressed += data->iov_len; + + return 0; +} + +static int pull_job_open_disk(PullJob *j) { + int r; + + assert(j); + + if (j->on_open_disk) { + r = j->on_open_disk(j); + if (r < 0) + return r; + } + + if (j->disk_fd >= 0) { + if (fstat(j->disk_fd, &j->disk_stat) < 0) + return log_error_errno(errno, "Failed to stat disk file: %m"); + + if (j->offset != UINT64_MAX) { + if (lseek(j->disk_fd, j->offset, SEEK_SET) < 0) + return log_error_errno(errno, "Failed to seek on file descriptor: %m"); + } + } + + if (j->calc_checksum) { + r = dlopen_libcrypto(LOG_ERR); + if (r < 0) + return r; + + j->checksum_ctx = sym_EVP_MD_CTX_new(); + if (!j->checksum_ctx) + return log_oom(); + + r = sym_EVP_DigestInit_ex(j->checksum_ctx, sym_EVP_sha256(), NULL); + if (r == 0) + return log_error_errno(SYNTHETIC_ERRNO(EIO), + "Failed to initialize hash context."); + } + + return 0; +} + static int pull_job_curl_on_finished(CurlSlot *slot, CURL *curl, CURLcode result, void *userdata) { PullJob *j = ASSERT_PTR(userdata); char *scheme = NULL; @@ -334,142 +470,6 @@ static int pull_job_curl_on_finished(CurlSlot *slot, CURL *curl, CURLcode result return pull_job_finish(j, 0); } -static int pull_job_write_uncompressed(const void *p, size_t sz, void *userdata) { - PullJob *j = ASSERT_PTR(userdata); - bool too_much = false; - int r; - - assert(p); - assert(sz > 0); - - if (j->written_uncompressed > UINT64_MAX - sz) - return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow"); - - if (j->written_uncompressed >= j->uncompressed_max) { - too_much = true; - goto finish; - } - - if (j->written_uncompressed + sz > j->uncompressed_max) { - too_much = true; - sz = j->uncompressed_max - j->written_uncompressed; /* since we have the data in memory - * already, we might as well write it to - * disk to the max */ - } - - if (j->disk_fd >= 0) { - - if (S_ISREG(j->disk_stat.st_mode) && j->offset == UINT64_MAX) { - ssize_t n; - - n = sparse_write(j->disk_fd, p, sz, 64); - if (n < 0) - return log_error_errno((int) n, "Failed to write file: %m"); - if ((size_t) n < sz) - return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short write"); - } else { - r = loop_write(j->disk_fd, p, sz); - if (r < 0) - return log_error_errno(r, "Failed to write file: %m"); - } - } - - if (j->disk_fd < 0 || j->force_memory) { - uint8_t *a = j->payload.iov_base; - - if (!GREEDY_REALLOC(a, j->payload.iov_len + sz + 1)) - return log_oom(); - - *((uint8_t*) mempcpy(a + j->payload.iov_len, p, sz)) = 0; - j->payload.iov_base = a; - j->payload.iov_len += sz; - } - - j->written_uncompressed += sz; - -finish: - if (too_much) - return log_error_errno(SYNTHETIC_ERRNO(EFBIG), "File overly large, refusing."); - - return 0; -} - -static int pull_job_write_compressed(PullJob *j, const struct iovec *data) { - int r; - - assert(j); - assert(iovec_is_valid(data)); - - if (!iovec_is_set(data)) - return 0; - - if (j->written_compressed + data->iov_len < j->written_compressed) - return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "File too large, overflow"); - - if (j->written_compressed + data->iov_len > j->compressed_max) - return log_error_errno(SYNTHETIC_ERRNO(EFBIG), "File overly large, refusing."); - - uint64_t cl = pull_job_content_length_effective(j); - if (cl != UINT64_MAX && - j->written_compressed + data->iov_len > cl) - return log_error_errno(SYNTHETIC_ERRNO(EFBIG), - "Content length incorrect."); - - if (j->checksum_ctx) { - r = sym_EVP_DigestUpdate(j->checksum_ctx, data->iov_base, data->iov_len); - if (r == 0) - return log_error_errno(SYNTHETIC_ERRNO(EIO), - "Could not hash chunk."); - } - - r = decompressor_push(j->compress, data->iov_base, data->iov_len, pull_job_write_uncompressed, j); - if (r < 0) - return r; - - j->written_compressed += data->iov_len; - - return 0; -} - -static int pull_job_open_disk(PullJob *j) { - int r; - - assert(j); - - if (j->on_open_disk) { - r = j->on_open_disk(j); - if (r < 0) - return r; - } - - if (j->disk_fd >= 0) { - if (fstat(j->disk_fd, &j->disk_stat) < 0) - return log_error_errno(errno, "Failed to stat disk file: %m"); - - if (j->offset != UINT64_MAX) { - if (lseek(j->disk_fd, j->offset, SEEK_SET) < 0) - return log_error_errno(errno, "Failed to seek on file descriptor: %m"); - } - } - - if (j->calc_checksum) { - r = dlopen_libcrypto(LOG_ERR); - if (r < 0) - return r; - - j->checksum_ctx = sym_EVP_MD_CTX_new(); - if (!j->checksum_ctx) - return log_oom(); - - r = sym_EVP_DigestInit_ex(j->checksum_ctx, sym_EVP_sha256(), NULL); - if (r == 0) - return log_error_errno(SYNTHETIC_ERRNO(EIO), - "Failed to initialize hash context."); - } - - return 0; -} - static int pull_job_detect_compression(PullJob *j) { int r; From 5f299e53a8a936d799bb1f223f2abbdcf3973ce9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20L=C3=BCke?= Date: Mon, 18 May 2026 16:46:28 +0900 Subject: [PATCH 2/2] import: Handle small files When systemd-pull encountered a file shorter than the compression magic headers it looks for, then it would complete the download in the analysis state and fail. When we are still in the analysis state and the download is done, we know there is no compression and we should leave the analysis state and continue writing out to disk as usual. --- src/import/pull-job.c | 52 ++++++++++++++++++++++++--------- test/units/TEST-72-SYSUPDATE.sh | 24 +++++++++++++++ 2 files changed, 62 insertions(+), 14 deletions(-) diff --git a/src/import/pull-job.c b/src/import/pull-job.c index 2dc960c392..31d6d34af6 100644 --- a/src/import/pull-job.c +++ b/src/import/pull-job.c @@ -299,6 +299,29 @@ static int pull_job_open_disk(PullJob *j) { return 0; } +static int pull_job_begin_running(PullJob *j) { + int r; + + assert(j); + assert(j->state == PULL_JOB_ANALYZING); + assert(j->compress); + + r = pull_job_open_disk(j); + if (r < 0) + return r; + + /* Now, take the payload we read so far, and decompress it */ + _cleanup_(iovec_done) struct iovec stub = TAKE_STRUCT(j->payload); + + j->state = PULL_JOB_RUNNING; + + r = pull_job_write_compressed(j, &stub); + if (r < 0) + return r; + + return 0; +} + static int pull_job_curl_on_finished(CurlSlot *slot, CURL *curl, CURLcode result, void *userdata) { PullJob *j = ASSERT_PTR(userdata); char *scheme = NULL; @@ -382,6 +405,20 @@ static int pull_job_curl_on_finished(CurlSlot *slot, CURL *curl, CURLcode result return pull_job_finish(j, log_error_errno(SYNTHETIC_ERRNO(EIO), "HTTP request to %s finished with unexpected code %li.", j->url, status)); } + if (j->state == PULL_JOB_ANALYZING) { + /* When curl finished the download while we were still looking for a compression magic + * header the content isn't compressed but should be written out as is. */ + assert(result == CURLE_OK); + + r = decompressor_force_off(&j->compress); + if (r < 0) + return pull_job_finish(j, r); + + r = pull_job_begin_running(j); + if (r < 0) + return pull_job_finish(j, r); + } + if (j->state != PULL_JOB_RUNNING) return pull_job_finish(j, log_error_errno(SYNTHETIC_ERRNO(EIO), "Premature connection termination.")); @@ -483,20 +520,7 @@ static int pull_job_detect_compression(PullJob *j) { log_debug("Stream is compressed: %s", compression_to_string(compressor_type(j->compress))); - r = pull_job_open_disk(j); - if (r < 0) - return r; - - /* Now, take the payload we read so far, and decompress it */ - _cleanup_(iovec_done) struct iovec stub = TAKE_STRUCT(j->payload); - - j->state = PULL_JOB_RUNNING; - - r = pull_job_write_compressed(j, &stub); - if (r < 0) - return r; - - return 0; + return pull_job_begin_running(j); } static size_t pull_job_write_callback(void *contents, size_t size, size_t nmemb, void *userdata) { diff --git a/test/units/TEST-72-SYSUPDATE.sh b/test/units/TEST-72-SYSUPDATE.sh index 0e1c9afa83..21a3ea00a5 100755 --- a/test/units/TEST-72-SYSUPDATE.sh +++ b/test/units/TEST-72-SYSUPDATE.sh @@ -552,4 +552,28 @@ EOF done done +# Make sure the processing of compressed streams still handles uncompressed streams shorter than +# COMPRESSION_MAGIC_BYTES_MAX correctly. +rm -rf "$CONFIGDIR" "$WORKDIR/blobs" +mkdir -p "$CONFIGDIR" "$WORKDIR/blobs" +printf 'xx' >"$WORKDIR/source/tiny-v1.bin" +(cd "$WORKDIR/source" && sha256sum tiny-v1.bin >SHA256SUMS) +cat >"$CONFIGDIR/01-tiny-url.transfer" <