From e825635af62552f51a9ce1fe2dd132c7be4bbf87 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 5 Nov 2025 17:31:24 +0100 Subject: [PATCH] import: rework pull logic to store download digests in binary form rather than string We generally want to store data in parsed form, not formatted form, hence let's follow our own rules on this, and store the message digest as "struct iovec" rather than as string. This is generally more efficient and safer, simply because of case issues. --- src/import/pull-common.c | 21 +++++++++++++-------- src/import/pull-common.h | 4 ++-- src/import/pull-job.c | 35 ++++++++++++++++++++++------------- src/import/pull-job.h | 3 ++- src/import/pull-raw.c | 21 ++++++++++----------- src/import/pull-raw.h | 2 +- src/import/pull-tar.c | 17 ++++++++--------- src/import/pull-tar.h | 2 +- src/import/pull.c | 19 +++++++++---------- 9 files changed, 68 insertions(+), 56 deletions(-) diff --git a/src/import/pull-common.c b/src/import/pull-common.c index 08a5a6192d..61ac57e555 100644 --- a/src/import/pull-common.c +++ b/src/import/pull-common.c @@ -6,6 +6,7 @@ #include "dirent-util.h" #include "escape.h" #include "fd-util.h" +#include "hexdecoct.h" #include "io-util.h" #include "log.h" #include "memory-util.h" @@ -247,7 +248,7 @@ int pull_make_verification_jobs( PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, - const char *checksum, /* set if literal checksum verification is requested, in which case 'verify' is set to _IMPORT_VERIFY_INVALID */ + const struct iovec *checksum, /* set if literal checksum verification is requested, in which case 'verify' is set to _IMPORT_VERIFY_INVALID */ const char *url, CurlGlue *glue, PullJobFinished on_finished, @@ -267,7 +268,7 @@ int pull_make_verification_jobs( /* If verification is turned off, or if the checksum to validate is already specified we don't need * to download a checksum file or signature, hence shortcut things */ - if (verify == IMPORT_VERIFY_NO || checksum) { + if (verify == IMPORT_VERIFY_NO || iovec_is_set(checksum)) { *ret_checksum_job = *ret_signature_job = NULL; return 0; } @@ -351,7 +352,7 @@ static int verify_one(PullJob *checksum_job, PullJob *job) { return 0; assert(job->calc_checksum); - assert(job->checksum); + assert(iovec_is_set(&job->checksum)); r = import_url_last_component(job->url, &fn); if (r < 0) @@ -365,6 +366,10 @@ static int verify_one(PullJob *checksum_job, PullJob *job) { return log_error_errno(SYNTHETIC_ERRNO(ELOOP), "Cannot verify checksum/signature files via themselves."); + _cleanup_free_ char *cs = hexmem(job->checksum.iov_base, job->checksum.iov_len); + if (!cs) + return log_oom(); + const char *p = NULL; FOREACH_STRING(separator, " *", /* separator for binary mode */ @@ -372,7 +377,7 @@ static int verify_one(PullJob *checksum_job, PullJob *job) { " " /* non-standard separator used by linuxcontainers.org */) { _cleanup_free_ char *line = NULL; - line = strjoin(job->checksum, separator, fn, "\n"); + line = strjoin(cs, separator, fn, "\n"); if (!line) return log_oom(); @@ -510,7 +515,7 @@ finish: } int pull_verify(ImportVerify verify, - const char *checksum, /* Verify with literal checksum */ + const struct iovec *checksum, /* Verify with literal checksum */ PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, @@ -543,9 +548,9 @@ int pull_verify(ImportVerify verify, assert(!verity_job); assert(main_job->calc_checksum); - assert(main_job->checksum); + assert(iovec_is_set(&main_job->checksum)); - if (!strcaseeq(checksum, main_job->checksum)) + if (iovec_memcmp(checksum, &main_job->checksum) != 0) return log_error_errno(SYNTHETIC_ERRNO(EBADMSG), "DOWNLOAD INVALID: Checksum of %s file did not check out, file has been tampered with.", main_job->url); @@ -566,7 +571,7 @@ int pull_verify(ImportVerify verify, verify_job = main_job; } else { assert(main_job->calc_checksum); - assert(main_job->checksum); + assert(iovec_is_set(&main_job->checksum)); assert(checksum_job); assert(checksum_job->state == PULL_JOB_DONE); diff --git a/src/import/pull-common.h b/src/import/pull-common.h index 160c035a21..48cb6c5ec4 100644 --- a/src/import/pull-common.h +++ b/src/import/pull-common.h @@ -14,9 +14,9 @@ int pull_find_old_etags(const char *url, const char *root, int dt, const char *p int pull_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret); int pull_make_auxiliary_job(PullJob **ret, const char *url, int (*strip_suffixes)(const char *name, char **ret), const char *suffix, ImportVerify verify, CurlGlue *glue, PullJobOpenDisk on_open_disk, PullJobFinished on_finished, void *userdata); -int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const char *checksum, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata); +int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const struct iovec *checksum, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata); -int pull_verify(ImportVerify verify, const char *checksum, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job); +int pull_verify(ImportVerify verify, const struct iovec *checksum, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job); typedef enum VerificationStyle { VERIFICATION_PER_FILE, /* SUSE-style ".sha256" files with detached gpg signature */ diff --git a/src/import/pull-job.c b/src/import/pull-job.c index 9459a89cea..57e9e95939 100644 --- a/src/import/pull-job.c +++ b/src/import/pull-job.c @@ -48,7 +48,7 @@ PullJob* pull_job_unref(PullJob *j) { free(j->etag); strv_free(j->old_etags); free(j->payload); - free(j->checksum); + iovec_done(&j->checksum); return mfree(j); } @@ -92,7 +92,7 @@ static int pull_job_restart(PullJob *j, const char *new_url) { j->etag = mfree(j->etag); j->etag_exists = false; j->mtime = 0; - j->checksum = mfree(j->checksum); + iovec_done(&j->checksum); j->expected_content_length = UINT64_MAX; curl_glue_remove_and_free(j->glue, j->curl); @@ -233,22 +233,31 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) { if (j->checksum_ctx) { unsigned checksum_len; - uint8_t k[EVP_MAX_MD_SIZE]; - r = EVP_DigestFinal_ex(j->checksum_ctx, k, &checksum_len); - if (r == 0) { - r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get checksum."); - goto finish; - } - assert(checksum_len <= sizeof k); - - j->checksum = hexmem(k, checksum_len); - if (!j->checksum) { + iovec_done(&j->checksum); + j->checksum.iov_base = malloc(EVP_MAX_MD_SIZE); + if (!j->checksum.iov_base) { r = log_oom(); goto finish; } - log_debug("SHA256 of %s is %s.", j->url, j->checksum); + r = EVP_DigestFinal_ex(j->checksum_ctx, j->checksum.iov_base, &checksum_len); + if (r == 0) { + r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get checksum."); + goto finish; + } + assert(checksum_len <= EVP_MAX_MD_SIZE); + j->checksum.iov_len = checksum_len; + + if (DEBUG_LOGGING) { + _cleanup_free_ char *h = hexmem(j->checksum.iov_base, j->checksum.iov_len); + if (!h) { + r = log_oom(); + goto finish; + } + + log_debug("%s of %s is %s.", EVP_MD_CTX_get0_name(j->checksum_ctx), j->url, h); + } } /* Do a couple of finishing disk operations, but only if we are the sole owner of the file (i.e. no diff --git a/src/import/pull-job.h b/src/import/pull-job.h index 34156e9dd6..7437aab01a 100644 --- a/src/import/pull-job.h +++ b/src/import/pull-job.h @@ -78,7 +78,8 @@ typedef struct PullJob { bool calc_checksum; EVP_MD_CTX *checksum_ctx; - char *checksum; + struct iovec checksum; + bool sync; bool force_memory; } PullJob; diff --git a/src/import/pull-raw.c b/src/import/pull-raw.c index 6ba8f6cdc9..274bd3ec0d 100644 --- a/src/import/pull-raw.c +++ b/src/import/pull-raw.c @@ -68,7 +68,7 @@ typedef struct RawPull { char *verity_path; char *verity_temp_path; - char *checksum; + struct iovec checksum; } RawPull; RawPull* raw_pull_unref(RawPull *i) { @@ -99,7 +99,7 @@ RawPull* raw_pull_unref(RawPull *i) { free(i->verity_path); free(i->image_root); free(i->local); - free(i->checksum); + iovec_done(&i->checksum); return mfree(i); } @@ -585,7 +585,7 @@ static void raw_pull_job_on_finished(PullJob *j) { raw_pull_report_progress(i, RAW_VERIFYING); r = pull_verify(i->verify, - i->checksum, + &i->checksum, i->raw_job, i->checksum_job, i->signature_job, @@ -827,7 +827,7 @@ int raw_pull_start( uint64_t size_max, ImportFlags flags, ImportVerify verify, - const char *checksum) { + const struct iovec *checksum) { int r; @@ -835,11 +835,11 @@ int raw_pull_start( assert(url); assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX); assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0); - assert((verify < 0) || !checksum); + assert((verify < 0) || !iovec_is_set(checksum)); assert(!(flags & ~IMPORT_PULL_FLAGS_MASK_RAW)); assert(offset == UINT64_MAX || FLAGS_SET(flags, IMPORT_DIRECT)); assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !(flags & IMPORT_DIRECT)); - assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !checksum); + assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !iovec_is_set(checksum)); if (!http_url_is_valid(url) && !file_url_is_valid(url)) return -EINVAL; @@ -854,9 +854,8 @@ int raw_pull_start( if (r < 0) return r; - r = free_and_strdup(&i->checksum, checksum); - if (r < 0) - return r; + if (!iovec_memdup(checksum, &i->checksum)) + return -ENOMEM; i->flags = flags; i->verify = verify; @@ -869,7 +868,7 @@ int raw_pull_start( i->raw_job->on_finished = raw_pull_job_on_finished; i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw; - if (checksum) + if (iovec_is_set(checksum)) i->raw_job->calc_checksum = true; else if (verify != IMPORT_VERIFY_NO) { /* Calculate checksum of the main download unless the users asks for a SHA256SUM file or its @@ -899,7 +898,7 @@ int raw_pull_start( &i->checksum_job, &i->signature_job, verify, - i->checksum, + &i->checksum, url, i->glue, raw_pull_job_on_finished, diff --git a/src/import/pull-raw.h b/src/import/pull-raw.h index c8126d4ff1..0dbd4dcd20 100644 --- a/src/import/pull-raw.h +++ b/src/import/pull-raw.h @@ -14,4 +14,4 @@ RawPull* raw_pull_unref(RawPull *pull); DEFINE_TRIVIAL_CLEANUP_FUNC(RawPull*, raw_pull_unref); -int raw_pull_start(RawPull *pull, const char *url, const char *local, uint64_t offset, uint64_t size_max, ImportFlags flags, ImportVerify verify, const char *checksum); +int raw_pull_start(RawPull *pull, const char *url, const char *local, uint64_t offset, uint64_t size_max, ImportFlags flags, ImportVerify verify, const struct iovec *checksum); diff --git a/src/import/pull-tar.c b/src/import/pull-tar.c index 23bdfb186a..a899e9a225 100644 --- a/src/import/pull-tar.c +++ b/src/import/pull-tar.c @@ -65,7 +65,7 @@ typedef struct TarPull { char *settings_path; char *settings_temp_path; - char *checksum; + struct iovec checksum; int tree_fd; int userns_fd; @@ -98,7 +98,7 @@ TarPull* tar_pull_unref(TarPull *i) { free(i->settings_path); free(i->image_root); free(i->local); - free(i->checksum); + iovec_done(&i->checksum); safe_close(i->tree_fd); safe_close(i->userns_fd); @@ -478,7 +478,7 @@ static void tar_pull_job_on_finished(PullJob *j) { clear_progress_bar(/* prefix= */ NULL); r = pull_verify(i->verify, - i->checksum, + &i->checksum, i->tar_job, i->checksum_job, i->signature_job, @@ -698,17 +698,17 @@ int tar_pull_start( const char *local, ImportFlags flags, ImportVerify verify, - const char *checksum) { + const struct iovec *checksum) { int r; assert(i); assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX); assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0); - assert((verify < 0) || !checksum); + assert((verify < 0) || !iovec_is_set(checksum)); assert(!(flags & ~IMPORT_PULL_FLAGS_MASK_TAR)); assert(!(flags & IMPORT_PULL_SETTINGS) || !(flags & IMPORT_DIRECT)); - assert(!(flags & IMPORT_PULL_SETTINGS) || !checksum); + assert(!(flags & IMPORT_PULL_SETTINGS) || !iovec_is_set(checksum)); if (!http_url_is_valid(url) && !file_url_is_valid(url)) return -EINVAL; @@ -723,9 +723,8 @@ int tar_pull_start( if (r < 0) return r; - r = free_and_strdup(&i->checksum, checksum); - if (r < 0) - return r; + if (!iovec_memdup(checksum, &i->checksum)) + return -ENOMEM; i->flags = flags; i->verify = verify; diff --git a/src/import/pull-tar.h b/src/import/pull-tar.h index 8f622dff3a..d2118272ff 100644 --- a/src/import/pull-tar.h +++ b/src/import/pull-tar.h @@ -14,4 +14,4 @@ TarPull* tar_pull_unref(TarPull *pull); DEFINE_TRIVIAL_CLEANUP_FUNC(TarPull*, tar_pull_unref); -int tar_pull_start(TarPull *pull, const char *url, const char *local, ImportFlags flags, ImportVerify verify, const char *checksum); +int tar_pull_start(TarPull *pull, const char *url, const char *local, ImportFlags flags, ImportVerify verify, const struct iovec *checksum); diff --git a/src/import/pull.c b/src/import/pull.c index a1b9b940fd..a57c1c039d 100644 --- a/src/import/pull.c +++ b/src/import/pull.c @@ -15,6 +15,7 @@ #include "import-common.h" #include "import-util.h" #include "io-util.h" +#include "iovec-util.h" #include "log.h" #include "main-func.h" #include "parse-argument.h" @@ -32,11 +33,11 @@ static char *arg_image_root = NULL; static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE; static ImportFlags arg_import_flags = IMPORT_PULL_SETTINGS | IMPORT_PULL_ROOTHASH | IMPORT_PULL_ROOTHASH_SIGNATURE | IMPORT_PULL_VERITY | IMPORT_BTRFS_SUBVOL | IMPORT_BTRFS_QUOTA | IMPORT_CONVERT_QCOW2 | IMPORT_SYNC; static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX; -static char *arg_checksum = NULL; +static struct iovec arg_checksum = {}; static ImageClass arg_class = IMAGE_MACHINE; static RuntimeScope arg_runtime_scope = _RUNTIME_SCOPE_INVALID; -STATIC_DESTRUCTOR_REGISTER(arg_checksum, freep); +STATIC_DESTRUCTOR_REGISTER(arg_checksum, iovec_done); STATIC_DESTRUCTOR_REGISTER(arg_image_root, freep); static int normalize_local(const char *local, const char *url, char **ret) { @@ -162,7 +163,7 @@ static int pull_tar(int argc, char *argv[], void *userdata) { normalized, arg_import_flags & IMPORT_PULL_FLAGS_MASK_TAR, arg_verify, - arg_checksum); + &arg_checksum); if (r < 0) return log_error_errno(r, "Failed to pull image: %m"); @@ -231,7 +232,7 @@ static int pull_raw(int argc, char *argv[], void *userdata) { arg_size_max, arg_import_flags & IMPORT_PULL_FLAGS_MASK_RAW, arg_verify, - arg_checksum); + &arg_checksum); if (r < 0) return log_error_errno(r, "Failed to pull image: %m"); @@ -371,7 +372,6 @@ static int parse_argv(int argc, char *argv[]) { v = import_verify_from_string(optarg); if (v < 0) { _cleanup_free_ void *h = NULL; - char *hh; size_t n; /* If this is not a valid verification mode, maybe it's a literally specified @@ -385,11 +385,10 @@ static int parse_argv(int argc, char *argv[]) { return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "64 hex character SHA256 hash required when specifying explicit checksum, %zu specified", n * 2); - hh = hexmem(h, n); /* bring into canonical (lowercase) form */ - if (!hh) - return log_oom(); + iovec_done(&arg_checksum); + arg_checksum.iov_base = TAKE_PTR(h); + arg_checksum.iov_len = n; - free_and_replace(arg_checksum, hh); arg_import_flags &= ~(IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY); arg_verify = _IMPORT_VERIFY_INVALID; } else @@ -542,7 +541,7 @@ static int parse_argv(int argc, char *argv[]) { if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_import_flags, IMPORT_DIRECT)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode."); - if (arg_checksum && (arg_import_flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) != 0) + if (iovec_is_set(&arg_checksum) && (arg_import_flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) != 0) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Literal checksum verification only supported if no associated files are downloaded."); if (!arg_image_root) {