SymbX/systemd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

import: rework pull logic to store download digests in binary form rather than string

Browse Source 
We generally want to store data in parsed form, not formatted form,
hence let's follow our own rules on this, and store the message digest
as "struct iovec" rather than as string. This is generally more
efficient and safer, simply because of case issues.
This commit is contained in:
Lennart Poettering
2025-11-08 09:28:50 +01:00
parent cb1caab98e
commit e825635af6
9 changed files with 68 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/import/pull-common.c
+13 -8
View File
@@ -6,6 +6,7 @@
#include "dirent-util.h"
#include "escape.h"
#include "fd-util.h"
#include "hexdecoct.h"
#include "io-util.h"
#include "log.h"
#include "memory-util.h"
@@ -247,7 +248,7 @@ int pull_make_verification_jobs(
PullJob **ret_checksum_job,
PullJob **ret_signature_job,
ImportVerify verify,
const char *checksum, /* set if literal checksum verification is requested, in which case 'verify' is set to _IMPORT_VERIFY_INVALID */
const struct iovec *checksum, /* set if literal checksum verification is requested, in which case 'verify' is set to _IMPORT_VERIFY_INVALID */
const char *url,
CurlGlue *glue,
PullJobFinished on_finished,
@@ -267,7 +268,7 @@ int pull_make_verification_jobs(
/* If verification is turned off, or if the checksum to validate is already specified we don't need
* to download a checksum file or signature, hence shortcut things */
if (verify == IMPORT_VERIFY_NO || checksum) {
if (verify == IMPORT_VERIFY_NO || iovec_is_set(checksum)) {
*ret_checksum_job = *ret_signature_job = NULL;
return 0;
}
@@ -351,7 +352,7 @@ static int verify_one(PullJob *checksum_job, PullJob *job) {
return 0;
assert(job->calc_checksum);
assert(job->checksum);
assert(iovec_is_set(&job->checksum));
r = import_url_last_component(job->url, &fn);
if (r < 0)
@@ -365,6 +366,10 @@ static int verify_one(PullJob *checksum_job, PullJob *job) {
return log_error_errno(SYNTHETIC_ERRNO(ELOOP),
"Cannot verify checksum/signature files via themselves.");
_cleanup_free_ char *cs = hexmem(job->checksum.iov_base, job->checksum.iov_len);
if (!cs)
return log_oom();
const char *p = NULL;
FOREACH_STRING(separator,
" *", /* separator for binary mode */
@@ -372,7 +377,7 @@ static int verify_one(PullJob *checksum_job, PullJob *job) {
" " /* non-standard separator used by linuxcontainers.org */) {
_cleanup_free_ char *line = NULL;
line = strjoin(job->checksum, separator, fn, "\n");
line = strjoin(cs, separator, fn, "\n");
if (!line)
return log_oom();
@@ -510,7 +515,7 @@ finish:
}
int pull_verify(ImportVerify verify,
const char *checksum, /* Verify with literal checksum */
const struct iovec *checksum, /* Verify with literal checksum */
PullJob *main_job,
PullJob *checksum_job,
PullJob *signature_job,
@@ -543,9 +548,9 @@ int pull_verify(ImportVerify verify,
assert(!verity_job);
assert(main_job->calc_checksum);
assert(main_job->checksum);
assert(iovec_is_set(&main_job->checksum));
if (!strcaseeq(checksum, main_job->checksum))
if (iovec_memcmp(checksum, &main_job->checksum) != 0)
return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
"DOWNLOAD INVALID: Checksum of %s file did not check out, file has been tampered with.",
main_job->url);
@@ -566,7 +571,7 @@ int pull_verify(ImportVerify verify,
verify_job = main_job;
} else {
assert(main_job->calc_checksum);
assert(main_job->checksum);
assert(iovec_is_set(&main_job->checksum));
assert(checksum_job);
assert(checksum_job->state == PULL_JOB_DONE);
src/import/pull-common.h
+2 -2
View File
@@ -14,9 +14,9 @@ int pull_find_old_etags(const char *url, const char *root, int dt, const char *p
int pull_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret);
int pull_make_auxiliary_job(PullJob **ret, const char *url, int (*strip_suffixes)(const char *name, char **ret), const char *suffix, ImportVerify verify, CurlGlue *glue, PullJobOpenDisk on_open_disk, PullJobFinished on_finished, void *userdata);
int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const char *checksum, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata);
int pull_make_verification_jobs(PullJob **ret_checksum_job, PullJob **ret_signature_job, ImportVerify verify, const struct iovec *checksum, const char *url, CurlGlue *glue, PullJobFinished on_finished, void *userdata);
int pull_verify(ImportVerify verify, const char *checksum, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job);
int pull_verify(ImportVerify verify, const struct iovec *checksum, PullJob *main_job, PullJob *checksum_job, PullJob *signature_job, PullJob *settings_job, PullJob *roothash_job, PullJob *roothash_signature_job, PullJob *verity_job);
typedef enum VerificationStyle {
VERIFICATION_PER_FILE, /* SUSE-style ".sha256" files with detached gpg signature */
src/import/pull-job.c
+22 -13
View File
@@ -48,7 +48,7 @@ PullJob* pull_job_unref(PullJob *j) {
free(j->etag);
strv_free(j->old_etags);
free(j->payload);
free(j->checksum);
iovec_done(&j->checksum);
return mfree(j);
}
@@ -92,7 +92,7 @@ static int pull_job_restart(PullJob *j, const char *new_url) {
j->etag = mfree(j->etag);
j->etag_exists = false;
j->mtime = 0;
j->checksum = mfree(j->checksum);
iovec_done(&j->checksum);
j->expected_content_length = UINT64_MAX;
curl_glue_remove_and_free(j->glue, j->curl);
@@ -233,22 +233,31 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) {
if (j->checksum_ctx) {
unsigned checksum_len;
uint8_t k[EVP_MAX_MD_SIZE];
r = EVP_DigestFinal_ex(j->checksum_ctx, k, &checksum_len);
if (r == 0) {
r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get checksum.");
goto finish;
}
assert(checksum_len <= sizeof k);
j->checksum = hexmem(k, checksum_len);
if (!j->checksum) {
iovec_done(&j->checksum);
j->checksum.iov_base = malloc(EVP_MAX_MD_SIZE);
if (!j->checksum.iov_base) {
r = log_oom();
goto finish;
}
log_debug("SHA256 of %s is %s.", j->url, j->checksum);
r = EVP_DigestFinal_ex(j->checksum_ctx, j->checksum.iov_base, &checksum_len);
if (r == 0) {
r = log_error_errno(SYNTHETIC_ERRNO(EIO), "Failed to get checksum.");
goto finish;
}
assert(checksum_len <= EVP_MAX_MD_SIZE);
j->checksum.iov_len = checksum_len;
if (DEBUG_LOGGING) {
_cleanup_free_ char *h = hexmem(j->checksum.iov_base, j->checksum.iov_len);
if (!h) {
r = log_oom();
goto finish;
}
log_debug("%s of %s is %s.", EVP_MD_CTX_get0_name(j->checksum_ctx), j->url, h);
}
}
/* Do a couple of finishing disk operations, but only if we are the sole owner of the file (i.e. no
src/import/pull-job.h
+2 -1
View File
@@ -78,7 +78,8 @@ typedef struct PullJob {
bool calc_checksum;
EVP_MD_CTX *checksum_ctx;
char *checksum;
struct iovec checksum;
bool sync;
bool force_memory;
} PullJob;
src/import/pull-raw.c
+10 -11
View File
@@ -68,7 +68,7 @@ typedef struct RawPull {
char *verity_path;
char *verity_temp_path;
char *checksum;
struct iovec checksum;
} RawPull;
RawPull* raw_pull_unref(RawPull *i) {
@@ -99,7 +99,7 @@ RawPull* raw_pull_unref(RawPull *i) {
free(i->verity_path);
free(i->image_root);
free(i->local);
free(i->checksum);
iovec_done(&i->checksum);
return mfree(i);
}
@@ -585,7 +585,7 @@ static void raw_pull_job_on_finished(PullJob *j) {
raw_pull_report_progress(i, RAW_VERIFYING);
r = pull_verify(i->verify,
i->checksum,
&i->checksum,
i->raw_job,
i->checksum_job,
i->signature_job,
@@ -827,7 +827,7 @@ int raw_pull_start(
uint64_t size_max,
ImportFlags flags,
ImportVerify verify,
const char *checksum) {
const struct iovec *checksum) {
int r;
@@ -835,11 +835,11 @@ int raw_pull_start(
assert(url);
assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
assert((verify < 0) || !checksum);
assert((verify < 0) || !iovec_is_set(checksum));
assert(!(flags & ~IMPORT_PULL_FLAGS_MASK_RAW));
assert(offset == UINT64_MAX || FLAGS_SET(flags, IMPORT_DIRECT));
assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !(flags & IMPORT_DIRECT));
assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !checksum);
assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !iovec_is_set(checksum));
if (!http_url_is_valid(url) && !file_url_is_valid(url))
return -EINVAL;
@@ -854,9 +854,8 @@ int raw_pull_start(
if (r < 0)
return r;
r = free_and_strdup(&i->checksum, checksum);
if (r < 0)
return r;
if (!iovec_memdup(checksum, &i->checksum))
return -ENOMEM;
i->flags = flags;
i->verify = verify;
@@ -869,7 +868,7 @@ int raw_pull_start(
i->raw_job->on_finished = raw_pull_job_on_finished;
i->raw_job->on_open_disk = raw_pull_job_on_open_disk_raw;
if (checksum)
if (iovec_is_set(checksum))
i->raw_job->calc_checksum = true;
else if (verify != IMPORT_VERIFY_NO) {
/* Calculate checksum of the main download unless the users asks for a SHA256SUM file or its
@@ -899,7 +898,7 @@ int raw_pull_start(
&i->checksum_job,
&i->signature_job,
verify,
i->checksum,
&i->checksum,
url,
i->glue,
raw_pull_job_on_finished,
src/import/pull-raw.h
+1 -1
View File
@@ -14,4 +14,4 @@ RawPull* raw_pull_unref(RawPull *pull);
DEFINE_TRIVIAL_CLEANUP_FUNC(RawPull*, raw_pull_unref);
int raw_pull_start(RawPull *pull, const char *url, const char *local, uint64_t offset, uint64_t size_max, ImportFlags flags, ImportVerify verify, const char *checksum);
int raw_pull_start(RawPull *pull, const char *url, const char *local, uint64_t offset, uint64_t size_max, ImportFlags flags, ImportVerify verify, const struct iovec *checksum);
src/import/pull-tar.c
+8 -9
View File
@@ -65,7 +65,7 @@ typedef struct TarPull {
char *settings_path;
char *settings_temp_path;
char *checksum;
struct iovec checksum;
int tree_fd;
int userns_fd;
@@ -98,7 +98,7 @@ TarPull* tar_pull_unref(TarPull *i) {
free(i->settings_path);
free(i->image_root);
free(i->local);
free(i->checksum);
iovec_done(&i->checksum);
safe_close(i->tree_fd);
safe_close(i->userns_fd);
@@ -478,7 +478,7 @@ static void tar_pull_job_on_finished(PullJob *j) {
clear_progress_bar(/* prefix= */ NULL);
r = pull_verify(i->verify,
i->checksum,
&i->checksum,
i->tar_job,
i->checksum_job,
i->signature_job,
@@ -698,17 +698,17 @@ int tar_pull_start(
const char *local,
ImportFlags flags,
ImportVerify verify,
const char *checksum) {
const struct iovec *checksum) {
int r;
assert(i);
assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
assert((verify < 0) || !checksum);
assert((verify < 0) || !iovec_is_set(checksum));
assert(!(flags & ~IMPORT_PULL_FLAGS_MASK_TAR));
assert(!(flags & IMPORT_PULL_SETTINGS) || !(flags & IMPORT_DIRECT));
assert(!(flags & IMPORT_PULL_SETTINGS) || !checksum);
assert(!(flags & IMPORT_PULL_SETTINGS) || !iovec_is_set(checksum));
if (!http_url_is_valid(url) && !file_url_is_valid(url))
return -EINVAL;
@@ -723,9 +723,8 @@ int tar_pull_start(
if (r < 0)
return r;
r = free_and_strdup(&i->checksum, checksum);
if (r < 0)
return r;
if (!iovec_memdup(checksum, &i->checksum))
return -ENOMEM;
i->flags = flags;
i->verify = verify;
src/import/pull-tar.h
+1 -1
View File
@@ -14,4 +14,4 @@ TarPull* tar_pull_unref(TarPull *pull);
DEFINE_TRIVIAL_CLEANUP_FUNC(TarPull*, tar_pull_unref);
int tar_pull_start(TarPull *pull, const char *url, const char *local, ImportFlags flags, ImportVerify verify, const char *checksum);
int tar_pull_start(TarPull *pull, const char *url, const char *local, ImportFlags flags, ImportVerify verify, const struct iovec *checksum);
src/import/pull.c
+9 -10
View File
@@ -15,6 +15,7 @@
#include "import-common.h"
#include "import-util.h"
#include "io-util.h"
#include "iovec-util.h"
#include "log.h"
#include "main-func.h"
#include "parse-argument.h"
@@ -32,11 +33,11 @@ static char *arg_image_root = NULL;
static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE;
static ImportFlags arg_import_flags = IMPORT_PULL_SETTINGS | IMPORT_PULL_ROOTHASH | IMPORT_PULL_ROOTHASH_SIGNATURE | IMPORT_PULL_VERITY | IMPORT_BTRFS_SUBVOL | IMPORT_BTRFS_QUOTA | IMPORT_CONVERT_QCOW2 | IMPORT_SYNC;
static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX;
static char *arg_checksum = NULL;
static struct iovec arg_checksum = {};
static ImageClass arg_class = IMAGE_MACHINE;
static RuntimeScope arg_runtime_scope = _RUNTIME_SCOPE_INVALID;
STATIC_DESTRUCTOR_REGISTER(arg_checksum, freep);
STATIC_DESTRUCTOR_REGISTER(arg_checksum, iovec_done);
STATIC_DESTRUCTOR_REGISTER(arg_image_root, freep);
static int normalize_local(const char *local, const char *url, char **ret) {
@@ -162,7 +163,7 @@ static int pull_tar(int argc, char *argv[], void *userdata) {
normalized,
arg_import_flags & IMPORT_PULL_FLAGS_MASK_TAR,
arg_verify,
arg_checksum);
&arg_checksum);
if (r < 0)
return log_error_errno(r, "Failed to pull image: %m");
@@ -231,7 +232,7 @@ static int pull_raw(int argc, char *argv[], void *userdata) {
arg_size_max,
arg_import_flags & IMPORT_PULL_FLAGS_MASK_RAW,
arg_verify,
arg_checksum);
&arg_checksum);
if (r < 0)
return log_error_errno(r, "Failed to pull image: %m");
@@ -371,7 +372,6 @@ static int parse_argv(int argc, char *argv[]) {
v = import_verify_from_string(optarg);
if (v < 0) {
_cleanup_free_ void *h = NULL;
char *hh;
size_t n;
/* If this is not a valid verification mode, maybe it's a literally specified
@@ -385,11 +385,10 @@ static int parse_argv(int argc, char *argv[]) {
return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
"64 hex character SHA256 hash required when specifying explicit checksum, %zu specified", n * 2);
hh = hexmem(h, n); /* bring into canonical (lowercase) form */
if (!hh)
return log_oom();
iovec_done(&arg_checksum);
arg_checksum.iov_base = TAKE_PTR(h);
arg_checksum.iov_len = n;
free_and_replace(arg_checksum, hh);
arg_import_flags &= ~(IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY);
arg_verify = _IMPORT_VERIFY_INVALID;
} else
@@ -542,7 +541,7 @@ static int parse_argv(int argc, char *argv[]) {
if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_import_flags, IMPORT_DIRECT))
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode.");
if (arg_checksum && (arg_import_flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) != 0)
if (iovec_is_set(&arg_checksum) && (arg_import_flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) != 0)
return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Literal checksum verification only supported if no associated files are downloaded.");
if (!arg_image_root) {