From 4a5aa6842d12a406651cdfa6e5278b0cb2df1872 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 23 Feb 2024 22:46:04 +0100 Subject: [PATCH 01/26] fd-util: O_DIRECTORY is fine in fd_verify_safe_flags() too --- src/basic/fd-util.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/basic/fd-util.c b/src/basic/fd-util.c index 371547facb..c16a2ab658 100644 --- a/src/basic/fd-util.c +++ b/src/basic/fd-util.c @@ -926,6 +926,8 @@ int fd_verify_safe_flags(int fd) { * * RAW_O_LARGEFILE: glibc secretly sets this and neglects to hide it from us if we call fcntl. * See comment in missing_fcntl.h for more details about this. + * + * O_DIRECTORY: this is set for directories, which are totally fine */ assert(fd >= 0); @@ -934,7 +936,7 @@ int fd_verify_safe_flags(int fd) { if (flags < 0) return -errno; - unexpected_flags = flags & ~(O_ACCMODE|O_NOFOLLOW|RAW_O_LARGEFILE); + unexpected_flags = flags & ~(O_ACCMODE|O_NOFOLLOW|RAW_O_LARGEFILE|O_DIRECTORY); if (unexpected_flags != 0) return log_debug_errno(SYNTHETIC_ERRNO(EREMOTEIO), "Unexpected flags set for extrinsic fd: 0%o", From 7253eaa0fb4520a6c05f93ea8e146b3f90b57c4a Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 23 Feb 2024 21:51:02 +0100 Subject: [PATCH 02/26] sd-event: make return code of sd_event_get_exit_code() optional --- man/sd_event_exit.xml | 11 +++++++---- src/libsystemd/sd-event/sd-event.c | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/man/sd_event_exit.xml b/man/sd_event_exit.xml index ccbd045509..aa3bc29e98 100644 --- a/man/sd_event_exit.xml +++ b/man/sd_event_exit.xml @@ -65,9 +65,11 @@ the exit code stored in the event loop object is updated, but otherwise no further operation is executed. - sd_event_get_exit_code() may be used to - query the exit code passed into - sd_event_exit() earlier. + sd_event_get_exit_code() may be used to query the exit code passed to an + earlier call of sd_event_exit(). The return parameter code + may be set to NULL, in order to simply check if sd_event_exit() + has been called before (as sd_event_get_exit_code() fails with + -ENODATA if that's not the case, see below). While the full positive and negative integer ranges may be used for the exit code, care should be taken not pick exit codes that @@ -122,7 +124,8 @@ -ENODATA - The event loop has not been requested to exit yet. + Returned by sd_event_get_exit_code() in case the event loop has not + been requested to exit yet. diff --git a/src/libsystemd/sd-event/sd-event.c b/src/libsystemd/sd-event/sd-event.c index c651d59502..12aa980593 100644 --- a/src/libsystemd/sd-event/sd-event.c +++ b/src/libsystemd/sd-event/sd-event.c @@ -4915,13 +4915,13 @@ _public_ int sd_event_get_state(sd_event *e) { _public_ int sd_event_get_exit_code(sd_event *e, int *code) { assert_return(e, -EINVAL); assert_return(e = event_resolve(e), -ENOPKG); - assert_return(code, -EINVAL); assert_return(!event_origin_changed(e), -ECHILD); if (!e->exit_requested) return -ENODATA; - *code = e->exit_code; + if (code) + *code = e->exit_code; return 0; } From 8bd4d506a49b70204400db53bfcec8d0abecc31b Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 23 Feb 2024 18:22:40 +0100 Subject: [PATCH 03/26] curl-util: fix downloads from file:// URLs if we try to open file:// URLs that don't exist, we'll not get IO/timer events about it, hence it is not sufficient to check for completion in these events. Let's add a defer event, to deal with that. Also, curl_multi_info_read() is a queue, make sure to handle all events that might be queued. --- src/import/curl-util.c | 38 +++++++++++++++++++++++++++++++------- src/import/curl-util.h | 1 + 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/import/curl-util.c b/src/import/curl-util.c index 94f718de17..10f2aa0468 100644 --- a/src/import/curl-util.c +++ b/src/import/curl-util.c @@ -10,20 +10,28 @@ #include "version.h" static void curl_glue_check_finished(CurlGlue *g) { - CURLMsg *msg; - int k = 0; + int r; assert(g); + /* sd_event_get_exit_code() returns -ENODATA if no exit was scheduled yet */ + r = sd_event_get_exit_code(g->event, /* ret_code= */ NULL); + if (r >= 0) + return; /* exit scheduled? Then don't process this anymore */ + if (r != -ENODATA) + log_debug_errno(r, "Unexpected error while checking for event loop exit code, ignoring: %m"); + + CURLMsg *msg; + int k = 0; msg = curl_multi_info_read(g->curl, &k); if (!msg) return; - if (msg->msg != CURLMSG_DONE) - return; - - if (g->on_finished) + if (msg->msg == CURLMSG_DONE && g->on_finished) g->on_finished(g, msg->easy_handle, msg->data.result); + + /* This is a queue, process another item soon, but do so in a later event loop iteration. */ + (void) sd_event_source_set_enabled(g->defer, SD_EVENT_ONESHOT); } static int curl_glue_on_io(sd_event_source *s, int fd, uint32_t revents, void *userdata) { @@ -153,6 +161,15 @@ static int curl_glue_timer_callback(CURLM *curl, long timeout_ms, void *userdata return 0; } +static int curl_glue_on_defer(sd_event_source *s, void *userdata) { + CurlGlue *g = ASSERT_PTR(userdata); + + assert(s); + + curl_glue_check_finished(g); + return 0; +} + CurlGlue *curl_glue_unref(CurlGlue *g) { sd_event_source *io; @@ -167,7 +184,8 @@ CurlGlue *curl_glue_unref(CurlGlue *g) { hashmap_free(g->ios); - sd_event_source_unref(g->timer); + sd_event_source_disable_unref(g->timer); + sd_event_source_disable_unref(g->defer); sd_event_unref(g->event); return mfree(g); } @@ -211,6 +229,12 @@ int curl_glue_new(CurlGlue **glue, sd_event *event) { if (curl_multi_setopt(g->curl, CURLMOPT_TIMERFUNCTION, curl_glue_timer_callback) != CURLM_OK) return -EINVAL; + r = sd_event_add_defer(g->event, &g->defer, curl_glue_on_defer, g); + if (r < 0) + return r; + + (void) sd_event_source_set_description(g->defer, "curl-defer"); + *glue = TAKE_PTR(g); return 0; diff --git a/src/import/curl-util.h b/src/import/curl-util.h index 6b4f992fe7..cef0b26986 100644 --- a/src/import/curl-util.h +++ b/src/import/curl-util.h @@ -16,6 +16,7 @@ struct CurlGlue { CURLM *curl; sd_event_source *timer; Hashmap *ios; + sd_event_source *defer; void (*on_finished)(CurlGlue *g, CURL *curl, CURLcode code); void *userdata; From 09232207b1491e75bb47e18aaef2f431b813567d Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 16 Feb 2024 18:20:24 +0100 Subject: [PATCH 04/26] importd: switch to pidref --- src/import/importd.c | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/import/importd.c b/src/import/importd.c index 24b8e9f92e..528fbfb6a1 100644 --- a/src/import/importd.c +++ b/src/import/importd.c @@ -15,6 +15,7 @@ #include "constants.h" #include "daemon-util.h" #include "env-util.h" +#include "event-util.h" #include "fd-util.h" #include "float.h" #include "hostname-util.h" @@ -68,7 +69,7 @@ struct Transfer { char *format; - pid_t pid; + PidRef pidref; int log_fd; @@ -131,8 +132,7 @@ static Transfer *transfer_unref(Transfer *t) { free(t->format); free(t->object_path); - if (t->pid > 1) - sigkill_wait(t->pid); + pidref_done_sigkill_wait(&t->pidref); safe_close(t->log_fd); safe_close(t->stdin_fd); @@ -302,7 +302,7 @@ static int transfer_cancel(Transfer *t) { assert(t); - r = kill_and_sigcont(t->pid, t->n_canceled < 3 ? SIGTERM : SIGKILL); + r = pidref_kill_and_sigcont(&t->pidref, t->n_canceled < 3 ? SIGTERM : SIGKILL); if (r < 0) return r; @@ -329,7 +329,7 @@ static int transfer_on_pid(sd_event_source *s, const siginfo_t *si, void *userda else log_error("Transfer process failed due to unknown reason."); - t->pid = 0; + pidref_done(&t->pidref); return transfer_finalize(t, success); } @@ -363,15 +363,17 @@ static int transfer_start(Transfer *t) { int r; assert(t); - assert(t->pid <= 0); + assert(!pidref_is_set(&t->pidref)); if (pipe2(pipefd, O_CLOEXEC) < 0) return -errno; - r = safe_fork_full("(sd-transfer)", - (int[]) { t->stdin_fd, t->stdout_fd < 0 ? pipefd[1] : t->stdout_fd, pipefd[1] }, - NULL, 0, - FORK_RESET_SIGNALS|FORK_CLOSE_ALL_FDS|FORK_DEATHSIG_SIGTERM|FORK_REARRANGE_STDIO, &t->pid); + r = pidref_safe_fork_full( + "(sd-transfer)", + (int[]) { t->stdin_fd, t->stdout_fd < 0 ? pipefd[1] : t->stdout_fd, pipefd[1] }, + /* except_fds= */ NULL, /* n_except_fds= */ 0, + FORK_RESET_SIGNALS|FORK_CLOSE_ALL_FDS|FORK_DEATHSIG_SIGTERM|FORK_REARRANGE_STDIO, + &t->pidref); if (r < 0) return r; if (r == 0) { @@ -488,8 +490,13 @@ static int transfer_start(Transfer *t) { t->stdin_fd = safe_close(t->stdin_fd); - r = sd_event_add_child(t->manager->event, &t->pid_event_source, - t->pid, WEXITED, transfer_on_pid, t); + r = event_add_child_pidref( + t->manager->event, + &t->pid_event_source, + &t->pidref, + WEXITED, + transfer_on_pid, + t); if (r < 0) return r; @@ -584,7 +591,7 @@ static int manager_on_notify(sd_event_source *s, int fd, uint32_t revents, void } HASHMAP_FOREACH(t, m->transfers) - if (ucred->pid == t->pid) + if (ucred->pid == t->pidref.pid) break; if (!t) { From b37ec1e7ca628f6b468eb97154490f87c2b47318 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 16 Feb 2024 18:20:36 +0100 Subject: [PATCH 05/26] importd: trivial modernizations --- src/import/export.c | 5 ++--- src/import/import-fs.c | 3 +-- src/import/import.c | 5 ++--- src/import/importd.c | 8 ++++---- src/import/pull-tar.c | 6 ++++-- src/import/pull.c | 8 ++++---- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/import/export.c b/src/import/export.c index cd67349a4e..074209b6dc 100644 --- a/src/import/export.c +++ b/src/import/export.c @@ -210,7 +210,7 @@ static int help(int argc, char *argv[], void *userdata) { "\n%2$sOptions:%3$s\n" " -h --help Show this help\n" " --version Show package version\n" - " --format=FORMAT Select format\n\n", + " --format=FORMAT Select format\n", program_invocation_short_name, ansi_underline(), ansi_normal(), @@ -288,8 +288,7 @@ static int run(int argc, char *argv[]) { int r; setlocale(LC_ALL, ""); - log_parse_environment(); - log_open(); + log_setup(); r = parse_argv(argc, argv); if (r <= 0) diff --git a/src/import/import-fs.c b/src/import/import-fs.c index fd79c8f01a..a89c835f57 100644 --- a/src/import/import-fs.c +++ b/src/import/import-fs.c @@ -379,8 +379,7 @@ static int run(int argc, char *argv[]) { int r; setlocale(LC_ALL, ""); - log_parse_environment(); - log_open(); + log_setup(); r = parse_argv(argc, argv); if (r <= 0) diff --git a/src/import/import.c b/src/import/import.c index a81617d38e..50059870a0 100644 --- a/src/import/import.c +++ b/src/import/import.c @@ -160,7 +160,7 @@ static int import_tar(int argc, char *argv[], void *userdata) { fd = open_source(path, normalized, &open_fd); if (fd < 0) - return r; + return fd; r = import_allocate_event_with_signals(&event); if (r < 0) @@ -475,8 +475,7 @@ static int run(int argc, char *argv[]) { int r; setlocale(LC_ALL, ""); - log_parse_environment(); - log_open(); + log_setup(); parse_env(); diff --git a/src/import/importd.c b/src/import/importd.c index 528fbfb6a1..624a5c24b2 100644 --- a/src/import/importd.c +++ b/src/import/importd.c @@ -108,11 +108,11 @@ struct Manager { static const char* const transfer_type_table[_TRANSFER_TYPE_MAX] = { [TRANSFER_IMPORT_TAR] = "import-tar", [TRANSFER_IMPORT_RAW] = "import-raw", - [TRANSFER_IMPORT_FS] = "import-fs", + [TRANSFER_IMPORT_FS] = "import-fs", [TRANSFER_EXPORT_TAR] = "export-tar", [TRANSFER_EXPORT_RAW] = "export-raw", - [TRANSFER_PULL_TAR] = "pull-tar", - [TRANSFER_PULL_RAW] = "pull-raw", + [TRANSFER_PULL_TAR] = "pull-tar", + [TRANSFER_PULL_RAW] = "pull-raw", }; DEFINE_PRIVATE_STRING_TABLE_LOOKUP_TO_STRING(transfer_type, TransferType); @@ -391,7 +391,7 @@ static int transfer_start(Transfer *t) { NULL, /* local */ NULL }; - unsigned k = 0; + size_t k = 0; /* Child */ diff --git a/src/import/pull-tar.c b/src/import/pull-tar.c index 9e185c255a..a031449560 100644 --- a/src/import/pull-tar.c +++ b/src/import/pull-tar.c @@ -219,7 +219,7 @@ static int tar_pull_determine_path( static int tar_pull_make_local_copy(TarPull *i) { _cleanup_(rm_rf_subvolume_and_freep) char *t = NULL; - const char *p; + _cleanup_free_ char *p = NULL; int r; assert(i); @@ -230,7 +230,9 @@ static int tar_pull_make_local_copy(TarPull *i) { assert(i->final_path); - p = prefix_roota(i->image_root, i->local); + p = path_join(i->image_root, i->local); + if (!p) + return log_oom(); r = tempfn_random(p, NULL, &t); if (r < 0) diff --git a/src/import/pull.c b/src/import/pull.c index 10e3f131a1..a2afc6f390 100644 --- a/src/import/pull.c +++ b/src/import/pull.c @@ -213,7 +213,8 @@ static int pull_raw(int argc, char *argv[], void *userdata) { if (!FLAGS_SET(arg_pull_flags, PULL_SYNC)) log_info("File system synchronization on completion is off."); - r = raw_pull_new(&pull, event, arg_image_root, on_raw_finished, event); + + r = raw_pull_new(&pull, event, arg_image_root, on_raw_finished, event); if (r < 0) return log_error_errno(r, "Failed to allocate puller: %m"); @@ -255,7 +256,7 @@ static int help(int argc, char *argv[], void *userdata) { " --roothash-signature=BOOL\n" " Download root hash signature file with image\n" " --verity=BOOL Download verity file with image\n" - " --image-root=PATH Image root directory\n\n" + " --image-root=PATH Image root directory\n" " --read-only Create a read-only image\n" " --direct Download directly to specified file\n" " --btrfs-subvol=BOOL Controls whether to create a btrfs subvolume\n" @@ -539,8 +540,7 @@ static int run(int argc, char *argv[]) { int r; setlocale(LC_ALL, ""); - log_parse_environment(); - log_open(); + log_setup(); parse_env(); From 423bba9926c2d6cb3f3afbc3b37f90d1ece09b87 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 16 Feb 2024 18:20:51 +0100 Subject: [PATCH 06/26] importd: modernize signal handling a bit --- src/import/export.c | 23 +++++------------------ src/import/import-common.c | 5 ++--- src/import/importd.c | 16 +++++++--------- 3 files changed, 14 insertions(+), 30 deletions(-) diff --git a/src/import/export.c b/src/import/export.c index 074209b6dc..ee65cc8d36 100644 --- a/src/import/export.c +++ b/src/import/export.c @@ -14,6 +14,7 @@ #include "fd-util.h" #include "fs-util.h" #include "hostname-util.h" +#include "import-common.h" #include "import-util.h" #include "main-func.h" #include "signal-util.h" @@ -43,12 +44,6 @@ static void determine_compression_from_filename(const char *p) { arg_compress = IMPORT_COMPRESS_UNCOMPRESSED; } -static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) { - log_notice("Transfer aborted."); - sd_event_exit(sd_event_source_get_event(s), EINTR); - return 0; -} - static void on_tar_finished(TarExport *export, int error, void *userdata) { sd_event *event = userdata; assert(export); @@ -101,13 +96,9 @@ static int export_tar(int argc, char *argv[], void *userdata) { log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), import_compress_type_to_string(arg_compress)); } - r = sd_event_default(&event); + r = import_allocate_event_with_signals(&event); if (r < 0) - return log_error_errno(r, "Failed to allocate event loop: %m"); - - assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT) >= 0); - (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); - (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + return r; r = tar_export_new(&export, event, on_tar_finished, event); if (r < 0) @@ -177,13 +168,9 @@ static int export_raw(int argc, char *argv[], void *userdata) { log_info("Exporting '%s', saving to '%s' with compression '%s'.", local, strna(pretty), import_compress_type_to_string(arg_compress)); } - r = sd_event_default(&event); + r = import_allocate_event_with_signals(&event); if (r < 0) - return log_error_errno(r, "Failed to allocate event loop: %m"); - - assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT) >= 0); - (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); - (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + return r; r = raw_export_new(&export, event, on_raw_finished, event); if (r < 0) diff --git a/src/import/import-common.c b/src/import/import-common.c index 4a8a9a9c20..9d1448545e 100644 --- a/src/import/import-common.c +++ b/src/import/import-common.c @@ -295,9 +295,8 @@ int import_allocate_event_with_signals(sd_event **ret) { if (r < 0) return log_error_errno(r, "Failed to allocate event loop: %m"); - assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT) >= 0); - (void) sd_event_add_signal(event, NULL, SIGTERM, interrupt_signal_handler, NULL); - (void) sd_event_add_signal(event, NULL, SIGINT, interrupt_signal_handler, NULL); + (void) sd_event_add_signal(event, NULL, SIGTERM|SD_EVENT_SIGNAL_PROCMASK, interrupt_signal_handler, NULL); + (void) sd_event_add_signal(event, NULL, SIGINT|SD_EVENT_SIGNAL_PROCMASK, interrupt_signal_handler, NULL); *ret = TAKE_PTR(event); return 0; diff --git a/src/import/importd.c b/src/import/importd.c index 624a5c24b2..dbf2fba4ab 100644 --- a/src/import/importd.c +++ b/src/import/importd.c @@ -642,17 +642,11 @@ static int manager_new(Manager **ret) { if (r < 0) return r; - (void) sd_event_set_watchdog(m->event, true); - - r = sd_event_add_signal(m->event, NULL, SIGINT, NULL, NULL); + r = sd_event_set_signal_exit(m->event, true); if (r < 0) return r; - r = sd_event_add_signal(m->event, NULL, SIGTERM, NULL, NULL); - if (r < 0) - return r; - - r = sd_event_add_signal(m->event, NULL, SIGRTMIN+18, sigrtmin18_handler, NULL); + r = sd_event_add_signal(m->event, NULL, (SIGRTMIN+18)|SD_EVENT_SIGNAL_PROCMASK, sigrtmin18_handler, NULL); if (r < 0) return r; @@ -660,6 +654,10 @@ static int manager_new(Manager **ret) { if (r < 0) log_debug_errno(r, "Failed allocate memory pressure event source, ignoring: %m"); + r = sd_event_set_watchdog(m->event, true); + if (r < 0) + log_debug_errno(r, "Failed to enable watchdog logic, ignoring: %m"); + r = sd_bus_default_system(&m->bus); if (r < 0) return r; @@ -1381,7 +1379,7 @@ static int run(int argc, char *argv[]) { umask(0022); - assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGCHLD, SIGTERM, SIGINT, SIGRTMIN+18) >= 0); + assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGCHLD) >= 0); r = manager_new(&m); if (r < 0) From 1db33ce50bc981d8ebce9c6e41f935038d1bead1 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 16 Feb 2024 18:40:47 +0100 Subject: [PATCH 07/26] importctl: add standalone client to importd This is pretty much a 1:1 copy of the importd specific part of machinectl. We turn this into a separate tool, so that we can eventually make the tool generic to also download other DDIs, not just machine images. --- src/import/importctl.c | 869 +++++++++++++++++++++++++++++++++++++++++ src/import/meson.build | 6 + 2 files changed, 875 insertions(+) create mode 100644 src/import/importctl.c diff --git a/src/import/importctl.c b/src/import/importctl.c new file mode 100644 index 0000000000..742f13002d --- /dev/null +++ b/src/import/importctl.c @@ -0,0 +1,869 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include + +#include "sd-bus.h" + +#include "alloc-util.h" +#include "build.h" +#include "bus-error.h" +#include "bus-locator.h" +#include "bus-util.h" +#include "fd-util.h" +#include "format-table.h" +#include "hostname-util.h" +#include "import-util.h" +#include "locale-util.h" +#include "log.h" +#include "macro.h" +#include "main-func.h" +#include "pager.h" +#include "parse-util.h" +#include "path-util.h" +#include "pretty-print.h" +#include "signal-util.h" +#include "sort-util.h" +#include "spawn-polkit-agent.h" +#include "string-table.h" +#include "verbs.h" +#include "web-util.h" + +static PagerFlags arg_pager_flags = 0; +static bool arg_legend = true; +static BusTransport arg_transport = BUS_TRANSPORT_LOCAL; +static const char *arg_host = NULL; +static bool arg_read_only = false; +static bool arg_quiet = false; +static bool arg_ask_password = true; +static bool arg_force = false; +static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE; +static const char* arg_format = NULL; + +static int match_log_message(sd_bus_message *m, void *userdata, sd_bus_error *error) { + const char **our_path = userdata, *line; + unsigned priority; + int r; + + assert(m); + assert(our_path); + + r = sd_bus_message_read(m, "us", &priority, &line); + if (r < 0) { + bus_log_parse_error(r); + return 0; + } + + if (!streq_ptr(*our_path, sd_bus_message_get_path(m))) + return 0; + + if (arg_quiet && LOG_PRI(priority) >= LOG_INFO) + return 0; + + log_full(priority, "%s", line); + return 0; +} + +static int match_transfer_removed(sd_bus_message *m, void *userdata, sd_bus_error *error) { + const char **our_path = userdata, *path, *result; + uint32_t id; + int r; + + assert(m); + assert(our_path); + + r = sd_bus_message_read(m, "uos", &id, &path, &result); + if (r < 0) { + bus_log_parse_error(r); + return 0; + } + + if (!streq_ptr(*our_path, path)) + return 0; + + sd_event_exit(sd_bus_get_event(sd_bus_message_get_bus(m)), !streq_ptr(result, "done")); + return 0; +} + +static int transfer_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) { + assert(s); + assert(si); + + if (!arg_quiet) + log_info("Continuing download in the background. Use \"%s cancel-transfer %" PRIu32 "\" to abort transfer.", + program_invocation_short_name, + PTR_TO_UINT32(userdata)); + + sd_event_exit(sd_event_source_get_event(s), EINTR); + return 0; +} + +static int transfer_image_common(sd_bus *bus, sd_bus_message *m) { + _cleanup_(sd_bus_slot_unrefp) sd_bus_slot *slot_job_removed = NULL, *slot_log_message = NULL; + _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL; + _cleanup_(sd_bus_message_unrefp) sd_bus_message *reply = NULL; + _cleanup_(sd_event_unrefp) sd_event* event = NULL; + const char *path = NULL; + uint32_t id; + int r; + + assert(bus); + assert(m); + + polkit_agent_open_if_enabled(arg_transport, arg_ask_password); + + r = sd_event_default(&event); + if (r < 0) + return log_error_errno(r, "Failed to get event loop: %m"); + + r = sd_bus_attach_event(bus, event, 0); + if (r < 0) + return log_error_errno(r, "Failed to attach bus to event loop: %m"); + + r = bus_match_signal_async( + bus, + &slot_job_removed, + bus_import_mgr, + "TransferRemoved", + match_transfer_removed, NULL, &path); + if (r < 0) + return log_error_errno(r, "Failed to request match: %m"); + + r = sd_bus_match_signal_async( + bus, + &slot_log_message, + "org.freedesktop.import1", + NULL, + "org.freedesktop.import1.Transfer", + "LogMessage", + match_log_message, NULL, &path); + if (r < 0) + return log_error_errno(r, "Failed to request match: %m"); + + r = sd_bus_call(bus, m, 0, &error, &reply); + if (r < 0) + return log_error_errno(r, "Failed to transfer image: %s", bus_error_message(&error, r)); + + r = sd_bus_message_read(reply, "uo", &id, &path); + if (r < 0) + return bus_log_parse_error(r); + + assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); + + if (!arg_quiet) + log_info("Enqueued transfer job %u. Press C-c to continue download in background.", id); + + (void) sd_event_add_signal(event, NULL, SIGINT, transfer_signal_handler, UINT32_TO_PTR(id)); + (void) sd_event_add_signal(event, NULL, SIGTERM, transfer_signal_handler, UINT32_TO_PTR(id)); + + r = sd_event_loop(event); + if (r < 0) + return log_error_errno(r, "Failed to run event loop: %m"); + + return -r; +} + +static int import_tar(int argc, char *argv[], void *userdata) { + _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL; + _cleanup_free_ char *ll = NULL, *fn = NULL; + const char *local = NULL, *path = NULL; + _cleanup_close_ int fd = -EBADF; + sd_bus *bus = ASSERT_PTR(userdata); + int r; + + if (argc >= 2) + path = empty_or_dash_to_null(argv[1]); + + if (argc >= 3) + local = empty_or_dash_to_null(argv[2]); + else if (path) { + r = path_extract_filename(path, &fn); + if (r < 0) + return log_error_errno(r, "Cannot extract container name from filename: %m"); + if (r == O_DIRECTORY) + return log_error_errno(SYNTHETIC_ERRNO(EISDIR), + "Path '%s' refers to directory, but we need a regular file: %m", path); + + local = fn; + } + if (!local) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Need either path or local name."); + + r = tar_strip_suffixes(local, &ll); + if (r < 0) + return log_oom(); + + local = ll; + + if (!hostname_is_valid(local, 0)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local name %s is not a suitable machine name.", + local); + + if (path) { + fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (fd < 0) + return log_error_errno(errno, "Failed to open %s: %m", path); + } + + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ImportTar"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "hsbb", + fd >= 0 ? fd : STDIN_FILENO, + local, + arg_force, + arg_read_only); + if (r < 0) + return bus_log_create_error(r); + + return transfer_image_common(bus, m); +} + +static int import_raw(int argc, char *argv[], void *userdata) { + _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL; + _cleanup_free_ char *ll = NULL, *fn = NULL; + const char *local = NULL, *path = NULL; + _cleanup_close_ int fd = -EBADF; + sd_bus *bus = ASSERT_PTR(userdata); + int r; + + if (argc >= 2) + path = empty_or_dash_to_null(argv[1]); + + if (argc >= 3) + local = empty_or_dash_to_null(argv[2]); + else if (path) { + r = path_extract_filename(path, &fn); + if (r < 0) + return log_error_errno(r, "Cannot extract container name from filename: %m"); + if (r == O_DIRECTORY) + return log_error_errno(SYNTHETIC_ERRNO(EISDIR), + "Path '%s' refers to directory, but we need a regular file: %m", path); + + local = fn; + } + if (!local) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Need either path or local name."); + + r = raw_strip_suffixes(local, &ll); + if (r < 0) + return log_oom(); + + local = ll; + + if (!hostname_is_valid(local, 0)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local name %s is not a suitable machine name.", + local); + + if (path) { + fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); + if (fd < 0) + return log_error_errno(errno, "Failed to open %s: %m", path); + } + + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ImportRaw"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "hsbb", + fd >= 0 ? fd : STDIN_FILENO, + local, + arg_force, + arg_read_only); + if (r < 0) + return bus_log_create_error(r); + + return transfer_image_common(bus, m); +} + +static int import_fs(int argc, char *argv[], void *userdata) { + _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL; + const char *local = NULL, *path = NULL; + _cleanup_free_ char *fn = NULL; + _cleanup_close_ int fd = -EBADF; + sd_bus *bus = ASSERT_PTR(userdata); + int r; + + if (argc >= 2) + path = empty_or_dash_to_null(argv[1]); + + if (argc >= 3) + local = empty_or_dash_to_null(argv[2]); + else if (path) { + r = path_extract_filename(path, &fn); + if (r < 0) + return log_error_errno(r, "Cannot extract container name from filename: %m"); + + local = fn; + } + if (!local) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Need either path or local name."); + + if (!hostname_is_valid(local, 0)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local name %s is not a suitable machine name.", + local); + + if (path) { + fd = open(path, O_DIRECTORY|O_RDONLY|O_CLOEXEC); + if (fd < 0) + return log_error_errno(errno, "Failed to open directory '%s': %m", path); + } + + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ImportFileSystem"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "hsbb", + fd >= 0 ? fd : STDIN_FILENO, + local, + arg_force, + arg_read_only); + if (r < 0) + return bus_log_create_error(r); + + return transfer_image_common(bus, m); +} + +static void determine_compression_from_filename(const char *p) { + if (arg_format) + return; + + if (!p) + return; + + if (endswith(p, ".xz")) + arg_format = "xz"; + else if (endswith(p, ".gz")) + arg_format = "gzip"; + else if (endswith(p, ".bz2")) + arg_format = "bzip2"; +} + +static int export_tar(int argc, char *argv[], void *userdata) { + _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL; + _cleanup_close_ int fd = -EBADF; + const char *local = NULL, *path = NULL; + sd_bus *bus = ASSERT_PTR(userdata); + int r; + + local = argv[1]; + if (!hostname_is_valid(local, 0)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Machine name %s is not valid.", local); + + if (argc >= 3) + path = argv[2]; + path = empty_or_dash_to_null(path); + + if (path) { + determine_compression_from_filename(path); + + fd = open(path, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_NOCTTY, 0666); + if (fd < 0) + return log_error_errno(errno, "Failed to open %s: %m", path); + } + + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ExportTar"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "shs", + local, + fd >= 0 ? fd : STDOUT_FILENO, + arg_format); + if (r < 0) + return bus_log_create_error(r); + + return transfer_image_common(bus, m); +} + +static int export_raw(int argc, char *argv[], void *userdata) { + _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL; + _cleanup_close_ int fd = -EBADF; + const char *local = NULL, *path = NULL; + sd_bus *bus = ASSERT_PTR(userdata); + int r; + + local = argv[1]; + if (!hostname_is_valid(local, 0)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Machine name %s is not valid.", local); + + if (argc >= 3) + path = argv[2]; + path = empty_or_dash_to_null(path); + + if (path) { + determine_compression_from_filename(path); + + fd = open(path, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_NOCTTY, 0666); + if (fd < 0) + return log_error_errno(errno, "Failed to open %s: %m", path); + } + + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ExportRaw"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "shs", + local, + fd >= 0 ? fd : STDOUT_FILENO, + arg_format); + if (r < 0) + return bus_log_create_error(r); + + return transfer_image_common(bus, m); +} + +static int pull_tar(int argc, char *argv[], void *userdata) { + _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL; + _cleanup_free_ char *l = NULL, *ll = NULL; + const char *local, *remote; + sd_bus *bus = ASSERT_PTR(userdata); + int r; + + remote = argv[1]; + if (!http_url_is_valid(remote) && !file_url_is_valid(remote)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "URL '%s' is not valid.", remote); + + if (argc >= 3) + local = argv[2]; + else { + r = import_url_last_component(remote, &l); + if (r < 0) + return log_error_errno(r, "Failed to get final component of URL: %m"); + + local = l; + } + + local = empty_or_dash_to_null(local); + + if (local) { + r = tar_strip_suffixes(local, &ll); + if (r < 0) + return log_oom(); + + local = ll; + + if (!hostname_is_valid(local, 0)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local name %s is not a suitable machine name.", + local); + } + + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "PullTar"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "sssb", + remote, + local, + import_verify_to_string(arg_verify), + arg_force); + if (r < 0) + return bus_log_create_error(r); + + return transfer_image_common(bus, m); +} + +static int pull_raw(int argc, char *argv[], void *userdata) { + _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL; + _cleanup_free_ char *l = NULL, *ll = NULL; + const char *local, *remote; + sd_bus *bus = ASSERT_PTR(userdata); + int r; + + remote = argv[1]; + if (!http_url_is_valid(remote) && !file_url_is_valid(remote)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "URL '%s' is not valid.", remote); + + if (argc >= 3) + local = argv[2]; + else { + r = import_url_last_component(remote, &l); + if (r < 0) + return log_error_errno(r, "Failed to get final component of URL: %m"); + + local = l; + } + + local = empty_or_dash_to_null(local); + + if (local) { + r = raw_strip_suffixes(local, &ll); + if (r < 0) + return log_oom(); + + local = ll; + + if (!hostname_is_valid(local, 0)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Local name %s is not a suitable machine name.", + local); + } + + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "PullRaw"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "sssb", + remote, + local, + import_verify_to_string(arg_verify), + arg_force); + if (r < 0) + return bus_log_create_error(r); + + return transfer_image_common(bus, m); +} + +typedef struct TransferInfo { + uint32_t id; + const char *type; + const char *remote; + const char *local; + double progress; +} TransferInfo; + +static int compare_transfer_info(const TransferInfo *a, const TransferInfo *b) { + return strcmp(a->local, b->local); +} + +static int list_transfers(int argc, char *argv[], void *userdata) { + size_t max_type = STRLEN("TYPE"), max_local = STRLEN("LOCAL"), max_remote = STRLEN("REMOTE"); + _cleanup_(sd_bus_message_unrefp) sd_bus_message *reply = NULL; + _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL; + _cleanup_free_ TransferInfo *transfers = NULL; + const char *type, *remote, *local; + sd_bus *bus = userdata; + uint32_t id, max_id = 0; + size_t n_transfers = 0; + double progress; + int r; + + pager_open(arg_pager_flags); + + r = bus_call_method(bus, bus_import_mgr, "ListTransfers", &error, &reply, NULL); + if (r < 0) + return log_error_errno(r, "Could not get transfers: %s", bus_error_message(&error, r)); + + r = sd_bus_message_enter_container(reply, 'a', "(usssdo)"); + if (r < 0) + return bus_log_parse_error(r); + + while ((r = sd_bus_message_read(reply, "(usssdo)", &id, &type, &remote, &local, &progress, NULL)) > 0) { + size_t l; + + if (!GREEDY_REALLOC(transfers, n_transfers + 1)) + return log_oom(); + + transfers[n_transfers].id = id; + transfers[n_transfers].type = type; + transfers[n_transfers].remote = remote; + transfers[n_transfers].local = local; + transfers[n_transfers].progress = progress; + + l = strlen(type); + if (l > max_type) + max_type = l; + + l = strlen(remote); + if (l > max_remote) + max_remote = l; + + l = strlen(local); + if (l > max_local) + max_local = l; + + if (id > max_id) + max_id = id; + + n_transfers++; + } + if (r < 0) + return bus_log_parse_error(r); + + r = sd_bus_message_exit_container(reply); + if (r < 0) + return bus_log_parse_error(r); + + typesafe_qsort(transfers, n_transfers, compare_transfer_info); + + if (arg_legend && n_transfers > 0) + printf("%-*s %-*s %-*s %-*s %-*s\n", + (int) MAX(2U, DECIMAL_STR_WIDTH(max_id)), "ID", + (int) 7, "PERCENT", + (int) max_type, "TYPE", + (int) max_local, "LOCAL", + (int) max_remote, "REMOTE"); + + for (size_t j = 0; j < n_transfers; j++) + + if (transfers[j].progress < 0) + printf("%*" PRIu32 " %*s %-*s %-*s %-*s\n", + (int) MAX(2U, DECIMAL_STR_WIDTH(max_id)), transfers[j].id, + (int) 7, "n/a", + (int) max_type, transfers[j].type, + (int) max_local, transfers[j].local, + (int) max_remote, transfers[j].remote); + else + printf("%*" PRIu32 " %*u%% %-*s %-*s %-*s\n", + (int) MAX(2U, DECIMAL_STR_WIDTH(max_id)), transfers[j].id, + (int) 6, (unsigned) (transfers[j].progress * 100), + (int) max_type, transfers[j].type, + (int) max_local, transfers[j].local, + (int) max_remote, transfers[j].remote); + + if (arg_legend) { + if (n_transfers > 0) + printf("\n%zu transfers listed.\n", n_transfers); + else + printf("No transfers.\n"); + } + + return 0; +} + +static int cancel_transfer(int argc, char *argv[], void *userdata) { + _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL; + sd_bus *bus = ASSERT_PTR(userdata); + int r; + + polkit_agent_open_if_enabled(arg_transport, arg_ask_password); + + for (int i = 1; i < argc; i++) { + uint32_t id; + + r = safe_atou32(argv[i], &id); + if (r < 0) + return log_error_errno(r, "Failed to parse transfer id: %s", argv[i]); + + r = bus_call_method(bus, bus_import_mgr, "CancelTransfer", &error, NULL, "u", id); + if (r < 0) + return log_error_errno(r, "Could not cancel transfer: %s", bus_error_message(&error, r)); + } + + return 0; +} + +static int help(int argc, char *argv[], void *userdata) { + _cleanup_free_ char *link = NULL; + int r; + + pager_open(arg_pager_flags); + + r = terminal_urlify_man("importctl", "1", &link); + if (r < 0) + return log_oom(); + + printf("%1$s [OPTIONS...] COMMAND ...\n\n" + "%5$sDownload machine images%6$s\n" + "\n%3$sImage Transfer Commands:%4$s\n" + " pull-tar URL [NAME] Download a TAR container image\n" + " pull-raw URL [NAME] Download a RAW container or VM image\n" + " import-tar FILE [NAME] Import a local TAR container image\n" + " import-raw FILE [NAME] Import a local RAW container or VM image\n" + " import-fs DIRECTORY [NAME] Import a local directory container image\n" + " export-tar NAME [FILE] Export a TAR container image locally\n" + " export-raw NAME [FILE] Export a RAW container or VM image locally\n" + " list-transfers Show list of downloads in progress\n" + " cancel-transfer Cancel a download\n" + "\n%3$sOptions:%4$s\n" + " -h --help Show this help\n" + " --version Show package version\n" + " --no-pager Do not pipe output into a pager\n" + " --no-legend Do not show the headers and footers\n" + " --no-ask-password Do not ask for system passwords\n" + " -H --host=[USER@]HOST Operate on remote host\n" + " -M --machine=CONTAINER Operate on local container\n" + " --read-only Create read-only bind mount\n" + " -q --quiet Suppress output\n" + " --verify=MODE Verification mode for downloaded images (no,\n" + " checksum, signature)\n" + " --format=xz|gzip|bzip2 Desired output format for export\n" + " --force Download image even if already exists\n" + "\nSee the %2$s for details.\n", + program_invocation_short_name, + link, + ansi_underline(), + ansi_normal(), + ansi_highlight(), + ansi_normal()); + + return 0; +} + +static int parse_argv(int argc, char *argv[]) { + + enum { + ARG_VERSION = 0x100, + ARG_NO_PAGER, + ARG_NO_LEGEND, + ARG_NO_ASK_PASSWORD, + ARG_READ_ONLY, + ARG_VERIFY, + ARG_FORCE, + ARG_FORMAT, + }; + + static const struct option options[] = { + { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, ARG_VERSION }, + { "no-pager", no_argument, NULL, ARG_NO_PAGER }, + { "no-legend", no_argument, NULL, ARG_NO_LEGEND }, + { "no-ask-password", no_argument, NULL, ARG_NO_ASK_PASSWORD }, + { "host", required_argument, NULL, 'H' }, + { "machine", required_argument, NULL, 'M' }, + { "read-only", no_argument, NULL, ARG_READ_ONLY }, + { "quiet", no_argument, NULL, 'q' }, + { "verify", required_argument, NULL, ARG_VERIFY }, + { "force", no_argument, NULL, ARG_FORCE }, + { "format", required_argument, NULL, ARG_FORMAT }, + {} + }; + + int c, r; + + assert(argc >= 0); + assert(argv); + + for (;;) { + c = getopt_long(argc, argv, "hH:M:q", options, NULL); + if (c < 0) + break; + + switch (c) { + + case 'h': + return help(0, NULL, NULL); + + case ARG_VERSION: + return version(); + + case ARG_NO_PAGER: + arg_pager_flags |= PAGER_DISABLE; + break; + + case ARG_NO_LEGEND: + arg_legend = false; + break; + + case ARG_NO_ASK_PASSWORD: + arg_ask_password = false; + break; + + case 'H': + arg_transport = BUS_TRANSPORT_REMOTE; + arg_host = optarg; + break; + + case 'M': + arg_transport = BUS_TRANSPORT_MACHINE; + arg_host = optarg; + break; + + case ARG_READ_ONLY: + arg_read_only = true; + break; + + case 'q': + arg_quiet = true; + break; + + case ARG_VERIFY: + if (streq(optarg, "help")) { + DUMP_STRING_TABLE(import_verify, ImportVerify, _IMPORT_VERIFY_MAX); + return 0; + } + + r = import_verify_from_string(optarg); + if (r < 0) + return log_error_errno(r, "Failed to parse --verify= setting: %s", optarg); + arg_verify = r; + break; + + case ARG_FORCE: + arg_force = true; + break; + + case ARG_FORMAT: + if (!STR_IN_SET(optarg, "uncompressed", "xz", "gzip", "bzip2")) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "Unknown format: %s", optarg); + + arg_format = optarg; + break; + + case '?': + return -EINVAL; + + default: + assert_not_reached(); + } + } + + return 1; +} + +static int importctl_main(int argc, char *argv[], sd_bus *bus) { + + static const Verb verbs[] = { + { "help", VERB_ANY, VERB_ANY, 0, help }, + { "import-tar", 2, 3, 0, import_tar }, + { "import-raw", 2, 3, 0, import_raw }, + { "import-fs", 2, 3, 0, import_fs }, + { "export-tar", 2, 3, 0, export_tar }, + { "export-raw", 2, 3, 0, export_raw }, + { "pull-tar", 2, 3, 0, pull_tar }, + { "pull-raw", 2, 3, 0, pull_raw }, + { "list-transfers", VERB_ANY, 1, VERB_DEFAULT, list_transfers }, + { "cancel-transfer", 2, VERB_ANY, 0, cancel_transfer }, + {} + }; + + return dispatch_verb(argc, argv, verbs, bus); +} + +static int run(int argc, char *argv[]) { + _cleanup_(sd_bus_flush_close_unrefp) sd_bus *bus = NULL; + int r; + + setlocale(LC_ALL, ""); + log_setup(); + + r = parse_argv(argc, argv); + if (r <= 0) + return r; + + r = bus_connect_transport(arg_transport, arg_host, RUNTIME_SCOPE_SYSTEM, &bus); + if (r < 0) + return bus_log_connect_error(r, arg_transport); + + (void) sd_bus_set_allow_interactive_authorization(bus, arg_ask_password); + + return importctl_main(argc, argv, bus); +} + +DEFINE_MAIN_FUNCTION(run); diff --git a/src/import/meson.build b/src/import/meson.build index 3f0acf8358..ca23726603 100644 --- a/src/import/meson.build +++ b/src/import/meson.build @@ -100,6 +100,12 @@ executables += [ 'link_with' : common_libs, 'dependencies' : common_deps, }, + executable_template + { + 'name' : 'importctl', + 'public' : true, + 'conditions' : ['ENABLE_IMPORTD'], + 'sources' : files('importctl.c'), + }, test_template + { 'sources' : files( 'test-qcow2.c', From 663a15e7cc7c09d05bbfe8ac52173e73218bf410 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 16 Feb 2024 18:55:21 +0100 Subject: [PATCH 08/26] importctl: modernize signal handling --- src/import/importctl.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/import/importctl.c b/src/import/importctl.c index 742f13002d..0cf641b990 100644 --- a/src/import/importctl.c +++ b/src/import/importctl.c @@ -147,13 +147,11 @@ static int transfer_image_common(sd_bus *bus, sd_bus_message *m) { if (r < 0) return bus_log_parse_error(r); - assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT, -1) >= 0); - if (!arg_quiet) log_info("Enqueued transfer job %u. Press C-c to continue download in background.", id); - (void) sd_event_add_signal(event, NULL, SIGINT, transfer_signal_handler, UINT32_TO_PTR(id)); - (void) sd_event_add_signal(event, NULL, SIGTERM, transfer_signal_handler, UINT32_TO_PTR(id)); + (void) sd_event_add_signal(event, NULL, SIGINT|SD_EVENT_SIGNAL_PROCMASK, transfer_signal_handler, UINT32_TO_PTR(id)); + (void) sd_event_add_signal(event, NULL, SIGTERM|SD_EVENT_SIGNAL_PROCMASK, transfer_signal_handler, UINT32_TO_PTR(id)); r = sd_event_loop(event); if (r < 0) From 8229e226fe462b4f3cc220b4c7a354bed3ed5f73 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 21 Feb 2024 16:42:40 +0100 Subject: [PATCH 09/26] importctl: port tabular output for format-table.h APIs --- src/import/importctl.c | 145 ++++++++++++++++++++--------------------- 1 file changed, 71 insertions(+), 74 deletions(-) diff --git a/src/import/importctl.c b/src/import/importctl.c index 0cf641b990..de8e263d97 100644 --- a/src/import/importctl.c +++ b/src/import/importctl.c @@ -18,6 +18,7 @@ #include "macro.h" #include "main-func.h" #include "pager.h" +#include "parse-argument.h" #include "parse-util.h" #include "path-util.h" #include "pretty-print.h" @@ -38,6 +39,7 @@ static bool arg_ask_password = true; static bool arg_force = false; static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE; static const char* arg_format = NULL; +static JsonFormatFlags arg_json_format_flags = JSON_FORMAT_OFF; static int match_log_message(sd_bus_message *m, void *userdata, sd_bus_error *error) { const char **our_path = userdata, *line; @@ -537,28 +539,11 @@ static int pull_raw(int argc, char *argv[], void *userdata) { return transfer_image_common(bus, m); } -typedef struct TransferInfo { - uint32_t id; - const char *type; - const char *remote; - const char *local; - double progress; -} TransferInfo; - -static int compare_transfer_info(const TransferInfo *a, const TransferInfo *b) { - return strcmp(a->local, b->local); -} - static int list_transfers(int argc, char *argv[], void *userdata) { - size_t max_type = STRLEN("TYPE"), max_local = STRLEN("LOCAL"), max_remote = STRLEN("REMOTE"); - _cleanup_(sd_bus_message_unrefp) sd_bus_message *reply = NULL; _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL; - _cleanup_free_ TransferInfo *transfers = NULL; - const char *type, *remote, *local; - sd_bus *bus = userdata; - uint32_t id, max_id = 0; - size_t n_transfers = 0; - double progress; + _cleanup_(sd_bus_message_unrefp) sd_bus_message *reply = NULL; + _cleanup_(table_unrefp) Table *t = NULL; + sd_bus *bus = ASSERT_PTR(userdata); int r; pager_open(arg_pager_flags); @@ -571,72 +556,66 @@ static int list_transfers(int argc, char *argv[], void *userdata) { if (r < 0) return bus_log_parse_error(r); - while ((r = sd_bus_message_read(reply, "(usssdo)", &id, &type, &remote, &local, &progress, NULL)) > 0) { - size_t l; + t = table_new("id", "progress", "type", "local", "remote"); + if (!t) + return log_oom(); - if (!GREEDY_REALLOC(transfers, n_transfers + 1)) - return log_oom(); + (void) table_set_sort(t, (size_t) 3, (size_t) 0); + table_set_ersatz_string(t, TABLE_ERSATZ_DASH); - transfers[n_transfers].id = id; - transfers[n_transfers].type = type; - transfers[n_transfers].remote = remote; - transfers[n_transfers].local = local; - transfers[n_transfers].progress = progress; + for (;;) { + const char *type, *remote, *local; + double progress; + uint32_t id; - l = strlen(type); - if (l > max_type) - max_type = l; + r = sd_bus_message_read(reply, "(usssdo)", &id, &type, &remote, &local, &progress, NULL); + if (r < 0) + return bus_log_parse_error(r); + if (r == 0) + break; - l = strlen(remote); - if (l > max_remote) - max_remote = l; + r = table_add_many( + t, + TABLE_UINT32, id, + TABLE_SET_ALIGN_PERCENT, 100); + if (r < 0) + return table_log_add_error(r); - l = strlen(local); - if (l > max_local) - max_local = l; - - if (id > max_id) - max_id = id; - - n_transfers++; + if (progress < 0) + r = table_add_many( + t, + TABLE_EMPTY, + TABLE_SET_ALIGN_PERCENT, 100); + else + r = table_add_many( + t, + TABLE_PERCENT, (int) (progress * 100), + TABLE_SET_ALIGN_PERCENT, 100); + if (r < 0) + return table_log_add_error(r); + r = table_add_many( + t, + TABLE_STRING, type, + TABLE_STRING, local, + TABLE_STRING, remote, + TABLE_SET_URL, remote); + if (r < 0) + return table_log_add_error(r); } - if (r < 0) - return bus_log_parse_error(r); r = sd_bus_message_exit_container(reply); if (r < 0) return bus_log_parse_error(r); - typesafe_qsort(transfers, n_transfers, compare_transfer_info); - - if (arg_legend && n_transfers > 0) - printf("%-*s %-*s %-*s %-*s %-*s\n", - (int) MAX(2U, DECIMAL_STR_WIDTH(max_id)), "ID", - (int) 7, "PERCENT", - (int) max_type, "TYPE", - (int) max_local, "LOCAL", - (int) max_remote, "REMOTE"); - - for (size_t j = 0; j < n_transfers; j++) - - if (transfers[j].progress < 0) - printf("%*" PRIu32 " %*s %-*s %-*s %-*s\n", - (int) MAX(2U, DECIMAL_STR_WIDTH(max_id)), transfers[j].id, - (int) 7, "n/a", - (int) max_type, transfers[j].type, - (int) max_local, transfers[j].local, - (int) max_remote, transfers[j].remote); - else - printf("%*" PRIu32 " %*u%% %-*s %-*s %-*s\n", - (int) MAX(2U, DECIMAL_STR_WIDTH(max_id)), transfers[j].id, - (int) 6, (unsigned) (transfers[j].progress * 100), - (int) max_type, transfers[j].type, - (int) max_local, transfers[j].local, - (int) max_remote, transfers[j].remote); + if (!table_isempty(t)) { + r = table_print_with_pager(t, arg_json_format_flags, arg_pager_flags, arg_legend); + if (r < 0) + return log_error_errno(r, "Failed to output table: %m"); + } if (arg_legend) { - if (n_transfers > 0) - printf("\n%zu transfers listed.\n", n_transfers); + if (!table_isempty(t)) + printf("\n%zu transfers listed.\n", table_get_rows(t) - 1); else printf("No transfers.\n"); } @@ -698,6 +677,9 @@ static int help(int argc, char *argv[], void *userdata) { " -M --machine=CONTAINER Operate on local container\n" " --read-only Create read-only bind mount\n" " -q --quiet Suppress output\n" + " --json=pretty|short|off Generate JSON output\n" + " -j Equvilant to --json=pretty on TTY, --json=short\n" + " otherwise\n" " --verify=MODE Verification mode for downloaded images (no,\n" " checksum, signature)\n" " --format=xz|gzip|bzip2 Desired output format for export\n" @@ -721,6 +703,7 @@ static int parse_argv(int argc, char *argv[]) { ARG_NO_LEGEND, ARG_NO_ASK_PASSWORD, ARG_READ_ONLY, + ARG_JSON, ARG_VERIFY, ARG_FORCE, ARG_FORMAT, @@ -735,6 +718,7 @@ static int parse_argv(int argc, char *argv[]) { { "host", required_argument, NULL, 'H' }, { "machine", required_argument, NULL, 'M' }, { "read-only", no_argument, NULL, ARG_READ_ONLY }, + { "json", required_argument, NULL, ARG_JSON }, { "quiet", no_argument, NULL, 'q' }, { "verify", required_argument, NULL, ARG_VERIFY }, { "force", no_argument, NULL, ARG_FORCE }, @@ -748,7 +732,7 @@ static int parse_argv(int argc, char *argv[]) { assert(argv); for (;;) { - c = getopt_long(argc, argv, "hH:M:q", options, NULL); + c = getopt_long(argc, argv, "hH:M:jq", options, NULL); if (c < 0) break; @@ -814,6 +798,19 @@ static int parse_argv(int argc, char *argv[]) { arg_format = optarg; break; + case ARG_JSON: + r = parse_json_argument(optarg, &arg_json_format_flags); + if (r <= 0) + return r; + + arg_legend = false; + break; + + case 'j': + arg_json_format_flags = JSON_FORMAT_PRETTY_AUTO|JSON_FORMAT_COLOR_AUTO; + arg_legend = false; + break; + case '?': return -EINVAL; From 83d74112096d6af390b1c86ca877f537539e2a4a Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 22 Feb 2024 13:32:35 +0100 Subject: [PATCH 10/26] import: merge PullFlags enum into ImportFlags The two enums are mostly the same, the former is just an extension of the latter. Let's merge them, to simplify things. This is particularly useful as we then can reuse this systematically as D-Bus method call flags too, in a generic fashion that works for both imports and pulls the same. Pretty much just renaming of flags. --- src/import/import-common.h | 29 ++++++++++++------ src/import/pull-common.c | 4 +-- src/import/pull-common.h | 21 ++----------- src/import/pull-raw.c | 62 +++++++++++++++++++------------------- src/import/pull-raw.h | 2 +- src/import/pull-tar.c | 54 ++++++++++++++++----------------- src/import/pull-tar.h | 2 +- src/import/pull.c | 54 ++++++++++++++++----------------- 8 files changed, 111 insertions(+), 117 deletions(-) diff --git a/src/import/import-common.h b/src/import/import-common.h index 97fc16d1b9..edde7fd8bd 100644 --- a/src/import/import-common.h +++ b/src/import/import-common.h @@ -6,16 +6,27 @@ #include "sd-event.h" typedef enum ImportFlags { - IMPORT_FORCE = 1 << 0, /* replace existing image */ - IMPORT_READ_ONLY = 1 << 1, /* make generated image read-only */ - IMPORT_BTRFS_SUBVOL = 1 << 2, /* tar: preferably create images as btrfs subvols */ - IMPORT_BTRFS_QUOTA = 1 << 3, /* tar: set up btrfs quota for new subvolume as child of parent subvolume */ - IMPORT_CONVERT_QCOW2 = 1 << 4, /* raw: if we detect a qcow2 image, unpack it */ - IMPORT_DIRECT = 1 << 5, /* import without rename games */ - IMPORT_SYNC = 1 << 6, /* fsync() right before we are done */ + IMPORT_FORCE = 1 << 0, /* replace existing image */ + IMPORT_READ_ONLY = 1 << 1, /* make generated image read-only */ + IMPORT_BTRFS_SUBVOL = 1 << 2, /* tar: preferably create images as btrfs subvols */ + IMPORT_BTRFS_QUOTA = 1 << 3, /* tar: set up btrfs quota for new subvolume as child of parent subvolume */ + IMPORT_CONVERT_QCOW2 = 1 << 4, /* raw: if we detect a qcow2 image, unpack it */ + IMPORT_DIRECT = 1 << 5, /* import without rename games */ + IMPORT_SYNC = 1 << 6, /* fsync() right before we are done */ - IMPORT_FLAGS_MASK_TAR = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_BTRFS_SUBVOL|IMPORT_BTRFS_QUOTA|IMPORT_DIRECT|IMPORT_SYNC, - IMPORT_FLAGS_MASK_RAW = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_CONVERT_QCOW2|IMPORT_DIRECT|IMPORT_SYNC, + /* When pulling these flags are defined too */ + IMPORT_PULL_SETTINGS = 1 << 7, /* download .nspawn settings file */ + IMPORT_PULL_ROOTHASH = 1 << 8, /* only for raw: download .roothash file for verity */ + IMPORT_PULL_ROOTHASH_SIGNATURE = 1 << 9, /* only for raw: download .roothash.p7s file for verity */ + IMPORT_PULL_VERITY = 1 << 10, /* only for raw: download .verity file for verity */ + + /* The supported flags for the tar and the raw importing */ + IMPORT_FLAGS_MASK_TAR = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_BTRFS_SUBVOL|IMPORT_BTRFS_QUOTA|IMPORT_DIRECT|IMPORT_SYNC, + IMPORT_FLAGS_MASK_RAW = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_CONVERT_QCOW2|IMPORT_DIRECT|IMPORT_SYNC, + + /* The supported flags for the tar and the raw pulling */ + IMPORT_PULL_FLAGS_MASK_TAR = IMPORT_FLAGS_MASK_TAR|IMPORT_PULL_SETTINGS, + IMPORT_PULL_FLAGS_MASK_RAW = IMPORT_FLAGS_MASK_RAW|IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY, } ImportFlags; int import_fork_tar_c(const char *path, pid_t *ret); diff --git a/src/import/pull-common.c b/src/import/pull-common.c index 37387a76e0..7839057769 100644 --- a/src/import/pull-common.c +++ b/src/import/pull-common.c @@ -643,9 +643,9 @@ int pull_job_restart_with_sha256sum(PullJob *j, char **ret) { return 1; } -bool pull_validate_local(const char *name, PullFlags flags) { +bool pull_validate_local(const char *name, ImportFlags flags) { - if (FLAGS_SET(flags, PULL_DIRECT)) + if (FLAGS_SET(flags, IMPORT_DIRECT)) return path_is_valid(name); return hostname_is_valid(name, 0); diff --git a/src/import/pull-common.h b/src/import/pull-common.h index 475613a907..82e25267e3 100644 --- a/src/import/pull-common.h +++ b/src/import/pull-common.h @@ -3,27 +3,10 @@ #include +#include "import-common.h" #include "import-util.h" #include "pull-job.h" -typedef enum PullFlags { - PULL_FORCE = 1 << 0, /* replace existing image */ - PULL_READ_ONLY = 1 << 1, /* make generated image read-only */ - PULL_SETTINGS = 1 << 2, /* download .nspawn settings file */ - PULL_ROOTHASH = 1 << 3, /* only for raw: download .roothash file for verity */ - PULL_ROOTHASH_SIGNATURE = 1 << 4, /* only for raw: download .roothash.p7s file for verity */ - PULL_VERITY = 1 << 5, /* only for raw: download .verity file for verity */ - PULL_BTRFS_SUBVOL = 1 << 6, /* tar: preferably create images as btrfs subvols */ - PULL_BTRFS_QUOTA = 1 << 7, /* tar: set up btrfs quota for new subvolume as child of parent subvolume */ - PULL_CONVERT_QCOW2 = 1 << 8, /* raw: if we detect a qcow2 image, unpack it */ - PULL_DIRECT = 1 << 9, /* download without rename games */ - PULL_SYNC = 1 << 10, /* fsync() right before we are done */ - - /* The supported flags for the tar and the raw pulling */ - PULL_FLAGS_MASK_TAR = PULL_FORCE|PULL_READ_ONLY|PULL_SETTINGS|PULL_BTRFS_SUBVOL|PULL_BTRFS_QUOTA|PULL_DIRECT|PULL_SYNC, - PULL_FLAGS_MASK_RAW = PULL_FORCE|PULL_READ_ONLY|PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY|PULL_CONVERT_QCOW2|PULL_DIRECT|PULL_SYNC, -} PullFlags; - int pull_find_old_etags(const char *url, const char *root, int dt, const char *prefix, const char *suffix, char ***etags); int pull_make_path(const char *url, const char *etag, const char *image_root, const char *prefix, const char *suffix, char **ret); @@ -44,6 +27,6 @@ int verification_style_from_url(const char *url, VerificationStyle *style); int pull_job_restart_with_sha256sum(PullJob *job, char **ret); -bool pull_validate_local(const char *name, PullFlags flags); +bool pull_validate_local(const char *name, ImportFlags flags); int pull_url_needs_checksum(const char *url); diff --git a/src/import/pull-raw.c b/src/import/pull-raw.c index dc5434cd2d..ebe479fb80 100644 --- a/src/import/pull-raw.c +++ b/src/import/pull-raw.c @@ -42,7 +42,7 @@ struct RawPull { sd_event *event; CurlGlue *glue; - PullFlags flags; + ImportFlags flags; ImportVerify verify; char *image_root; @@ -244,9 +244,9 @@ static int raw_pull_maybe_convert_qcow2(RawPull *i) { assert(i); assert(i->raw_job); - assert(!FLAGS_SET(i->flags, PULL_DIRECT)); + assert(!FLAGS_SET(i->flags, IMPORT_DIRECT)); - if (!FLAGS_SET(i->flags, PULL_CONVERT_QCOW2)) + if (!FLAGS_SET(i->flags, IMPORT_CONVERT_QCOW2)) return 0; assert(i->final_path); @@ -328,8 +328,8 @@ static int raw_pull_copy_auxiliary_file( local, 0644, COPY_REFLINK | - (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0) | - (FLAGS_SET(i->flags, PULL_SYNC) ? COPY_FSYNC_FULL : 0)); + (FLAGS_SET(i->flags, IMPORT_FORCE) ? COPY_REPLACE : 0) | + (FLAGS_SET(i->flags, IMPORT_SYNC) ? COPY_FSYNC_FULL : 0)); if (r == -EEXIST) log_warning_errno(r, "File %s already exists, not replacing.", local); else if (r == -ENOENT) @@ -351,7 +351,7 @@ static int raw_pull_make_local_copy(RawPull *i) { assert(i); assert(i->raw_job); - assert(!FLAGS_SET(i->flags, PULL_DIRECT)); + assert(!FLAGS_SET(i->flags, IMPORT_DIRECT)); if (!i->local) return 0; @@ -401,9 +401,9 @@ static int raw_pull_make_local_copy(RawPull *i) { r = install_file(AT_FDCWD, tp, AT_FDCWD, p, - (i->flags & PULL_FORCE ? INSTALL_REPLACE : 0) | - (i->flags & PULL_READ_ONLY ? INSTALL_READ_ONLY : 0) | - (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0)); + (i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) | + (i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) | + (i->flags & IMPORT_SYNC ? INSTALL_FSYNC_FULL : 0)); if (r < 0) return log_error_errno(errno, "Failed to move local image into place '%s': %m", p); @@ -411,25 +411,25 @@ static int raw_pull_make_local_copy(RawPull *i) { log_info("Created new local image '%s'.", i->local); - if (FLAGS_SET(i->flags, PULL_SETTINGS)) { + if (FLAGS_SET(i->flags, IMPORT_PULL_SETTINGS)) { r = raw_pull_copy_auxiliary_file(i, ".nspawn", &i->settings_path); if (r < 0) return r; } - if (FLAGS_SET(i->flags, PULL_ROOTHASH)) { + if (FLAGS_SET(i->flags, IMPORT_PULL_ROOTHASH)) { r = raw_pull_copy_auxiliary_file(i, ".roothash", &i->roothash_path); if (r < 0) return r; } - if (FLAGS_SET(i->flags, PULL_ROOTHASH_SIGNATURE)) { + if (FLAGS_SET(i->flags, IMPORT_PULL_ROOTHASH_SIGNATURE)) { r = raw_pull_copy_auxiliary_file(i, ".roothash.p7s", &i->roothash_signature_path); if (r < 0) return r; } - if (FLAGS_SET(i->flags, PULL_VERITY)) { + if (FLAGS_SET(i->flags, IMPORT_PULL_VERITY)) { r = raw_pull_copy_auxiliary_file(i, ".verity", &i->verity_path); if (r < 0) return r; @@ -485,7 +485,7 @@ static int raw_pull_rename_auxiliary_file( AT_FDCWD, *temp_path, AT_FDCWD, *path, INSTALL_READ_ONLY| - (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0)); + (i->flags & IMPORT_SYNC ? INSTALL_FSYNC_FULL : 0)); if (r < 0) return log_error_errno(r, "Failed to move '%s' into place: %m", *path); @@ -588,7 +588,7 @@ static void raw_pull_job_on_finished(PullJob *j) { goto finish; } - if (i->flags & PULL_DIRECT) { + if (i->flags & IMPORT_DIRECT) { assert(!i->settings_job); assert(!i->roothash_job); assert(!i->roothash_signature_job); @@ -599,8 +599,8 @@ static void raw_pull_job_on_finished(PullJob *j) { if (i->local) { r = install_file(AT_FDCWD, i->local, AT_FDCWD, NULL, - ((i->flags & PULL_READ_ONLY) && i->offset == UINT64_MAX ? INSTALL_READ_ONLY : 0) | - (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0)); + ((i->flags & IMPORT_READ_ONLY) && i->offset == UINT64_MAX ? INSTALL_READ_ONLY : 0) | + (i->flags & IMPORT_SYNC ? INSTALL_FSYNC_FULL : 0)); if (r < 0) { log_error_errno(r, "Failed to finalize raw file to '%s': %m", i->local); goto finish; @@ -628,7 +628,7 @@ static void raw_pull_job_on_finished(PullJob *j) { r = install_file(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path, INSTALL_READ_ONLY| - (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0)); + (i->flags & IMPORT_SYNC ? INSTALL_FSYNC_FULL : 0)); if (r < 0) { log_error_errno(r, "Failed to move raw file to '%s': %m", i->final_path); goto finish; @@ -694,7 +694,7 @@ static int raw_pull_job_on_open_disk_generic( assert(extra); assert(temp_path); - assert(!FLAGS_SET(i->flags, PULL_DIRECT)); + assert(!FLAGS_SET(i->flags, IMPORT_DIRECT)); if (!*temp_path) { r = tempfn_random_child(i->image_root, extra, temp_path); @@ -722,7 +722,7 @@ static int raw_pull_job_on_open_disk_raw(PullJob *j) { assert(i->raw_job == j); assert(j->disk_fd < 0); - if (i->flags & PULL_DIRECT) { + if (i->flags & IMPORT_DIRECT) { if (!i->local) { /* If no local name specified, the pull job will write its data to stdout */ j->disk_fd = STDOUT_FILENO; @@ -816,7 +816,7 @@ int raw_pull_start( const char *local, uint64_t offset, uint64_t size_max, - PullFlags flags, + ImportFlags flags, ImportVerify verify, const char *checksum) { @@ -827,10 +827,10 @@ int raw_pull_start( assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX); assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0); assert((verify < 0) || !checksum); - assert(!(flags & ~PULL_FLAGS_MASK_RAW)); - assert(offset == UINT64_MAX || FLAGS_SET(flags, PULL_DIRECT)); - assert(!(flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) || !(flags & PULL_DIRECT)); - assert(!(flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) || !checksum); + assert(!(flags & ~IMPORT_PULL_FLAGS_MASK_RAW)); + assert(offset == UINT64_MAX || FLAGS_SET(flags, IMPORT_DIRECT)); + assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !(flags & IMPORT_DIRECT)); + assert(!(flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) || !checksum); if (!http_url_is_valid(url) && !file_url_is_valid(url)) return -EINVAL; @@ -880,7 +880,7 @@ int raw_pull_start( if (offset != UINT64_MAX) i->raw_job->offset = i->offset = offset; - if (!FLAGS_SET(flags, PULL_DIRECT)) { + if (!FLAGS_SET(flags, IMPORT_DIRECT)) { r = pull_find_old_etags(url, i->image_root, DT_REG, ".raw-", ".raw", &i->raw_job->old_etags); if (r < 0) return r; @@ -898,7 +898,7 @@ int raw_pull_start( if (r < 0) return r; - if (FLAGS_SET(flags, PULL_SETTINGS)) { + if (FLAGS_SET(flags, IMPORT_PULL_SETTINGS)) { r = pull_make_auxiliary_job( &i->settings_job, url, @@ -913,7 +913,7 @@ int raw_pull_start( return r; } - if (FLAGS_SET(flags, PULL_ROOTHASH)) { + if (FLAGS_SET(flags, IMPORT_PULL_ROOTHASH)) { r = pull_make_auxiliary_job( &i->roothash_job, url, @@ -928,7 +928,7 @@ int raw_pull_start( return r; } - if (FLAGS_SET(flags, PULL_ROOTHASH_SIGNATURE)) { + if (FLAGS_SET(flags, IMPORT_PULL_ROOTHASH_SIGNATURE)) { r = pull_make_auxiliary_job( &i->roothash_signature_job, url, @@ -943,7 +943,7 @@ int raw_pull_start( return r; } - if (FLAGS_SET(flags, PULL_VERITY)) { + if (FLAGS_SET(flags, IMPORT_PULL_VERITY)) { r = pull_make_auxiliary_job( &i->verity_job, url, @@ -972,7 +972,7 @@ int raw_pull_start( continue; j->on_progress = raw_pull_job_on_progress; - j->sync = FLAGS_SET(flags, PULL_SYNC); + j->sync = FLAGS_SET(flags, IMPORT_SYNC); r = pull_job_begin(j); if (r < 0) diff --git a/src/import/pull-raw.h b/src/import/pull-raw.h index b39e4e257c..e50ba326f6 100644 --- a/src/import/pull-raw.h +++ b/src/import/pull-raw.h @@ -16,4 +16,4 @@ RawPull* raw_pull_unref(RawPull *pull); DEFINE_TRIVIAL_CLEANUP_FUNC(RawPull*, raw_pull_unref); -int raw_pull_start(RawPull *pull, const char *url, const char *local, uint64_t offset, uint64_t size_max, PullFlags flags, ImportVerify verify, const char *checksum); +int raw_pull_start(RawPull *pull, const char *url, const char *local, uint64_t offset, uint64_t size_max, ImportFlags flags, ImportVerify verify, const char *checksum); diff --git a/src/import/pull-tar.c b/src/import/pull-tar.c index a031449560..99d8ff0553 100644 --- a/src/import/pull-tar.c +++ b/src/import/pull-tar.c @@ -41,7 +41,7 @@ struct TarPull { sd_event *event; CurlGlue *glue; - PullFlags flags; + ImportFlags flags; ImportVerify verify; char *image_root; @@ -238,11 +238,11 @@ static int tar_pull_make_local_copy(TarPull *i) { if (r < 0) return log_error_errno(r, "Failed to generate temporary filename for %s: %m", p); - if (i->flags & PULL_BTRFS_SUBVOL) + if (i->flags & IMPORT_BTRFS_SUBVOL) r = btrfs_subvol_snapshot_at( AT_FDCWD, i->final_path, AT_FDCWD, t, - (i->flags & PULL_BTRFS_QUOTA ? BTRFS_SNAPSHOT_QUOTA : 0)| + (i->flags & IMPORT_BTRFS_QUOTA ? BTRFS_SNAPSHOT_QUOTA : 0)| BTRFS_SNAPSHOT_FALLBACK_COPY| BTRFS_SNAPSHOT_FALLBACK_DIRECTORY| BTRFS_SNAPSHOT_RECURSIVE); @@ -253,9 +253,9 @@ static int tar_pull_make_local_copy(TarPull *i) { r = install_file(AT_FDCWD, t, AT_FDCWD, p, - (i->flags & PULL_FORCE ? INSTALL_REPLACE : 0) | - (i->flags & PULL_READ_ONLY ? INSTALL_READ_ONLY : 0) | - (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0)); + (i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) | + (i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) | + (i->flags & IMPORT_SYNC ? INSTALL_SYNCFS : 0)); if (r < 0) return log_error_errno(r, "Failed to install local image '%s': %m", p); @@ -263,7 +263,7 @@ static int tar_pull_make_local_copy(TarPull *i) { log_info("Created new local image '%s'.", i->local); - if (FLAGS_SET(i->flags, PULL_SETTINGS)) { + if (FLAGS_SET(i->flags, IMPORT_PULL_SETTINGS)) { const char *local_settings; assert(i->settings_job); @@ -278,8 +278,8 @@ static int tar_pull_make_local_copy(TarPull *i) { local_settings, 0664, COPY_REFLINK | - (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0) | - (FLAGS_SET(i->flags, PULL_SYNC) ? COPY_FSYNC_FULL : 0)); + (FLAGS_SET(i->flags, IMPORT_FORCE) ? COPY_REPLACE : 0) | + (FLAGS_SET(i->flags, IMPORT_SYNC) ? COPY_FSYNC_FULL : 0)); if (r == -EEXIST) log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings); else if (r == -ENOENT) @@ -394,7 +394,7 @@ static void tar_pull_job_on_finished(PullJob *j) { goto finish; } - if (i->flags & PULL_DIRECT) { + if (i->flags & IMPORT_DIRECT) { assert(!i->settings_job); assert(i->local); assert(!i->temp_path); @@ -408,8 +408,8 @@ static void tar_pull_job_on_finished(PullJob *j) { r = install_file( AT_FDCWD, i->local, AT_FDCWD, NULL, - (i->flags & PULL_READ_ONLY) ? INSTALL_READ_ONLY : 0 | - (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0)); + (i->flags & IMPORT_READ_ONLY) ? INSTALL_READ_ONLY : 0 | + (i->flags & IMPORT_SYNC ? INSTALL_SYNCFS : 0)); if (r < 0) { log_error_errno(r, "Failed to finalize '%s': %m", i->local); goto finish; @@ -435,7 +435,7 @@ static void tar_pull_job_on_finished(PullJob *j) { AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path, INSTALL_READ_ONLY| - (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0)); + (i->flags & IMPORT_SYNC ? INSTALL_SYNCFS : 0)); if (r < 0) { log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path); goto finish; @@ -462,7 +462,7 @@ static void tar_pull_job_on_finished(PullJob *j) { AT_FDCWD, i->settings_temp_path, AT_FDCWD, i->settings_path, INSTALL_READ_ONLY| - (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0)); + (i->flags & IMPORT_SYNC ? INSTALL_FSYNC_FULL : 0)); if (r < 0) { log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path); goto finish; @@ -500,7 +500,7 @@ static int tar_pull_job_on_open_disk_tar(PullJob *j) { assert(i->tar_job == j); assert(i->tar_pid <= 0); - if (i->flags & PULL_DIRECT) + if (i->flags & IMPORT_DIRECT) where = i->local; else { if (!i->temp_path) { @@ -514,20 +514,20 @@ static int tar_pull_job_on_open_disk_tar(PullJob *j) { (void) mkdir_parents_label(where, 0700); - if (FLAGS_SET(i->flags, PULL_DIRECT|PULL_FORCE)) + if (FLAGS_SET(i->flags, IMPORT_DIRECT|IMPORT_FORCE)) (void) rm_rf(where, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME); - if (i->flags & PULL_BTRFS_SUBVOL) + if (i->flags & IMPORT_BTRFS_SUBVOL) r = btrfs_subvol_make_fallback(AT_FDCWD, where, 0755); else r = RET_NERRNO(mkdir(where, 0755)); - if (r == -EEXIST && (i->flags & PULL_DIRECT)) /* EEXIST is OK if in direct mode, but not otherwise, + if (r == -EEXIST && (i->flags & IMPORT_DIRECT)) /* EEXIST is OK if in direct mode, but not otherwise, * because in that case our temporary path collided */ r = 0; if (r < 0) return log_error_errno(r, "Failed to create directory/subvolume %s: %m", where); - if (r > 0 && (i->flags & PULL_BTRFS_QUOTA)) { /* actually btrfs subvol */ - if (!(i->flags & PULL_DIRECT)) + if (r > 0 && (i->flags & IMPORT_BTRFS_QUOTA)) { /* actually btrfs subvol */ + if (!(i->flags & IMPORT_DIRECT)) (void) import_assign_pool_quota_and_warn(i->image_root); (void) import_assign_pool_quota_and_warn(where); } @@ -579,7 +579,7 @@ int tar_pull_start( TarPull *i, const char *url, const char *local, - PullFlags flags, + ImportFlags flags, ImportVerify verify, const char *checksum) { @@ -589,9 +589,9 @@ int tar_pull_start( assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX); assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0); assert((verify < 0) || !checksum); - assert(!(flags & ~PULL_FLAGS_MASK_TAR)); - assert(!(flags & PULL_SETTINGS) || !(flags & PULL_DIRECT)); - assert(!(flags & PULL_SETTINGS) || !checksum); + assert(!(flags & ~IMPORT_PULL_FLAGS_MASK_TAR)); + assert(!(flags & IMPORT_PULL_SETTINGS) || !(flags & IMPORT_DIRECT)); + assert(!(flags & IMPORT_PULL_SETTINGS) || !checksum); if (!http_url_is_valid(url) && !file_url_is_valid(url)) return -EINVAL; @@ -622,7 +622,7 @@ int tar_pull_start( i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar; i->tar_job->calc_checksum = checksum || IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE); - if (!FLAGS_SET(flags, PULL_DIRECT)) { + if (!FLAGS_SET(flags, IMPORT_DIRECT)) { r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags); if (r < 0) return r; @@ -642,7 +642,7 @@ int tar_pull_start( return r; /* Set up download job for the settings file (.nspawn) */ - if (FLAGS_SET(flags, PULL_SETTINGS)) { + if (FLAGS_SET(flags, IMPORT_PULL_SETTINGS)) { r = pull_make_auxiliary_job( &i->settings_job, url, @@ -668,7 +668,7 @@ int tar_pull_start( continue; j->on_progress = tar_pull_job_on_progress; - j->sync = FLAGS_SET(flags, PULL_SYNC); + j->sync = FLAGS_SET(flags, IMPORT_SYNC); r = pull_job_begin(j); if (r < 0) diff --git a/src/import/pull-tar.h b/src/import/pull-tar.h index e54c01c302..1a5b740463 100644 --- a/src/import/pull-tar.h +++ b/src/import/pull-tar.h @@ -16,4 +16,4 @@ TarPull* tar_pull_unref(TarPull *pull); DEFINE_TRIVIAL_CLEANUP_FUNC(TarPull*, tar_pull_unref); -int tar_pull_start(TarPull *pull, const char *url, const char *local, PullFlags flags, ImportVerify verify, const char *checksum); +int tar_pull_start(TarPull *pull, const char *url, const char *local, ImportFlags flags, ImportVerify verify, const char *checksum); diff --git a/src/import/pull.c b/src/import/pull.c index a2afc6f390..518755b61d 100644 --- a/src/import/pull.c +++ b/src/import/pull.c @@ -28,7 +28,7 @@ static const char *arg_image_root = "/var/lib/machines"; static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE; -static PullFlags arg_pull_flags = PULL_SETTINGS | PULL_ROOTHASH | PULL_ROOTHASH_SIGNATURE | PULL_VERITY | PULL_BTRFS_SUBVOL | PULL_BTRFS_QUOTA | PULL_CONVERT_QCOW2 | PULL_SYNC; +static ImportFlags arg_import_flags = IMPORT_PULL_SETTINGS | IMPORT_PULL_ROOTHASH | IMPORT_PULL_ROOTHASH_SIGNATURE | IMPORT_PULL_VERITY | IMPORT_BTRFS_SUBVOL | IMPORT_BTRFS_QUOTA | IMPORT_CONVERT_QCOW2 | IMPORT_SYNC; static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX; static char *arg_checksum = NULL; @@ -38,7 +38,7 @@ static int normalize_local(const char *local, const char *url, char **ret) { _cleanup_free_ char *ll = NULL; int r; - if (arg_pull_flags & PULL_DIRECT) { + if (arg_import_flags & IMPORT_DIRECT) { if (!local) log_debug("Writing downloaded data to STDOUT."); @@ -63,7 +63,7 @@ static int normalize_local(const char *local, const char *url, char **ret) { "Local image name '%s' is not valid.", local); - if (!FLAGS_SET(arg_pull_flags, PULL_FORCE)) { + if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) { r = image_find(IMAGE_MACHINE, local, NULL, NULL); if (r < 0) { if (r != -ENOENT) @@ -130,7 +130,7 @@ static int pull_tar(int argc, char *argv[], void *userdata) { local = ll; } - if (!local && FLAGS_SET(arg_pull_flags, PULL_DIRECT)) + if (!local && FLAGS_SET(arg_import_flags, IMPORT_DIRECT)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Pulling tar images to STDOUT is not supported."); r = normalize_local(local, url, &normalized); @@ -141,7 +141,7 @@ static int pull_tar(int argc, char *argv[], void *userdata) { if (r < 0) return r; - if (!FLAGS_SET(arg_pull_flags, PULL_SYNC)) + if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC)) log_info("File system synchronization on completion is off."); r = tar_pull_new(&pull, event, arg_image_root, on_tar_finished, event); @@ -152,7 +152,7 @@ static int pull_tar(int argc, char *argv[], void *userdata) { pull, url, normalized, - arg_pull_flags & PULL_FLAGS_MASK_TAR, + arg_import_flags & IMPORT_PULL_FLAGS_MASK_TAR, arg_verify, arg_checksum); if (r < 0) @@ -211,7 +211,7 @@ static int pull_raw(int argc, char *argv[], void *userdata) { if (r < 0) return r; - if (!FLAGS_SET(arg_pull_flags, PULL_SYNC)) + if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC)) log_info("File system synchronization on completion is off."); r = raw_pull_new(&pull, event, arg_image_root, on_raw_finished, event); @@ -224,7 +224,7 @@ static int pull_raw(int argc, char *argv[], void *userdata) { normalized, arg_offset, arg_size_max, - arg_pull_flags & PULL_FLAGS_MASK_RAW, + arg_import_flags & IMPORT_PULL_FLAGS_MASK_RAW, arg_verify, arg_checksum); if (r < 0) @@ -335,7 +335,7 @@ static int parse_argv(int argc, char *argv[]) { return version(); case ARG_FORCE: - arg_pull_flags |= PULL_FORCE; + arg_import_flags |= IMPORT_FORCE; break; case ARG_IMAGE_ROOT: @@ -367,7 +367,7 @@ static int parse_argv(int argc, char *argv[]) { return log_oom(); free_and_replace(arg_checksum, hh); - arg_pull_flags &= ~(PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY); + arg_import_flags &= ~(IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY); arg_verify = _IMPORT_VERIFY_INVALID; } else arg_verify = v; @@ -380,7 +380,7 @@ static int parse_argv(int argc, char *argv[]) { if (r < 0) return r; - SET_FLAG(arg_pull_flags, PULL_SETTINGS, r); + SET_FLAG(arg_import_flags, IMPORT_PULL_SETTINGS, r); break; case ARG_ROOTHASH: @@ -388,11 +388,11 @@ static int parse_argv(int argc, char *argv[]) { if (r < 0) return r; - SET_FLAG(arg_pull_flags, PULL_ROOTHASH, r); + SET_FLAG(arg_import_flags, IMPORT_PULL_ROOTHASH, r); /* If we were asked to turn off the root hash, implicitly also turn off the root hash signature */ if (!r) - SET_FLAG(arg_pull_flags, PULL_ROOTHASH_SIGNATURE, false); + SET_FLAG(arg_import_flags, IMPORT_PULL_ROOTHASH_SIGNATURE, false); break; case ARG_ROOTHASH_SIGNATURE: @@ -400,7 +400,7 @@ static int parse_argv(int argc, char *argv[]) { if (r < 0) return r; - SET_FLAG(arg_pull_flags, PULL_ROOTHASH_SIGNATURE, r); + SET_FLAG(arg_import_flags, IMPORT_PULL_ROOTHASH_SIGNATURE, r); break; case ARG_VERITY: @@ -408,16 +408,16 @@ static int parse_argv(int argc, char *argv[]) { if (r < 0) return r; - SET_FLAG(arg_pull_flags, PULL_VERITY, r); + SET_FLAG(arg_import_flags, IMPORT_PULL_VERITY, r); break; case ARG_READ_ONLY: - arg_pull_flags |= PULL_READ_ONLY; + arg_import_flags |= IMPORT_READ_ONLY; break; case ARG_DIRECT: - arg_pull_flags |= PULL_DIRECT; - arg_pull_flags &= ~(PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY); + arg_import_flags |= IMPORT_DIRECT; + arg_import_flags &= ~(IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY); break; case ARG_BTRFS_SUBVOL: @@ -425,7 +425,7 @@ static int parse_argv(int argc, char *argv[]) { if (r < 0) return r; - SET_FLAG(arg_pull_flags, PULL_BTRFS_SUBVOL, r); + SET_FLAG(arg_import_flags, IMPORT_BTRFS_SUBVOL, r); break; case ARG_BTRFS_QUOTA: @@ -433,7 +433,7 @@ static int parse_argv(int argc, char *argv[]) { if (r < 0) return r; - SET_FLAG(arg_pull_flags, PULL_BTRFS_QUOTA, r); + SET_FLAG(arg_import_flags, IMPORT_BTRFS_QUOTA, r); break; case ARG_CONVERT_QCOW2: @@ -441,7 +441,7 @@ static int parse_argv(int argc, char *argv[]) { if (r < 0) return r; - SET_FLAG(arg_pull_flags, PULL_CONVERT_QCOW2, r); + SET_FLAG(arg_import_flags, IMPORT_CONVERT_QCOW2, r); break; case ARG_SYNC: @@ -449,7 +449,7 @@ static int parse_argv(int argc, char *argv[]) { if (r < 0) return r; - SET_FLAG(arg_pull_flags, PULL_SYNC, r); + SET_FLAG(arg_import_flags, IMPORT_SYNC, r); break; case ARG_OFFSET: { @@ -491,10 +491,10 @@ static int parse_argv(int argc, char *argv[]) { !FILE_SIZE_VALID(arg_offset + arg_size_max))) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset und maximum size out of range."); - if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_pull_flags, PULL_DIRECT)) + if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_import_flags, IMPORT_DIRECT)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode."); - if (arg_checksum && (arg_pull_flags & (PULL_SETTINGS|PULL_ROOTHASH|PULL_ROOTHASH_SIGNATURE|PULL_VERITY)) != 0) + if (arg_checksum && (arg_import_flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) != 0) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Literal checksum verification only supported if no associated files are downloaded."); return 1; @@ -508,19 +508,19 @@ static void parse_env(void) { r = getenv_bool("SYSTEMD_IMPORT_BTRFS_SUBVOL"); if (r >= 0) - SET_FLAG(arg_pull_flags, PULL_BTRFS_SUBVOL, r); + SET_FLAG(arg_import_flags, IMPORT_BTRFS_SUBVOL, r); else if (r != -ENXIO) log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_SUBVOL: %m"); r = getenv_bool("SYSTEMD_IMPORT_BTRFS_QUOTA"); if (r >= 0) - SET_FLAG(arg_pull_flags, PULL_BTRFS_QUOTA, r); + SET_FLAG(arg_import_flags, IMPORT_BTRFS_QUOTA, r); else if (r != -ENXIO) log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_BTRFS_QUOTA: %m"); r = getenv_bool("SYSTEMD_IMPORT_SYNC"); if (r >= 0) - SET_FLAG(arg_pull_flags, PULL_SYNC, r); + SET_FLAG(arg_import_flags, IMPORT_SYNC, r); else if (r != -ENXIO) log_warning_errno(r, "Failed to parse $SYSTEMD_IMPORT_SYNC: %m"); } From 7af5785d77ced29a01fdf636274ac883aa9ff2d0 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 22 Feb 2024 09:43:01 +0100 Subject: [PATCH 11/26] importd: add support for downloading sysext/confext/portable images too This adds "Ex" versions of all bus calls import implements, that make two changes: 1. A "class" parameter is added that allows choosing between machine/sysext/confext/portable images to download. Depending on the chose class the target directory is selected differently (i.e. not just /var/lib/machines/, but alternatively /var/lib/portables/, /var/lib/extensions/, /var/lib/confexts/. 2. The boolean flags are replaced by a 64bit flags parameter. --- src/import/export.c | 20 +- src/import/import-common.h | 2 + src/import/import-fs.c | 21 +- src/import/import-raw.c | 3 +- src/import/import-tar.c | 3 +- src/import/import.c | 23 +- src/import/importd.c | 397 ++++++++++++++++++---- src/import/org.freedesktop.import1.conf | 32 ++ src/import/org.freedesktop.import1.policy | 22 +- src/import/pull-common.c | 8 +- src/import/pull-raw.c | 5 +- src/import/pull-tar.c | 3 +- src/import/pull.c | 29 +- src/shared/discover-image.c | 9 + src/shared/discover-image.h | 2 + 15 files changed, 473 insertions(+), 106 deletions(-) diff --git a/src/import/export.c b/src/import/export.c index ee65cc8d36..ac81b2fbab 100644 --- a/src/import/export.c +++ b/src/import/export.c @@ -23,6 +23,7 @@ #include "verbs.h" static ImportCompressType arg_compress = IMPORT_COMPRESS_UNKNOWN; +static ImageClass arg_class = IMAGE_MACHINE; static void determine_compression_from_filename(const char *p) { @@ -63,7 +64,7 @@ static int export_tar(int argc, char *argv[], void *userdata) { int r, fd; if (hostname_is_valid(argv[1], 0)) { - r = image_find(IMAGE_MACHINE, argv[1], NULL, &image); + r = image_find(arg_class, argv[1], NULL, &image); if (r == -ENOENT) return log_error_errno(r, "Machine image %s not found.", argv[1]); if (r < 0) @@ -135,7 +136,7 @@ static int export_raw(int argc, char *argv[], void *userdata) { int r, fd; if (hostname_is_valid(argv[1], 0)) { - r = image_find(IMAGE_MACHINE, argv[1], NULL, &image); + r = image_find(arg_class, argv[1], NULL, &image); if (r == -ENOENT) return log_error_errno(r, "Machine image %s not found.", argv[1]); if (r < 0) @@ -190,14 +191,16 @@ static int export_raw(int argc, char *argv[], void *userdata) { static int help(int argc, char *argv[], void *userdata) { printf("%1$s [OPTIONS...] {COMMAND} ...\n" - "\n%4$sExport container or virtual machine images.%5$s\n" + "\n%4$sExport disk images.%5$s\n" "\n%2$sCommands:%3$s\n" " tar NAME [FILE] Export a TAR image\n" " raw NAME [FILE] Export a RAW image\n" "\n%2$sOptions:%3$s\n" " -h --help Show this help\n" " --version Show package version\n" - " --format=FORMAT Select format\n", + " --format=FORMAT Select format\n" + " --class=CLASS Select image class (machine, sysext, confext,\n" + " portable)\n", program_invocation_short_name, ansi_underline(), ansi_normal(), @@ -212,12 +215,14 @@ static int parse_argv(int argc, char *argv[]) { enum { ARG_VERSION = 0x100, ARG_FORMAT, + ARG_CLASS, }; static const struct option options[] = { { "help", no_argument, NULL, 'h' }, { "version", no_argument, NULL, ARG_VERSION }, { "format", required_argument, NULL, ARG_FORMAT }, + { "class", required_argument, NULL, ARG_CLASS }, {} }; @@ -250,6 +255,13 @@ static int parse_argv(int argc, char *argv[]) { "Unknown format: %s", optarg); break; + case ARG_CLASS: + arg_class = image_class_from_string(optarg); + if (arg_class < 0) + return log_error_errno(arg_class, "Failed to parse --class= argument: %s", optarg); + + break; + case '?': return -EINVAL; diff --git a/src/import/import-common.h b/src/import/import-common.h index edde7fd8bd..2bb2075462 100644 --- a/src/import/import-common.h +++ b/src/import/import-common.h @@ -27,6 +27,8 @@ typedef enum ImportFlags { /* The supported flags for the tar and the raw pulling */ IMPORT_PULL_FLAGS_MASK_TAR = IMPORT_FLAGS_MASK_TAR|IMPORT_PULL_SETTINGS, IMPORT_PULL_FLAGS_MASK_RAW = IMPORT_FLAGS_MASK_RAW|IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY, + + _IMPORT_FLAGS_INVALID = -EINVAL, } ImportFlags; int import_fork_tar_c(const char *path, pid_t *ret); diff --git a/src/import/import-fs.c b/src/import/import-fs.c index a89c835f57..e37fd7b41d 100644 --- a/src/import/import-fs.c +++ b/src/import/import-fs.c @@ -31,7 +31,8 @@ static bool arg_btrfs_subvol = true; static bool arg_btrfs_quota = true; static bool arg_sync = true; static bool arg_direct = false; -static const char *arg_image_root = "/var/lib/machines"; +static const char *arg_image_root = NULL; +static ImageClass arg_class = IMAGE_MACHINE; typedef struct ProgressInfo { RateLimit limit; @@ -143,7 +144,7 @@ static int import_fs(int argc, char *argv[], void *userdata) { return log_oom(); if (!arg_force) { - r = image_find(IMAGE_MACHINE, local, NULL, NULL); + r = image_find(arg_class, local, NULL, NULL); if (r < 0) { if (r != -ENOENT) return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); @@ -266,7 +267,9 @@ static int help(int argc, char *argv[], void *userdata) { " instead of a directory\n" " --btrfs-quota=BOOL Controls whether to set up quota for btrfs\n" " subvolume\n" - " --sync=BOOL Controls whether to sync() before completing\n", + " --sync=BOOL Controls whether to sync() before completing\n" + " --class=CLASS Select image class (machine, sysext, confext,\n" + " portable)\n", program_invocation_short_name, ansi_underline(), ansi_normal(), @@ -287,6 +290,7 @@ static int parse_argv(int argc, char *argv[]) { ARG_BTRFS_SUBVOL, ARG_BTRFS_QUOTA, ARG_SYNC, + ARG_CLASS, }; static const struct option options[] = { @@ -299,6 +303,7 @@ static int parse_argv(int argc, char *argv[]) { { "btrfs-subvol", required_argument, NULL, ARG_BTRFS_SUBVOL }, { "btrfs-quota", required_argument, NULL, ARG_BTRFS_QUOTA }, { "sync", required_argument, NULL, ARG_SYNC }, + { "class", required_argument, NULL, ARG_CLASS }, {} }; @@ -354,6 +359,13 @@ static int parse_argv(int argc, char *argv[]) { break; + case ARG_CLASS: + arg_class = image_class_from_string(optarg); + if (arg_class < 0) + return log_error_errno(arg_class, "Failed to parse --class= argument: %s", optarg); + + break; + case '?': return -EINVAL; @@ -361,6 +373,9 @@ static int parse_argv(int argc, char *argv[]) { assert_not_reached(); } + if (!arg_image_root) + arg_image_root = image_root_to_string(arg_class); + return 1; } diff --git a/src/import/import-raw.c b/src/import/import-raw.c index f7ed163d86..ee9b297bfe 100644 --- a/src/import/import-raw.c +++ b/src/import/import-raw.c @@ -95,8 +95,9 @@ int raw_import_new( int r; assert(ret); + assert(image_root); - root = strdup(image_root ?: "/var/lib/machines"); + root = strdup(image_root); if (!root) return -ENOMEM; diff --git a/src/import/import-tar.c b/src/import/import-tar.c index 90202709ec..39df11b5ff 100644 --- a/src/import/import-tar.c +++ b/src/import/import-tar.c @@ -97,8 +97,9 @@ int tar_import_new( int r; assert(ret); + assert(image_root); - root = strdup(image_root ?: "/var/lib/machines"); + root = strdup(image_root); if (!root) return -ENOMEM; diff --git a/src/import/import.c b/src/import/import.c index 50059870a0..22ce6f5aaa 100644 --- a/src/import/import.c +++ b/src/import/import.c @@ -25,9 +25,10 @@ #include "terminal-util.h" #include "verbs.h" -static const char *arg_image_root = "/var/lib/machines"; +static const char *arg_image_root = NULL; static ImportFlags arg_import_flags = IMPORT_BTRFS_SUBVOL | IMPORT_BTRFS_QUOTA | IMPORT_CONVERT_QCOW2 | IMPORT_SYNC; static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX; +static ImageClass arg_class = IMAGE_MACHINE; static int normalize_local(const char *local, char **ret) { _cleanup_free_ char *ll = NULL; @@ -61,7 +62,7 @@ static int normalize_local(const char *local, char **ret) { local = "imported"; if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) { - r = image_find(IMAGE_MACHINE, local, NULL, NULL); + r = image_find(arg_class, local, NULL, NULL); if (r < 0) { if (r != -ENOENT) return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); @@ -266,7 +267,7 @@ static int import_raw(int argc, char *argv[], void *userdata) { static int help(int argc, char *argv[], void *userdata) { printf("%1$s [OPTIONS...] {COMMAND} ...\n" - "\n%4$sImport container or virtual machine images.%5$s\n" + "\n%4$sImport disk images.%5$s\n" "\n%2$sCommands:%3$s\n" " tar FILE [NAME] Import a TAR image\n" " raw FILE [NAME] Import a RAW image\n" @@ -285,7 +286,9 @@ static int help(int argc, char *argv[], void *userdata) { " regular disk images\n" " --sync=BOOL Controls whether to sync() before completing\n" " --offset=BYTES Offset to seek to in destination\n" - " --size-max=BYTES Maximum number of bytes to write to destination\n", + " --size-max=BYTES Maximum number of bytes to write to destination\n" + " --class=CLASS Select image class (machine, sysext, confext,\n" + " portable)\n", program_invocation_short_name, ansi_underline(), ansi_normal(), @@ -309,6 +312,7 @@ static int parse_argv(int argc, char *argv[]) { ARG_SYNC, ARG_OFFSET, ARG_SIZE_MAX, + ARG_CLASS, }; static const struct option options[] = { @@ -324,6 +328,7 @@ static int parse_argv(int argc, char *argv[]) { { "sync", required_argument, NULL, ARG_SYNC }, { "offset", required_argument, NULL, ARG_OFFSET }, { "size-max", required_argument, NULL, ARG_SIZE_MAX }, + { "class", required_argument, NULL, ARG_CLASS }, {} }; @@ -416,6 +421,13 @@ static int parse_argv(int argc, char *argv[]) { break; } + case ARG_CLASS: + arg_class = image_class_from_string(optarg); + if (arg_class < 0) + return log_error_errno(arg_class, "Failed to parse --class= argument: %s", optarg); + + break; + case '?': return -EINVAL; @@ -432,6 +444,9 @@ static int parse_argv(int argc, char *argv[]) { if (arg_offset != UINT64_MAX && !FLAGS_SET(arg_import_flags, IMPORT_DIRECT)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "File offset only supported in --direct mode."); + if (!arg_image_root) + arg_image_root = image_root_to_string(arg_class); + return 1; } diff --git a/src/import/importd.c b/src/import/importd.c index dbf2fba4ab..47dfb2dfaf 100644 --- a/src/import/importd.c +++ b/src/import/importd.c @@ -14,16 +14,19 @@ #include "common-signal.h" #include "constants.h" #include "daemon-util.h" +#include "discover-image.h" #include "env-util.h" #include "event-util.h" #include "fd-util.h" #include "float.h" #include "hostname-util.h" +#include "import-common.h" #include "import-util.h" #include "machine-pool.h" #include "main-func.h" #include "missing_capability.h" #include "mkdir-label.h" +#include "os-util.h" #include "parse-util.h" #include "path-util.h" #include "percent-util.h" @@ -64,9 +67,8 @@ struct Transfer { char *remote; char *local; - bool force_local; - bool read_only; - + ImageClass class; + ImportFlags flags; char *format; PidRef pidref; @@ -382,6 +384,8 @@ static int transfer_start(Transfer *t) { NULL, /* tar, raw */ NULL, /* --verify= */ NULL, /* verify argument */ + NULL, /* --class= */ + NULL, /* class argument */ NULL, /* maybe --force */ NULL, /* maybe --read-only */ NULL, /* if so: the actual URL */ @@ -457,9 +461,14 @@ static int transfer_start(Transfer *t) { cmd[k++] = import_verify_to_string(t->verify); } - if (t->force_local) + if (t->class != IMAGE_MACHINE) { + cmd[k++] = "--class"; + cmd[k++] = image_class_to_string(t->class); + } + + if (FLAGS_SET(t->flags, IMPORT_FORCE)) cmd[k++] = "--force"; - if (t->read_only) + if (FLAGS_SET(t->flags, IMPORT_READ_ONLY)) cmd[k++] = "--read-only"; if (t->format) { @@ -702,12 +711,13 @@ static Transfer *manager_find(Manager *m, TransferType type, const char *remote) static int method_import_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_error *error) { _cleanup_(transfer_unrefp) Transfer *t = NULL; - int fd, force, read_only, r; - const char *local, *object; + ImageClass class = _IMAGE_CLASS_INVALID; Manager *m = ASSERT_PTR(userdata); + const char *local; TransferType type; struct stat st; - uint32_t id; + uint64_t flags; + int fd, r; assert(msg); @@ -722,9 +732,34 @@ static int method_import_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_ if (r == 0) return 1; /* Will call us back */ - r = sd_bus_message_read(msg, "hsbb", &fd, &local, &force, &read_only); - if (r < 0) - return r; + if (endswith(sd_bus_message_get_member(msg), "Ex")) { + const char *sclass; + + r = sd_bus_message_read(msg, "hsst", &fd, &local, &sclass, &flags); + if (r < 0) + return r; + + class = image_class_from_string(sclass); + if (class < 0) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Image class '%s' not known", sclass); + + if (flags & ~(IMPORT_READ_ONLY|IMPORT_FORCE)) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Flags 0x%" PRIx64 " invalid", flags); + } else { + int force, read_only; + + r = sd_bus_message_read(msg, "hsbb", &fd, &local, &force, &read_only); + if (r < 0) + return r; + + class = IMAGE_MACHINE; + + flags = 0; + SET_FLAG(flags, IMPORT_FORCE, force); + SET_FLAG(flags, IMPORT_READ_ONLY, read_only); + } if (fstat(fd, &st) < 0) return -errno; @@ -736,11 +771,13 @@ static int method_import_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_ return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Local name %s is invalid", local); - r = setup_machine_directory(error, m->use_btrfs_subvol, m->use_btrfs_quota); - if (r < 0) - return r; + if (class == IMAGE_MACHINE) { + r = setup_machine_directory(error, m->use_btrfs_subvol, m->use_btrfs_quota); + if (r < 0) + return r; + } - type = streq_ptr(sd_bus_message_get_member(msg), "ImportTar") ? + type = startswith(sd_bus_message_get_member(msg), "ImportTar") ? TRANSFER_IMPORT_TAR : TRANSFER_IMPORT_RAW; r = transfer_new(m, &t); @@ -748,8 +785,8 @@ static int method_import_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_ return r; t->type = type; - t->force_local = force; - t->read_only = read_only; + t->class = class; + t->flags = flags; t->local = strdup(local); if (!t->local) @@ -763,19 +800,21 @@ static int method_import_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_ if (r < 0) return r; - object = t->object_path; - id = t->id; - t = NULL; + r = sd_bus_reply_method_return(msg, "uo", t->id, t->object_path); + if (r < 0) + return r; - return sd_bus_reply_method_return(msg, "uo", id, object); + TAKE_PTR(t); + return 1; } static int method_import_fs(sd_bus_message *msg, void *userdata, sd_bus_error *error) { _cleanup_(transfer_unrefp) Transfer *t = NULL; - int fd, force, read_only, r; - const char *local, *object; + ImageClass class = _IMAGE_CLASS_INVALID; Manager *m = ASSERT_PTR(userdata); - uint32_t id; + const char *local; + uint64_t flags; + int fd, r; assert(msg); @@ -790,9 +829,34 @@ static int method_import_fs(sd_bus_message *msg, void *userdata, sd_bus_error *e if (r == 0) return 1; /* Will call us back */ - r = sd_bus_message_read(msg, "hsbb", &fd, &local, &force, &read_only); - if (r < 0) - return r; + if (endswith(sd_bus_message_get_member(msg), "Ex")) { + const char *sclass; + + r = sd_bus_message_read(msg, "hsst", &fd, &local, &sclass, &flags); + if (r < 0) + return r; + + class = image_class_from_string(sclass); + if (class < 0) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Image class '%s' not known", sclass); + + if (flags & ~(IMPORT_READ_ONLY|IMPORT_FORCE)) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Flags 0x%" PRIx64 " invalid", flags); + } else { + int force, read_only; + + r = sd_bus_message_read(msg, "hsbb", &fd, &local, &force, &read_only); + if (r < 0) + return r; + + class = IMAGE_MACHINE; + + flags = 0; + SET_FLAG(flags, IMPORT_FORCE, force); + SET_FLAG(flags, IMPORT_READ_ONLY, read_only); + } r = fd_verify_directory(fd); if (r < 0) @@ -802,17 +866,19 @@ static int method_import_fs(sd_bus_message *msg, void *userdata, sd_bus_error *e return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Local name %s is invalid", local); - r = setup_machine_directory(error, m->use_btrfs_subvol, m->use_btrfs_quota); - if (r < 0) - return r; + if (class == IMAGE_MACHINE) { + r = setup_machine_directory(error, m->use_btrfs_subvol, m->use_btrfs_quota); + if (r < 0) + return r; + } r = transfer_new(m, &t); if (r < 0) return r; t->type = TRANSFER_IMPORT_FS; - t->force_local = force; - t->read_only = read_only; + t->class = class; + t->flags = flags; t->local = strdup(local); if (!t->local) @@ -826,21 +892,23 @@ static int method_import_fs(sd_bus_message *msg, void *userdata, sd_bus_error *e if (r < 0) return r; - object = t->object_path; - id = t->id; - t = NULL; + r = sd_bus_reply_method_return(msg, "uo", t->id, t->object_path); + if (r < 0) + return r; - return sd_bus_reply_method_return(msg, "uo", id, object); + TAKE_PTR(t); + return 1; } static int method_export_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_error *error) { _cleanup_(transfer_unrefp) Transfer *t = NULL; - int fd, r; - const char *local, *object, *format; + ImageClass class = _IMAGE_CLASS_INVALID; Manager *m = ASSERT_PTR(userdata); + const char *local, *format; TransferType type; + uint64_t flags; struct stat st; - uint32_t id; + int fd, r; assert(msg); @@ -855,9 +923,29 @@ static int method_export_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_ if (r == 0) return 1; /* Will call us back */ - r = sd_bus_message_read(msg, "shs", &local, &fd, &format); - if (r < 0) - return r; + if (endswith(sd_bus_message_get_member(msg), "Ex")) { + const char *sclass; + + r = sd_bus_message_read(msg, "sshst", &local, &sclass, &fd, &format, &flags); + if (r < 0) + return r; + + class = image_class_from_string(sclass); + if (class < 0) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Image class '%s' not known", sclass); + + if (flags != 0) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Flags 0x%" PRIx64 " invalid", flags); + } else { + r = sd_bus_message_read(msg, "shs", &local, &fd, &format); + if (r < 0) + return r; + + class = IMAGE_MACHINE; + flags = 0; + } if (!hostname_is_valid(local, 0)) return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, @@ -869,7 +957,7 @@ static int method_export_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_ if (!S_ISREG(st.st_mode) && !S_ISFIFO(st.st_mode)) return -EINVAL; - type = streq_ptr(sd_bus_message_get_member(msg), "ExportTar") ? + type = startswith(sd_bus_message_get_member(msg), "ExportTar") ? TRANSFER_EXPORT_TAR : TRANSFER_EXPORT_RAW; r = transfer_new(m, &t); @@ -877,6 +965,8 @@ static int method_export_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_ return r; t->type = type; + t->class = class; + t->flags = flags; if (!isempty(format)) { t->format = strdup(format); @@ -896,21 +986,23 @@ static int method_export_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_ if (r < 0) return r; - object = t->object_path; - id = t->id; - t = NULL; + r = sd_bus_reply_method_return(msg, "uo", t->id, t->object_path); + if (r < 0) + return r; - return sd_bus_reply_method_return(msg, "uo", id, object); + TAKE_PTR(t); + return 1; } static int method_pull_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_error *error) { _cleanup_(transfer_unrefp) Transfer *t = NULL; - const char *remote, *local, *verify, *object; + ImageClass class = _IMAGE_CLASS_INVALID; + const char *remote, *local, *verify; Manager *m = ASSERT_PTR(userdata); - ImportVerify v; TransferType type; - int force, r; - uint32_t id; + uint64_t flags; + ImportVerify v; + int r; assert(msg); @@ -925,9 +1017,33 @@ static int method_pull_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_er if (r == 0) return 1; /* Will call us back */ - r = sd_bus_message_read(msg, "sssb", &remote, &local, &verify, &force); - if (r < 0) - return r; + if (endswith(sd_bus_message_get_member(msg), "Ex")) { + const char *sclass; + + r = sd_bus_message_read(msg, "sssst", &remote, &local, &sclass, &verify, &flags); + if (r < 0) + return r; + + class = image_class_from_string(sclass); + if (class < 0) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Image class '%s' not known", sclass); + + if (flags & ~(IMPORT_FORCE|IMPORT_READ_ONLY)) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Flags 0x%" PRIx64 " invalid", flags); + } else { + int force; + + r = sd_bus_message_read(msg, "sssb", &remote, &local, &verify, &force); + if (r < 0) + return r; + + class = IMAGE_MACHINE; + + flags = 0; + SET_FLAG(flags, IMPORT_FORCE, force); + } if (!http_url_is_valid(remote) && !file_url_is_valid(remote)) return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, @@ -947,11 +1063,13 @@ static int method_pull_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_er return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Unknown verification mode %s", verify); - r = setup_machine_directory(error, m->use_btrfs_subvol, m->use_btrfs_quota); - if (r < 0) - return r; + if (class == IMAGE_MACHINE) { + r = setup_machine_directory(error, m->use_btrfs_subvol, m->use_btrfs_quota); + if (r < 0) + return r; + } - type = streq_ptr(sd_bus_message_get_member(msg), "PullTar") ? + type = startswith(sd_bus_message_get_member(msg), "PullTar") ? TRANSFER_PULL_TAR : TRANSFER_PULL_RAW; if (manager_find(m, type, remote)) @@ -964,7 +1082,8 @@ static int method_pull_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_er t->type = type; t->verify = v; - t->force_local = force; + t->flags = flags; + t->class = class; t->remote = strdup(remote); if (!t->remote) @@ -980,40 +1099,81 @@ static int method_pull_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_er if (r < 0) return r; - object = t->object_path; - id = t->id; - t = NULL; + r = sd_bus_reply_method_return(msg, "uo", t->id, t->object_path); + if (r < 0) + return r; - return sd_bus_reply_method_return(msg, "uo", id, object); + TAKE_PTR(t); + return 1; } static int method_list_transfers(sd_bus_message *msg, void *userdata, sd_bus_error *error) { _cleanup_(sd_bus_message_unrefp) sd_bus_message *reply = NULL; Manager *m = ASSERT_PTR(userdata); + ImageClass class = _IMAGE_CLASS_INVALID; Transfer *t; int r; assert(msg); + bool ex = endswith(sd_bus_message_get_member(msg), "Ex"); + if (ex) { + const char *sclass; + uint64_t flags; + + r = sd_bus_message_read(msg, "st", &sclass, &flags); + if (r < 0) + return bus_log_parse_error(r); + + if (!isempty(sclass)) { + class = image_class_from_string(sclass); + if (class < 0) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Image class '%s' not known", sclass); + } + + if (flags != 0) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Flags 0x%" PRIx64 " invalid", flags); + } + r = sd_bus_message_new_method_return(msg, &reply); if (r < 0) return r; - r = sd_bus_message_open_container(reply, 'a', "(usssdo)"); + if (ex) + r = sd_bus_message_open_container(reply, 'a', "(ussssdo)"); + else + r = sd_bus_message_open_container(reply, 'a', "(usssdo)"); if (r < 0) return r; HASHMAP_FOREACH(t, m->transfers) { - r = sd_bus_message_append( - reply, - "(usssdo)", - t->id, - transfer_type_to_string(t->type), - t->remote, - t->local, - transfer_percent_as_double(t), - t->object_path); + if (class >= 0 && class != t->class) + continue; + + if (ex) + r = sd_bus_message_append( + reply, + "(ussssdo)", + t->id, + transfer_type_to_string(t->type), + t->remote, + t->local, + image_class_to_string(t->class), + transfer_percent_as_double(t), + t->object_path); + else + r = sd_bus_message_append( + reply, + "(usssdo)", + t->id, + transfer_type_to_string(t->type), + t->remote, + t->local, + transfer_percent_as_double(t), + t->object_path); if (r < 0) return r; } @@ -1059,7 +1219,7 @@ static int method_cancel_transfer(sd_bus_message *msg, void *userdata, sd_bus_er r = bus_verify_polkit_async( msg, - "org.freedesktop.import1.pull", + "org.freedesktop.import1.cancel", /* details= */ NULL, &m->polkit_registry, error); @@ -1212,6 +1372,17 @@ static const sd_bus_vtable manager_vtable[] = { SD_BUS_PARAM(transfer_path), method_import_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("ImportTarEx", + "hsst", + SD_BUS_PARAM(fd) + SD_BUS_PARAM(local_name) + SD_BUS_PARAM(class) + SD_BUS_PARAM(flags), + "uo", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path), + method_import_tar_or_raw, + SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD_WITH_NAMES("ImportRaw", "hsbb", SD_BUS_PARAM(fd) @@ -1223,6 +1394,17 @@ static const sd_bus_vtable manager_vtable[] = { SD_BUS_PARAM(transfer_path), method_import_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("ImportRawEx", + "hsst", + SD_BUS_PARAM(fd) + SD_BUS_PARAM(local_name) + SD_BUS_PARAM(class) + SD_BUS_PARAM(flags), + "uo", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path), + method_import_tar_or_raw, + SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD_WITH_NAMES("ImportFileSystem", "hsbb", SD_BUS_PARAM(fd) @@ -1234,6 +1416,17 @@ static const sd_bus_vtable manager_vtable[] = { SD_BUS_PARAM(transfer_path), method_import_fs, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("ImportFileSystemEx", + "hsst", + SD_BUS_PARAM(fd) + SD_BUS_PARAM(local_name) + SD_BUS_PARAM(class) + SD_BUS_PARAM(flags), + "uo", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path), + method_import_fs, + SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD_WITH_NAMES("ExportTar", "shs", SD_BUS_PARAM(local_name) @@ -1244,6 +1437,18 @@ static const sd_bus_vtable manager_vtable[] = { SD_BUS_PARAM(transfer_path), method_export_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("ExportTarEx", + "sshst", + SD_BUS_PARAM(local_name) + SD_BUS_PARAM(class) + SD_BUS_PARAM(fd) + SD_BUS_PARAM(format) + SD_BUS_PARAM(flags), + "uo", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path), + method_export_tar_or_raw, + SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD_WITH_NAMES("ExportRaw", "shs", SD_BUS_PARAM(local_name) @@ -1254,6 +1459,18 @@ static const sd_bus_vtable manager_vtable[] = { SD_BUS_PARAM(transfer_path), method_export_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("ExportRawEx", + "sshst", + SD_BUS_PARAM(local_name) + SD_BUS_PARAM(class) + SD_BUS_PARAM(fd) + SD_BUS_PARAM(format) + SD_BUS_PARAM(flags), + "uo", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path), + method_export_tar_or_raw, + SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD_WITH_NAMES("PullTar", "sssb", SD_BUS_PARAM(url) @@ -1265,6 +1482,18 @@ static const sd_bus_vtable manager_vtable[] = { SD_BUS_PARAM(transfer_path), method_pull_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("PullTarEx", + "sssst", + SD_BUS_PARAM(url) + SD_BUS_PARAM(local_name) + SD_BUS_PARAM(class) + SD_BUS_PARAM(verify_mode) + SD_BUS_PARAM(flags), + "uo", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path), + method_pull_tar_or_raw, + SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD_WITH_NAMES("PullRaw", "sssb", SD_BUS_PARAM(url) @@ -1276,12 +1505,32 @@ static const sd_bus_vtable manager_vtable[] = { SD_BUS_PARAM(transfer_path), method_pull_tar_or_raw, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("PullRawEx", + "sssst", + SD_BUS_PARAM(url) + SD_BUS_PARAM(local_name) + SD_BUS_PARAM(class) + SD_BUS_PARAM(verify_mode) + SD_BUS_PARAM(flags), + "uo", + SD_BUS_PARAM(transfer_id) + SD_BUS_PARAM(transfer_path), + method_pull_tar_or_raw, + SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD_WITH_NAMES("ListTransfers", NULL,, "a(usssdo)", SD_BUS_PARAM(transfers), method_list_transfers, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("ListTransfersEx", + "st", + SD_BUS_PARAM(class) + SD_BUS_PARAM(flags), + "a(ussssdo)", + SD_BUS_PARAM(transfers), + method_list_transfers, + SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD_WITH_NAMES("CancelTransfer", "u", SD_BUS_PARAM(transfer_id), diff --git a/src/import/org.freedesktop.import1.conf b/src/import/org.freedesktop.import1.conf index d252ff6460..f775aa061c 100644 --- a/src/import/org.freedesktop.import1.conf +++ b/src/import/org.freedesktop.import1.conf @@ -42,6 +42,10 @@ send_interface="org.freedesktop.import1.Manager" send_member="ListTransfers"/> + + @@ -50,30 +54,58 @@ send_interface="org.freedesktop.import1.Manager" send_member="ImportTar"/> + + + + + + + + + + + + + + diff --git a/src/import/org.freedesktop.import1.policy b/src/import/org.freedesktop.import1.policy index 88e436dc3a..45c11def67 100644 --- a/src/import/org.freedesktop.import1.policy +++ b/src/import/org.freedesktop.import1.policy @@ -19,8 +19,8 @@ https://systemd.io - Import a VM or container image - Authentication is required to import a VM or container image + Import a disk image + Authentication is required to import an image auth_admin auth_admin @@ -29,8 +29,8 @@ - Export a VM or container image - Authentication is required to export a VM or container image + Export a disk image + Authentication is required to export disk image auth_admin auth_admin @@ -39,8 +39,18 @@ - Download a VM or container image - Authentication is required to download a VM or container image + Download a disk image + Authentication is required to download a disk image + + auth_admin + auth_admin + auth_admin_keep + + + + + Cancel transfer of a disk image + Authentication is required to cancel the ongoing transfer of a disk image auth_admin auth_admin diff --git a/src/import/pull-common.c b/src/import/pull-common.c index 7839057769..4fca08437c 100644 --- a/src/import/pull-common.c +++ b/src/import/pull-common.c @@ -37,11 +37,9 @@ int pull_find_old_etags( int r; assert(url); + assert(image_root); assert(etags); - if (!image_root) - image_root = "/var/lib/machines"; - _cleanup_free_ char *escaped_url = xescape(url, FILENAME_ESCAPE); if (!escaped_url) return -ENOMEM; @@ -128,11 +126,9 @@ int pull_make_path(const char *url, const char *etag, const char *image_root, co char *path; assert(url); + assert(image_root); assert(ret); - if (!image_root) - image_root = "/var/lib/machines"; - escaped_url = xescape(url, FILENAME_ESCAPE); if (!escaped_url) return -ENOMEM; diff --git a/src/import/pull-raw.c b/src/import/pull-raw.c index ebe479fb80..50785fe431 100644 --- a/src/import/pull-raw.c +++ b/src/import/pull-raw.c @@ -60,7 +60,7 @@ struct RawPull { void *userdata; char *local; /* In PULL_DIRECT mode the path we are supposed to place things in, otherwise the - * machine name of the final copy we make */ + * image name of the final copy we make */ char *final_path; char *temp_path; @@ -127,8 +127,9 @@ int raw_pull_new( int r; assert(ret); + assert(image_root); - root = strdup(image_root ?: "/var/lib/machines"); + root = strdup(image_root); if (!root) return -ENOMEM; diff --git a/src/import/pull-tar.c b/src/import/pull-tar.c index 99d8ff0553..ae573f1b54 100644 --- a/src/import/pull-tar.c +++ b/src/import/pull-tar.c @@ -106,9 +106,10 @@ int tar_pull_new( _cleanup_free_ char *root = NULL; int r; + assert(image_root); assert(ret); - root = strdup(image_root ?: "/var/lib/machines"); + root = strdup(image_root); if (!root) return -ENOMEM; diff --git a/src/import/pull.c b/src/import/pull.c index 518755b61d..f281ab04fd 100644 --- a/src/import/pull.c +++ b/src/import/pull.c @@ -26,11 +26,12 @@ #include "verbs.h" #include "web-util.h" -static const char *arg_image_root = "/var/lib/machines"; +static const char *arg_image_root = NULL; static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE; static ImportFlags arg_import_flags = IMPORT_PULL_SETTINGS | IMPORT_PULL_ROOTHASH | IMPORT_PULL_ROOTHASH_SIGNATURE | IMPORT_PULL_VERITY | IMPORT_BTRFS_SUBVOL | IMPORT_BTRFS_QUOTA | IMPORT_CONVERT_QCOW2 | IMPORT_SYNC; static uint64_t arg_offset = UINT64_MAX, arg_size_max = UINT64_MAX; static char *arg_checksum = NULL; +static ImageClass arg_class = IMAGE_MACHINE; STATIC_DESTRUCTOR_REGISTER(arg_checksum, freep); @@ -64,7 +65,7 @@ static int normalize_local(const char *local, const char *url, char **ret) { local); if (!FLAGS_SET(arg_import_flags, IMPORT_FORCE)) { - r = image_find(IMAGE_MACHINE, local, NULL, NULL); + r = image_find(arg_class, local, NULL, NULL); if (r < 0) { if (r != -ENOENT) return log_error_errno(r, "Failed to check whether image '%s' exists: %m", local); @@ -241,7 +242,7 @@ static int pull_raw(int argc, char *argv[], void *userdata) { static int help(int argc, char *argv[], void *userdata) { printf("%1$s [OPTIONS...] {COMMAND} ...\n" - "\n%4$sDownload container or virtual machine images.%5$s\n" + "\n%4$sDownload disk images.%5$s\n" "\n%2$sCommands:%3$s\n" " tar URL [NAME] Download a TAR image\n" " raw URL [NAME] Download a RAW image\n" @@ -267,7 +268,9 @@ static int help(int argc, char *argv[], void *userdata) { " regular disk images\n" " --sync=BOOL Controls whether to sync() before completing\n" " --offset=BYTES Offset to seek to in destination\n" - " --size-max=BYTES Maximum number of bytes to write to destination\n", + " --size-max=BYTES Maximum number of bytes to write to destination\n" + " --class=CLASS Select image class (machine, sysext, confext,\n" + " portable)\n", program_invocation_short_name, ansi_underline(), ansi_normal(), @@ -296,6 +299,7 @@ static int parse_argv(int argc, char *argv[]) { ARG_SYNC, ARG_OFFSET, ARG_SIZE_MAX, + ARG_CLASS, }; static const struct option options[] = { @@ -316,10 +320,12 @@ static int parse_argv(int argc, char *argv[]) { { "sync", required_argument, NULL, ARG_SYNC }, { "offset", required_argument, NULL, ARG_OFFSET }, { "size-max", required_argument, NULL, ARG_SIZE_MAX }, + { "class", required_argument, NULL, ARG_CLASS }, {} }; int c, r; + bool auto_settings = true; assert(argc >= 0); assert(argv); @@ -381,6 +387,7 @@ static int parse_argv(int argc, char *argv[]) { return r; SET_FLAG(arg_import_flags, IMPORT_PULL_SETTINGS, r); + auto_settings = false; break; case ARG_ROOTHASH: @@ -478,6 +485,13 @@ static int parse_argv(int argc, char *argv[]) { break; } + case ARG_CLASS: + arg_class = image_class_from_string(optarg); + if (arg_class < 0) + return log_error_errno(arg_class, "Failed to parse --class= argument: %s", optarg); + + break; + case '?': return -EINVAL; @@ -497,6 +511,13 @@ static int parse_argv(int argc, char *argv[]) { if (arg_checksum && (arg_import_flags & (IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY)) != 0) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Literal checksum verification only supported if no associated files are downloaded."); + if (!arg_image_root) + arg_image_root = image_root_to_string(arg_class); + + /* .nspawn settings files only really make sense for machine images, not for sysext/confext/portable */ + if (auto_settings && arg_class != IMAGE_MACHINE) + arg_import_flags &= ~IMPORT_PULL_SETTINGS; + return 1; } diff --git a/src/shared/discover-image.c b/src/shared/discover-image.c index 72f20c8eb7..00d32a9c87 100644 --- a/src/shared/discover-image.c +++ b/src/shared/discover-image.c @@ -99,6 +99,15 @@ static const char* image_class_suffix_table[_IMAGE_CLASS_MAX] = { DEFINE_PRIVATE_STRING_TABLE_LOOKUP_TO_STRING(image_class_suffix, ImageClass); +static const char *const image_root_table[_IMAGE_CLASS_MAX] = { + [IMAGE_MACHINE] = "/var/lib/machines", + [IMAGE_PORTABLE] = "/var/lib/portables", + [IMAGE_SYSEXT] = "/var/lib/extensions", + [IMAGE_CONFEXT] = "/var/lib/confexts", +}; + +DEFINE_STRING_TABLE_LOOKUP_TO_STRING(image_root, ImageClass); + static Image *image_free(Image *i) { assert(i); diff --git a/src/shared/discover-image.h b/src/shared/discover-image.h index a30a3d92ba..350c3a22cc 100644 --- a/src/shared/discover-image.h +++ b/src/shared/discover-image.h @@ -119,4 +119,6 @@ static inline bool IMAGE_IS_HOST(const struct Image *i) { int image_to_json(const struct Image *i, JsonVariant **ret); +const char *image_root_to_string(ImageClass c) _const_; + extern const struct hash_ops image_hash_ops; From c3c892b41fd73fa653729c2abd4084ccecee7c20 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 22 Feb 2024 11:47:34 +0100 Subject: [PATCH 12/26] importd: tighten checks in fds passed to us --- src/import/importd.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/import/importd.c b/src/import/importd.c index 47dfb2dfaf..0f6bf71419 100644 --- a/src/import/importd.c +++ b/src/import/importd.c @@ -761,6 +761,10 @@ static int method_import_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_ SET_FLAG(flags, IMPORT_READ_ONLY, read_only); } + r = fd_verify_safe_flags(fd); + if (r < 0) + return r; + if (fstat(fd, &st) < 0) return -errno; @@ -858,6 +862,10 @@ static int method_import_fs(sd_bus_message *msg, void *userdata, sd_bus_error *e SET_FLAG(flags, IMPORT_READ_ONLY, read_only); } + r = fd_verify_safe_flags(fd); + if (r < 0) + return r; + r = fd_verify_directory(fd); if (r < 0) return r; @@ -951,6 +959,10 @@ static int method_export_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_ return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Local name %s is invalid", local); + r = fd_verify_safe_flags(fd); + if (r < 0) + return r; + if (fstat(fd, &st) < 0) return -errno; From 67c7ee11afb5ff703c3ca8e390c8360140298a0f Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 22 Feb 2024 10:16:43 +0100 Subject: [PATCH 13/26] importctl: add support for selecting image class to download --- src/import/importctl.c | 370 +++++++++++++++++++++++++++++++---------- 1 file changed, 283 insertions(+), 87 deletions(-) diff --git a/src/import/importctl.c b/src/import/importctl.c index de8e263d97..8035a69a38 100644 --- a/src/import/importctl.c +++ b/src/import/importctl.c @@ -9,14 +9,17 @@ #include "bus-error.h" #include "bus-locator.h" #include "bus-util.h" +#include "discover-image.h" #include "fd-util.h" #include "format-table.h" #include "hostname-util.h" +#include "import-common.h" #include "import-util.h" #include "locale-util.h" #include "log.h" #include "macro.h" #include "main-func.h" +#include "os-util.h" #include "pager.h" #include "parse-argument.h" #include "parse-util.h" @@ -33,13 +36,31 @@ static PagerFlags arg_pager_flags = 0; static bool arg_legend = true; static BusTransport arg_transport = BUS_TRANSPORT_LOCAL; static const char *arg_host = NULL; -static bool arg_read_only = false; +static ImportFlags arg_import_flags = 0; static bool arg_quiet = false; static bool arg_ask_password = true; -static bool arg_force = false; static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE; static const char* arg_format = NULL; static JsonFormatFlags arg_json_format_flags = JSON_FORMAT_OFF; +static ImageClass arg_image_class = _IMAGE_CLASS_INVALID; + +static int settle_image_class(void) { + + if (arg_image_class < 0) { + _cleanup_free_ char *j = NULL; + + for (ImageClass class = 0; class < _IMAGE_CLASS_MAX; class++) + if (strextendf_with_separator(&j, ", ", "%s (downloads to %s/)", + image_class_to_string(class), + image_root_to_string(class)) < 0) + return log_oom(); + + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), + "No image class specified, retry with --class= set to one of: %s.", j); + } + + return 0; +} static int match_log_message(sd_bus_message *m, void *userdata, sd_bus_error *error) { const char **our_path = userdata, *line; @@ -170,6 +191,10 @@ static int import_tar(int argc, char *argv[], void *userdata) { sd_bus *bus = ASSERT_PTR(userdata); int r; + r = settle_image_class(); + if (r < 0) + return r; + if (argc >= 2) path = empty_or_dash_to_null(argv[1]); @@ -206,17 +231,31 @@ static int import_tar(int argc, char *argv[], void *userdata) { return log_error_errno(errno, "Failed to open %s: %m", path); } - r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ImportTar"); - if (r < 0) - return bus_log_create_error(r); + if (arg_image_class == IMAGE_MACHINE && (arg_image_class & ~(IMPORT_FORCE|IMPORT_READ_ONLY)) == 0) { + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ImportTar"); + if (r < 0) + return bus_log_create_error(r); - r = sd_bus_message_append( - m, - "hsbb", - fd >= 0 ? fd : STDIN_FILENO, - local, - arg_force, - arg_read_only); + r = sd_bus_message_append( + m, + "hsbb", + fd >= 0 ? fd : STDIN_FILENO, + local, + FLAGS_SET(arg_import_flags, IMPORT_FORCE), + FLAGS_SET(arg_import_flags, IMPORT_READ_ONLY)); + } else { + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ImportTarEx"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "hsst", + fd >= 0 ? fd : STDIN_FILENO, + local, + image_class_to_string(arg_image_class), + (uint64_t) arg_import_flags & (IMPORT_FORCE|IMPORT_READ_ONLY)); + } if (r < 0) return bus_log_create_error(r); @@ -231,6 +270,10 @@ static int import_raw(int argc, char *argv[], void *userdata) { sd_bus *bus = ASSERT_PTR(userdata); int r; + r = settle_image_class(); + if (r < 0) + return r; + if (argc >= 2) path = empty_or_dash_to_null(argv[1]); @@ -267,17 +310,31 @@ static int import_raw(int argc, char *argv[], void *userdata) { return log_error_errno(errno, "Failed to open %s: %m", path); } - r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ImportRaw"); - if (r < 0) - return bus_log_create_error(r); + if (arg_image_class == IMAGE_MACHINE && (arg_image_class & ~(IMPORT_FORCE|IMPORT_READ_ONLY)) == 0) { + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ImportRaw"); + if (r < 0) + return bus_log_create_error(r); - r = sd_bus_message_append( - m, - "hsbb", - fd >= 0 ? fd : STDIN_FILENO, - local, - arg_force, - arg_read_only); + r = sd_bus_message_append( + m, + "hsbb", + fd >= 0 ? fd : STDIN_FILENO, + local, + FLAGS_SET(arg_import_flags, IMPORT_FORCE), + FLAGS_SET(arg_import_flags, IMPORT_READ_ONLY)); + } else { + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ImportRawEx"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "hsst", + fd >= 0 ? fd : STDIN_FILENO, + local, + image_class_to_string(arg_image_class), + (uint64_t) arg_import_flags & (IMPORT_FORCE|IMPORT_READ_ONLY)); + } if (r < 0) return bus_log_create_error(r); @@ -292,6 +349,10 @@ static int import_fs(int argc, char *argv[], void *userdata) { sd_bus *bus = ASSERT_PTR(userdata); int r; + r = settle_image_class(); + if (r < 0) + return r; + if (argc >= 2) path = empty_or_dash_to_null(argv[1]); @@ -319,19 +380,34 @@ static int import_fs(int argc, char *argv[], void *userdata) { return log_error_errno(errno, "Failed to open directory '%s': %m", path); } - r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ImportFileSystem"); + if (arg_image_class == IMAGE_MACHINE && (arg_image_class & ~(IMPORT_FORCE|IMPORT_READ_ONLY)) == 0) { + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ImportFileSystem"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "hsbb", + fd >= 0 ? fd : STDIN_FILENO, + local, + FLAGS_SET(arg_import_flags, IMPORT_FORCE), + FLAGS_SET(arg_import_flags, IMPORT_READ_ONLY)); + } else { + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ImportFileSystemEx"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "hsst", + fd >= 0 ? fd : STDIN_FILENO, + local, + image_class_to_string(arg_image_class), + (uint64_t) arg_import_flags & (IMPORT_FORCE|IMPORT_READ_ONLY)); + } if (r < 0) return bus_log_create_error(r); - r = sd_bus_message_append( - m, - "hsbb", - fd >= 0 ? fd : STDIN_FILENO, - local, - arg_force, - arg_read_only); - if (r < 0) - return bus_log_create_error(r); return transfer_image_common(bus, m); } @@ -358,6 +434,10 @@ static int export_tar(int argc, char *argv[], void *userdata) { sd_bus *bus = ASSERT_PTR(userdata); int r; + r = settle_image_class(); + if (r < 0) + return r; + local = argv[1]; if (!hostname_is_valid(local, 0)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), @@ -375,16 +455,31 @@ static int export_tar(int argc, char *argv[], void *userdata) { return log_error_errno(errno, "Failed to open %s: %m", path); } - r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ExportTar"); - if (r < 0) - return bus_log_create_error(r); + if (arg_image_class == IMAGE_MACHINE && arg_import_flags == 0) { + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ExportTar"); + if (r < 0) + return bus_log_create_error(r); - r = sd_bus_message_append( - m, - "shs", - local, - fd >= 0 ? fd : STDOUT_FILENO, - arg_format); + r = sd_bus_message_append( + m, + "shs", + local, + fd >= 0 ? fd : STDOUT_FILENO, + arg_format); + } else { + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ExportTarEx"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "sshst", + local, + image_class_to_string(arg_image_class), + fd >= 0 ? fd : STDOUT_FILENO, + arg_format, + /* flags= */ 0); + } if (r < 0) return bus_log_create_error(r); @@ -398,6 +493,10 @@ static int export_raw(int argc, char *argv[], void *userdata) { sd_bus *bus = ASSERT_PTR(userdata); int r; + r = settle_image_class(); + if (r < 0) + return r; + local = argv[1]; if (!hostname_is_valid(local, 0)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), @@ -415,16 +514,31 @@ static int export_raw(int argc, char *argv[], void *userdata) { return log_error_errno(errno, "Failed to open %s: %m", path); } - r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ExportRaw"); - if (r < 0) - return bus_log_create_error(r); + if (arg_image_class == IMAGE_MACHINE && arg_import_flags == 0) { + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ExportRaw"); + if (r < 0) + return bus_log_create_error(r); - r = sd_bus_message_append( - m, - "shs", - local, - fd >= 0 ? fd : STDOUT_FILENO, - arg_format); + r = sd_bus_message_append( + m, + "shs", + local, + fd >= 0 ? fd : STDOUT_FILENO, + arg_format); + } else { + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ExportRawEx"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "sshst", + local, + image_class_to_string(arg_image_class), + fd >= 0 ? fd : STDOUT_FILENO, + arg_format, + /* flags= */ 0); + } if (r < 0) return bus_log_create_error(r); @@ -438,6 +552,10 @@ static int pull_tar(int argc, char *argv[], void *userdata) { sd_bus *bus = ASSERT_PTR(userdata); int r; + r = settle_image_class(); + if (r < 0) + return r; + remote = argv[1]; if (!http_url_is_valid(remote) && !file_url_is_valid(remote)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), @@ -468,17 +586,32 @@ static int pull_tar(int argc, char *argv[], void *userdata) { local); } - r = bus_message_new_method_call(bus, &m, bus_import_mgr, "PullTar"); - if (r < 0) - return bus_log_create_error(r); + if (arg_image_class == IMAGE_MACHINE && (arg_image_class & ~(IMPORT_FORCE|IMPORT_READ_ONLY)) == 0) { + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "PullTar"); + if (r < 0) + return bus_log_create_error(r); - r = sd_bus_message_append( - m, - "sssb", - remote, - local, - import_verify_to_string(arg_verify), - arg_force); + r = sd_bus_message_append( + m, + "sssb", + remote, + local, + import_verify_to_string(arg_verify), + FLAGS_SET(arg_import_flags, IMPORT_FORCE)); + } else { + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "PullTarEx"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "sssst", + remote, + local, + image_class_to_string(arg_image_class), + import_verify_to_string(arg_verify), + (uint64_t) arg_import_flags & (IMPORT_FORCE|IMPORT_READ_ONLY)); + } if (r < 0) return bus_log_create_error(r); @@ -492,6 +625,10 @@ static int pull_raw(int argc, char *argv[], void *userdata) { sd_bus *bus = ASSERT_PTR(userdata); int r; + r = settle_image_class(); + if (r < 0) + return r; + remote = argv[1]; if (!http_url_is_valid(remote) && !file_url_is_valid(remote)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), @@ -522,17 +659,32 @@ static int pull_raw(int argc, char *argv[], void *userdata) { local); } - r = bus_message_new_method_call(bus, &m, bus_import_mgr, "PullRaw"); - if (r < 0) - return bus_log_create_error(r); + if (arg_image_class == IMAGE_MACHINE && (arg_image_class & ~(IMPORT_FORCE|IMPORT_READ_ONLY)) == 0) { + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "PullRaw"); + if (r < 0) + return bus_log_create_error(r); - r = sd_bus_message_append( - m, - "sssb", - remote, - local, - import_verify_to_string(arg_verify), - arg_force); + r = sd_bus_message_append( + m, + "sssb", + remote, + local, + import_verify_to_string(arg_verify), + FLAGS_SET(arg_import_flags, IMPORT_FORCE)); + } else { + r = bus_message_new_method_call(bus, &m, bus_import_mgr, "PullRawEx"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_message_append( + m, + "sssst", + remote, + local, + image_class_to_string(arg_image_class), + import_verify_to_string(arg_verify), + (uint64_t) arg_import_flags & (IMPORT_FORCE|IMPORT_READ_ONLY)); + } if (r < 0) return bus_log_create_error(r); @@ -548,27 +700,42 @@ static int list_transfers(int argc, char *argv[], void *userdata) { pager_open(arg_pager_flags); - r = bus_call_method(bus, bus_import_mgr, "ListTransfers", &error, &reply, NULL); - if (r < 0) - return log_error_errno(r, "Could not get transfers: %s", bus_error_message(&error, r)); + bool ex; + r = bus_call_method(bus, bus_import_mgr, "ListTransfersEx", &error, &reply, NULL); + if (r < 0) { + if (sd_bus_error_has_name(&error, SD_BUS_ERROR_UNKNOWN_METHOD)) { + sd_bus_error_free(&error); - r = sd_bus_message_enter_container(reply, 'a', "(usssdo)"); + r = bus_call_method(bus, bus_import_mgr, "ListTransfers", &error, &reply, NULL); + } + if (r < 0) + return log_error_errno(r, "Could not get transfers: %s", bus_error_message(&error, r)); + + ex = false; + r = sd_bus_message_enter_container(reply, 'a', "(usssdo)"); + } else { + ex = true; + r = sd_bus_message_enter_container(reply, 'a', "(ussssdo)"); + } if (r < 0) return bus_log_parse_error(r); - t = table_new("id", "progress", "type", "local", "remote"); + t = table_new("id", "progress", "type", "class", "local", "remote"); if (!t) return log_oom(); - (void) table_set_sort(t, (size_t) 3, (size_t) 0); + (void) table_set_sort(t, (size_t) 4, (size_t) 0); table_set_ersatz_string(t, TABLE_ERSATZ_DASH); for (;;) { - const char *type, *remote, *local; + const char *type, *remote, *local, *class = "machine"; double progress; uint32_t id; - r = sd_bus_message_read(reply, "(usssdo)", &id, &type, &remote, &local, &progress, NULL); + if (ex) + r = sd_bus_message_read(reply, "(ussssdo)", &id, &type, &remote, &local, &class, &progress, NULL); + else + r = sd_bus_message_read(reply, "(usssdo)", &id, &type, &remote, &local, &progress, NULL); if (r < 0) return bus_log_parse_error(r); if (r == 0) @@ -596,6 +763,7 @@ static int list_transfers(int argc, char *argv[], void *userdata) { r = table_add_many( t, TABLE_STRING, type, + TABLE_STRING, class, TABLE_STRING, local, TABLE_STRING, remote, TABLE_SET_URL, remote); @@ -656,8 +824,8 @@ static int help(int argc, char *argv[], void *userdata) { return log_oom(); printf("%1$s [OPTIONS...] COMMAND ...\n\n" - "%5$sDownload machine images%6$s\n" - "\n%3$sImage Transfer Commands:%4$s\n" + "%5$sDownload, import or export disk images%6$s\n" + "\n%3$sCommands:%4$s\n" " pull-tar URL [NAME] Download a TAR container image\n" " pull-raw URL [NAME] Download a RAW container or VM image\n" " import-tar FILE [NAME] Import a local TAR container image\n" @@ -665,8 +833,8 @@ static int help(int argc, char *argv[], void *userdata) { " import-fs DIRECTORY [NAME] Import a local directory container image\n" " export-tar NAME [FILE] Export a TAR container image locally\n" " export-raw NAME [FILE] Export a RAW container or VM image locally\n" - " list-transfers Show list of downloads in progress\n" - " cancel-transfer Cancel a download\n" + " list-transfers Show list of transfers in progress\n" + " cancel-transfer [ID...] Cancel a transfer\n" "\n%3$sOptions:%4$s\n" " -h --help Show this help\n" " --version Show package version\n" @@ -675,7 +843,7 @@ static int help(int argc, char *argv[], void *userdata) { " --no-ask-password Do not ask for system passwords\n" " -H --host=[USER@]HOST Operate on remote host\n" " -M --machine=CONTAINER Operate on local container\n" - " --read-only Create read-only bind mount\n" + " --read-only Create read-only image\n" " -q --quiet Suppress output\n" " --json=pretty|short|off Generate JSON output\n" " -j Equvilant to --json=pretty on TTY, --json=short\n" @@ -683,7 +851,11 @@ static int help(int argc, char *argv[], void *userdata) { " --verify=MODE Verification mode for downloaded images (no,\n" " checksum, signature)\n" " --format=xz|gzip|bzip2 Desired output format for export\n" - " --force Download image even if already exists\n" + " --force Install image even if already exists\n" + " -m --class=machine Install as machine image\n" + " -P --class=portable Install as portable service image\n" + " -S --class=sysext Install as system extension image\n" + " -C --class=confext Install as configuration extension image\n" "\nSee the %2$s for details.\n", program_invocation_short_name, link, @@ -707,6 +879,7 @@ static int parse_argv(int argc, char *argv[]) { ARG_VERIFY, ARG_FORCE, ARG_FORMAT, + ARG_CLASS, }; static const struct option options[] = { @@ -723,6 +896,7 @@ static int parse_argv(int argc, char *argv[]) { { "verify", required_argument, NULL, ARG_VERIFY }, { "force", no_argument, NULL, ARG_FORCE }, { "format", required_argument, NULL, ARG_FORMAT }, + { "class", required_argument, NULL, ARG_CLASS }, {} }; @@ -732,7 +906,7 @@ static int parse_argv(int argc, char *argv[]) { assert(argv); for (;;) { - c = getopt_long(argc, argv, "hH:M:jq", options, NULL); + c = getopt_long(argc, argv, "hH:M:jqmPSC", options, NULL); if (c < 0) break; @@ -767,7 +941,7 @@ static int parse_argv(int argc, char *argv[]) { break; case ARG_READ_ONLY: - arg_read_only = true; + arg_import_flags |= IMPORT_READ_ONLY; break; case 'q': @@ -787,7 +961,7 @@ static int parse_argv(int argc, char *argv[]) { break; case ARG_FORCE: - arg_force = true; + arg_import_flags |= IMPORT_FORCE; break; case ARG_FORMAT: @@ -811,6 +985,28 @@ static int parse_argv(int argc, char *argv[]) { arg_legend = false; break; + case ARG_CLASS: + arg_image_class = image_class_from_string(optarg); + if (arg_image_class < 0) + return log_error_errno(arg_image_class, "Failed to parse --class= parameter: %s", optarg); + break; + + case 'm': + arg_image_class = IMAGE_MACHINE; + break; + + case 'P': + arg_image_class = IMAGE_PORTABLE; + break; + + case 'S': + arg_image_class = IMAGE_SYSEXT; + break; + + case 'C': + arg_image_class = IMAGE_CONFEXT; + break; + case '?': return -EINVAL; From 420b8ce7989206607902819ab32977a9f7d8eccb Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 1 Mar 2024 16:13:59 +0100 Subject: [PATCH 14/26] importlisttransfersxclient --- src/import/importctl.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/import/importctl.c b/src/import/importctl.c index 8035a69a38..3028f7d4bd 100644 --- a/src/import/importctl.c +++ b/src/import/importctl.c @@ -701,7 +701,7 @@ static int list_transfers(int argc, char *argv[], void *userdata) { pager_open(arg_pager_flags); bool ex; - r = bus_call_method(bus, bus_import_mgr, "ListTransfersEx", &error, &reply, NULL); + r = bus_call_method(bus, bus_import_mgr, "ListTransfersEx", &error, &reply, "st", image_class_to_string(arg_image_class), UINT64_C(0)); if (r < 0) { if (sd_bus_error_has_name(&error, SD_BUS_ERROR_UNKNOWN_METHOD)) { sd_bus_error_free(&error); @@ -741,6 +741,10 @@ static int list_transfers(int argc, char *argv[], void *userdata) { if (r == 0) break; + /* Ideally we use server-side filtering. But if the server can't do it, we need to do it client side */ + if (arg_image_class >= 0 && image_class_from_string(class) != arg_image_class) + continue; + r = table_add_many( t, TABLE_UINT32, id, From 8f20b498bd072bc9a0e9640b8560685ed11262f8 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 22 Feb 2024 12:31:03 +0100 Subject: [PATCH 15/26] importd: validate local image names with the right helper A while back we introduced image_name_is_valid() for validating image file names. It's more liberal than hostname_is_valid() in many ways (and allows version suffixes and such). Since importd deals in offline images (as opposed to machined otherwise which deals in running machines), let's hence use the right helper to validate the identifiers. --- src/import/export.c | 18 ++++++++++-------- src/import/import-common.c | 2 +- src/import/import-fs.c | 2 +- src/import/import.c | 2 +- src/import/importctl.c | 28 ++++++++++++++-------------- src/import/importd.c | 16 ++++++++-------- src/import/pull-common.c | 3 ++- src/import/pull.c | 2 +- 8 files changed, 38 insertions(+), 35 deletions(-) diff --git a/src/import/export.c b/src/import/export.c index ac81b2fbab..cdb1d6246a 100644 --- a/src/import/export.c +++ b/src/import/export.c @@ -63,12 +63,13 @@ static int export_tar(int argc, char *argv[], void *userdata) { _cleanup_close_ int open_fd = -EBADF; int r, fd; - if (hostname_is_valid(argv[1], 0)) { - r = image_find(arg_class, argv[1], NULL, &image); + local = argv[1]; + if (image_name_is_valid(local)) { + r = image_find(arg_class, local, NULL, &image); if (r == -ENOENT) - return log_error_errno(r, "Machine image %s not found.", argv[1]); + return log_error_errno(r, "Image %s not found.", local); if (r < 0) - return log_error_errno(r, "Failed to look for machine %s: %m", argv[1]); + return log_error_errno(r, "Failed to look for image %s: %m", local); local = image->path; } else @@ -135,12 +136,13 @@ static int export_raw(int argc, char *argv[], void *userdata) { _cleanup_close_ int open_fd = -EBADF; int r, fd; - if (hostname_is_valid(argv[1], 0)) { - r = image_find(arg_class, argv[1], NULL, &image); + local = argv[1]; + if (image_name_is_valid(local)) { + r = image_find(arg_class, local, NULL, &image); if (r == -ENOENT) - return log_error_errno(r, "Machine image %s not found.", argv[1]); + return log_error_errno(r, "Image %s not found.", local); if (r < 0) - return log_error_errno(r, "Failed to look for machine %s: %m", argv[1]); + return log_error_errno(r, "Failed to look for image %s: %m", local); local = image->path; } else diff --git a/src/import/import-common.c b/src/import/import-common.c index 9d1448545e..09faf16f9c 100644 --- a/src/import/import-common.c +++ b/src/import/import-common.c @@ -276,7 +276,7 @@ bool import_validate_local(const char *name, ImportFlags flags) { if (FLAGS_SET(flags, IMPORT_DIRECT)) return path_is_valid(name); - return hostname_is_valid(name, 0); + return image_name_is_valid(name); } static int interrupt_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) { diff --git a/src/import/import-fs.c b/src/import/import-fs.c index e37fd7b41d..b1619ef9ea 100644 --- a/src/import/import-fs.c +++ b/src/import/import-fs.c @@ -133,7 +133,7 @@ static int import_fs(int argc, char *argv[], void *userdata) { "Local path name '%s' is not valid.", final_path); } else { if (local) { - if (!hostname_is_valid(local, 0)) + if (!image_name_is_valid(local)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Local image name '%s' is not valid.", local); } else diff --git a/src/import/import.c b/src/import/import.c index 22ce6f5aaa..37dd0b2078 100644 --- a/src/import/import.c +++ b/src/import/import.c @@ -54,7 +54,7 @@ static int normalize_local(const char *local, char **ret) { "Local path name '%s' is not valid.", local); } else { if (local) { - if (!hostname_is_valid(local, 0)) + if (!image_name_is_valid(local)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Local image name '%s' is not valid.", local); diff --git a/src/import/importctl.c b/src/import/importctl.c index 3028f7d4bd..1fc13e1326 100644 --- a/src/import/importctl.c +++ b/src/import/importctl.c @@ -220,9 +220,9 @@ static int import_tar(int argc, char *argv[], void *userdata) { local = ll; - if (!hostname_is_valid(local, 0)) + if (!image_name_is_valid(local)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Local name %s is not a suitable machine name.", + "Local name %s is not a suitable image name.", local); if (path) { @@ -299,9 +299,9 @@ static int import_raw(int argc, char *argv[], void *userdata) { local = ll; - if (!hostname_is_valid(local, 0)) + if (!image_name_is_valid(local)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Local name %s is not a suitable machine name.", + "Local name %s is not a suitable image name.", local); if (path) { @@ -369,9 +369,9 @@ static int import_fs(int argc, char *argv[], void *userdata) { return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Need either path or local name."); - if (!hostname_is_valid(local, 0)) + if (!image_name_is_valid(local)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Local name %s is not a suitable machine name.", + "Local name %s is not a suitable image name.", local); if (path) { @@ -439,9 +439,9 @@ static int export_tar(int argc, char *argv[], void *userdata) { return r; local = argv[1]; - if (!hostname_is_valid(local, 0)) + if (!image_name_is_valid(local)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Machine name %s is not valid.", local); + "Image name %s is not valid.", local); if (argc >= 3) path = argv[2]; @@ -498,9 +498,9 @@ static int export_raw(int argc, char *argv[], void *userdata) { return r; local = argv[1]; - if (!hostname_is_valid(local, 0)) + if (!image_name_is_valid(local)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Machine name %s is not valid.", local); + "Image name %s is not valid.", local); if (argc >= 3) path = argv[2]; @@ -580,9 +580,9 @@ static int pull_tar(int argc, char *argv[], void *userdata) { local = ll; - if (!hostname_is_valid(local, 0)) + if (!image_name_is_valid(local)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Local name %s is not a suitable machine name.", + "Local name %s is not a suitable image name.", local); } @@ -653,9 +653,9 @@ static int pull_raw(int argc, char *argv[], void *userdata) { local = ll; - if (!hostname_is_valid(local, 0)) + if (!image_name_is_valid(local)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Local name %s is not a suitable machine name.", + "Local name %s is not a suitable image name.", local); } diff --git a/src/import/importd.c b/src/import/importd.c index 0f6bf71419..41c329b0f2 100644 --- a/src/import/importd.c +++ b/src/import/importd.c @@ -771,9 +771,9 @@ static int method_import_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_ if (!S_ISREG(st.st_mode) && !S_ISFIFO(st.st_mode)) return -EINVAL; - if (!hostname_is_valid(local, 0)) + if (!image_name_is_valid(local)) return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, - "Local name %s is invalid", local); + "Local image name %s is invalid", local); if (class == IMAGE_MACHINE) { r = setup_machine_directory(error, m->use_btrfs_subvol, m->use_btrfs_quota); @@ -870,9 +870,9 @@ static int method_import_fs(sd_bus_message *msg, void *userdata, sd_bus_error *e if (r < 0) return r; - if (!hostname_is_valid(local, 0)) + if (!image_name_is_valid(local)) return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, - "Local name %s is invalid", local); + "Local image name %s is invalid", local); if (class == IMAGE_MACHINE) { r = setup_machine_directory(error, m->use_btrfs_subvol, m->use_btrfs_quota); @@ -955,9 +955,9 @@ static int method_export_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_ flags = 0; } - if (!hostname_is_valid(local, 0)) + if (!image_name_is_valid(local)) return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, - "Local name %s is invalid", local); + "Local image name %s is invalid", local); r = fd_verify_safe_flags(fd); if (r < 0) @@ -1063,9 +1063,9 @@ static int method_pull_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_er if (isempty(local)) local = NULL; - else if (!hostname_is_valid(local, 0)) + else if (!image_name_is_valid(local)) return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, - "Local name %s is invalid", local); + "Local image name %s is invalid", local); if (isempty(verify)) v = IMPORT_VERIFY_SIGNATURE; diff --git a/src/import/pull-common.c b/src/import/pull-common.c index 4fca08437c..9a2ced002b 100644 --- a/src/import/pull-common.c +++ b/src/import/pull-common.c @@ -7,6 +7,7 @@ #include "capability-util.h" #include "copy.h" #include "dirent-util.h" +#include "discover-image.h" #include "escape.h" #include "fd-util.h" #include "hostname-util.h" @@ -644,7 +645,7 @@ bool pull_validate_local(const char *name, ImportFlags flags) { if (FLAGS_SET(flags, IMPORT_DIRECT)) return path_is_valid(name); - return hostname_is_valid(name, 0); + return image_name_is_valid(name); } int pull_url_needs_checksum(const char *url) { diff --git a/src/import/pull.c b/src/import/pull.c index f281ab04fd..19cfbe0d8d 100644 --- a/src/import/pull.c +++ b/src/import/pull.c @@ -59,7 +59,7 @@ static int normalize_local(const char *local, const char *url, char **ret) { } else if (local) { - if (!hostname_is_valid(local, 0)) + if (!image_name_is_valid(local)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Local image name '%s' is not valid.", local); From b146afc4492e083e8f08301285b88f7202e93052 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 22 Feb 2024 18:50:32 +0100 Subject: [PATCH 16/26] importd: make keeping pristine copy of downloaded images optional Previously, when downloading an image, importd would first download them into one image which it would then consider immutable (named after the originating URL/etag), and then immediately make a copy of it (named after the client chosen name). This makes some sense in VM/container cases where the images are typically mutable, and thus the original downloaded copy is of some value. For sysexts/confexts/portable this doesn't make much sense though, as they are typically immutable. Hence make the concept optional. This adds --keep-download=yes/no as a new option that controls the above. Moreover it disables the behaviour for all image classes but "machine". The behaviour remains enabled for "machine", for compat. --- src/import/import-common.h | 26 +++++++----- src/import/importctl.c | 36 ++++++++++++++--- src/import/importd.c | 7 +++- src/import/pull-raw.c | 83 +++++++++++++++++++++++--------------- src/import/pull-tar.c | 68 ++++++++++++++++++------------- src/import/pull.c | 22 +++++++++- 6 files changed, 163 insertions(+), 79 deletions(-) diff --git a/src/import/import-common.h b/src/import/import-common.h index 2bb2075462..219802492d 100644 --- a/src/import/import-common.h +++ b/src/import/import-common.h @@ -6,27 +6,31 @@ #include "sd-event.h" typedef enum ImportFlags { + /* Public Flags (i.e. accessible via D-Bus, must stay stable! */ IMPORT_FORCE = 1 << 0, /* replace existing image */ IMPORT_READ_ONLY = 1 << 1, /* make generated image read-only */ - IMPORT_BTRFS_SUBVOL = 1 << 2, /* tar: preferably create images as btrfs subvols */ - IMPORT_BTRFS_QUOTA = 1 << 3, /* tar: set up btrfs quota for new subvolume as child of parent subvolume */ - IMPORT_CONVERT_QCOW2 = 1 << 4, /* raw: if we detect a qcow2 image, unpack it */ - IMPORT_DIRECT = 1 << 5, /* import without rename games */ - IMPORT_SYNC = 1 << 6, /* fsync() right before we are done */ + IMPORT_PULL_KEEP_DOWNLOAD = 1 << 2, /* keep a pristine copy of the downloaded file around */ + + /* Private flags */ + IMPORT_BTRFS_SUBVOL = 1 << 3, /* tar: preferably create images as btrfs subvols */ + IMPORT_BTRFS_QUOTA = 1 << 4, /* tar: set up btrfs quota for new subvolume as child of parent subvolume */ + IMPORT_CONVERT_QCOW2 = 1 << 5, /* raw: if we detect a qcow2 image, unpack it */ + IMPORT_DIRECT = 1 << 6, /* import without rename games */ + IMPORT_SYNC = 1 << 7, /* fsync() right before we are done */ /* When pulling these flags are defined too */ - IMPORT_PULL_SETTINGS = 1 << 7, /* download .nspawn settings file */ - IMPORT_PULL_ROOTHASH = 1 << 8, /* only for raw: download .roothash file for verity */ - IMPORT_PULL_ROOTHASH_SIGNATURE = 1 << 9, /* only for raw: download .roothash.p7s file for verity */ - IMPORT_PULL_VERITY = 1 << 10, /* only for raw: download .verity file for verity */ + IMPORT_PULL_SETTINGS = 1 << 8, /* download .nspawn settings file */ + IMPORT_PULL_ROOTHASH = 1 << 9, /* only for raw: download .roothash file for verity */ + IMPORT_PULL_ROOTHASH_SIGNATURE = 1 << 10, /* only for raw: download .roothash.p7s file for verity */ + IMPORT_PULL_VERITY = 1 << 11, /* only for raw: download .verity file for verity */ /* The supported flags for the tar and the raw importing */ IMPORT_FLAGS_MASK_TAR = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_BTRFS_SUBVOL|IMPORT_BTRFS_QUOTA|IMPORT_DIRECT|IMPORT_SYNC, IMPORT_FLAGS_MASK_RAW = IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_CONVERT_QCOW2|IMPORT_DIRECT|IMPORT_SYNC, /* The supported flags for the tar and the raw pulling */ - IMPORT_PULL_FLAGS_MASK_TAR = IMPORT_FLAGS_MASK_TAR|IMPORT_PULL_SETTINGS, - IMPORT_PULL_FLAGS_MASK_RAW = IMPORT_FLAGS_MASK_RAW|IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY, + IMPORT_PULL_FLAGS_MASK_TAR = IMPORT_FLAGS_MASK_TAR|IMPORT_PULL_KEEP_DOWNLOAD|IMPORT_PULL_SETTINGS, + IMPORT_PULL_FLAGS_MASK_RAW = IMPORT_FLAGS_MASK_RAW|IMPORT_PULL_KEEP_DOWNLOAD|IMPORT_PULL_SETTINGS|IMPORT_PULL_ROOTHASH|IMPORT_PULL_ROOTHASH_SIGNATURE|IMPORT_PULL_VERITY, _IMPORT_FLAGS_INVALID = -EINVAL, } ImportFlags; diff --git a/src/import/importctl.c b/src/import/importctl.c index 1fc13e1326..688e583d07 100644 --- a/src/import/importctl.c +++ b/src/import/importctl.c @@ -37,6 +37,7 @@ static bool arg_legend = true; static BusTransport arg_transport = BUS_TRANSPORT_LOCAL; static const char *arg_host = NULL; static ImportFlags arg_import_flags = 0; +static ImportFlags arg_import_flags_mask = 0; /* Indicates which flags have been explicitly set to on or to off */ static bool arg_quiet = false; static bool arg_ask_password = true; static ImportVerify arg_verify = IMPORT_VERIFY_SIGNATURE; @@ -59,6 +60,11 @@ static int settle_image_class(void) { "No image class specified, retry with --class= set to one of: %s.", j); } + /* Keep the original pristine downloaded file as a copy only when dealing with machine images, + * because unlike sysext/confext/portable they are typically modified during runtime. */ + if (!FLAGS_SET(arg_import_flags_mask, IMPORT_PULL_KEEP_DOWNLOAD)) + SET_FLAG(arg_import_flags, IMPORT_PULL_KEEP_DOWNLOAD, arg_image_class == IMAGE_MACHINE); + return 0; } @@ -586,7 +592,7 @@ static int pull_tar(int argc, char *argv[], void *userdata) { local); } - if (arg_image_class == IMAGE_MACHINE && (arg_image_class & ~(IMPORT_FORCE|IMPORT_READ_ONLY)) == 0) { + if (arg_image_class == IMAGE_MACHINE && (arg_image_class & ~IMPORT_FORCE) == 0) { r = bus_message_new_method_call(bus, &m, bus_import_mgr, "PullTar"); if (r < 0) return bus_log_create_error(r); @@ -610,7 +616,7 @@ static int pull_tar(int argc, char *argv[], void *userdata) { local, image_class_to_string(arg_image_class), import_verify_to_string(arg_verify), - (uint64_t) arg_import_flags & (IMPORT_FORCE|IMPORT_READ_ONLY)); + (uint64_t) arg_import_flags & (IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_PULL_KEEP_DOWNLOAD)); } if (r < 0) return bus_log_create_error(r); @@ -659,7 +665,7 @@ static int pull_raw(int argc, char *argv[], void *userdata) { local); } - if (arg_image_class == IMAGE_MACHINE && (arg_image_class & ~(IMPORT_FORCE|IMPORT_READ_ONLY)) == 0) { + if (arg_image_class == IMAGE_MACHINE && (arg_image_class & ~IMPORT_FORCE) == 0) { r = bus_message_new_method_call(bus, &m, bus_import_mgr, "PullRaw"); if (r < 0) return bus_log_create_error(r); @@ -683,7 +689,7 @@ static int pull_raw(int argc, char *argv[], void *userdata) { local, image_class_to_string(arg_image_class), import_verify_to_string(arg_verify), - (uint64_t) arg_import_flags & (IMPORT_FORCE|IMPORT_READ_ONLY)); + (uint64_t) arg_import_flags & (IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_PULL_KEEP_DOWNLOAD)); } if (r < 0) return bus_log_create_error(r); @@ -860,6 +866,8 @@ static int help(int argc, char *argv[], void *userdata) { " -P --class=portable Install as portable service image\n" " -S --class=sysext Install as system extension image\n" " -C --class=confext Install as configuration extension image\n" + " --keep-download=BOOL Control whether to keep pristine copy of download\n" + " -N Shortcut for --keep-download=no\n" "\nSee the %2$s for details.\n", program_invocation_short_name, link, @@ -884,6 +892,7 @@ static int parse_argv(int argc, char *argv[]) { ARG_FORCE, ARG_FORMAT, ARG_CLASS, + ARG_KEEP_DOWNLOAD, }; static const struct option options[] = { @@ -901,6 +910,7 @@ static int parse_argv(int argc, char *argv[]) { { "force", no_argument, NULL, ARG_FORCE }, { "format", required_argument, NULL, ARG_FORMAT }, { "class", required_argument, NULL, ARG_CLASS }, + { "keep-download", required_argument, NULL, ARG_KEEP_DOWNLOAD }, {} }; @@ -910,7 +920,7 @@ static int parse_argv(int argc, char *argv[]) { assert(argv); for (;;) { - c = getopt_long(argc, argv, "hH:M:jqmPSC", options, NULL); + c = getopt_long(argc, argv, "hH:M:jqmPSCN", options, NULL); if (c < 0) break; @@ -946,6 +956,7 @@ static int parse_argv(int argc, char *argv[]) { case ARG_READ_ONLY: arg_import_flags |= IMPORT_READ_ONLY; + arg_import_flags_mask |= IMPORT_READ_ONLY; break; case 'q': @@ -966,6 +977,7 @@ static int parse_argv(int argc, char *argv[]) { case ARG_FORCE: arg_import_flags |= IMPORT_FORCE; + arg_import_flags_mask |= IMPORT_FORCE; break; case ARG_FORMAT: @@ -1011,6 +1023,20 @@ static int parse_argv(int argc, char *argv[]) { arg_image_class = IMAGE_CONFEXT; break; + case ARG_KEEP_DOWNLOAD: + r = parse_boolean(optarg); + if (r < 0) + return log_error_errno(r, "Failed to parse --keep-download= value: %s", optarg); + + SET_FLAG(arg_import_flags, IMPORT_PULL_KEEP_DOWNLOAD, r); + arg_import_flags_mask |= IMPORT_PULL_KEEP_DOWNLOAD; + break; + + case 'N': + arg_import_flags_mask &= ~IMPORT_PULL_KEEP_DOWNLOAD; + arg_import_flags_mask |= IMPORT_PULL_KEEP_DOWNLOAD; + break; + case '?': return -EINVAL; diff --git a/src/import/importd.c b/src/import/importd.c index 41c329b0f2..a07ceb2288 100644 --- a/src/import/importd.c +++ b/src/import/importd.c @@ -386,6 +386,7 @@ static int transfer_start(Transfer *t) { NULL, /* verify argument */ NULL, /* --class= */ NULL, /* class argument */ + NULL, /* --keep-download= */ NULL, /* maybe --force */ NULL, /* maybe --read-only */ NULL, /* if so: the actual URL */ @@ -466,6 +467,10 @@ static int transfer_start(Transfer *t) { cmd[k++] = image_class_to_string(t->class); } + if (IN_SET(t->type, TRANSFER_PULL_TAR, TRANSFER_PULL_RAW)) + cmd[k++] = FLAGS_SET(t->flags, IMPORT_PULL_KEEP_DOWNLOAD) ? + "--keep-download=yes" : "--keep-download=no"; + if (FLAGS_SET(t->flags, IMPORT_FORCE)) cmd[k++] = "--force"; if (FLAGS_SET(t->flags, IMPORT_READ_ONLY)) @@ -1041,7 +1046,7 @@ static int method_pull_tar_or_raw(sd_bus_message *msg, void *userdata, sd_bus_er return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Image class '%s' not known", sclass); - if (flags & ~(IMPORT_FORCE|IMPORT_READ_ONLY)) + if (flags & ~(IMPORT_FORCE|IMPORT_READ_ONLY|IMPORT_PULL_KEEP_DOWNLOAD)) return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, "Flags 0x%" PRIx64 " invalid", flags); } else { diff --git a/src/import/pull-raw.c b/src/import/pull-raw.c index 50785fe431..f3e6b3af8c 100644 --- a/src/import/pull-raw.c +++ b/src/import/pull-raw.c @@ -311,7 +311,7 @@ static int raw_pull_copy_auxiliary_file( const char *suffix, char **path /* input + output (!) */) { - const char *local; + _cleanup_free_ char *local = NULL; int r; assert(i); @@ -322,21 +322,29 @@ static int raw_pull_copy_auxiliary_file( if (r < 0) return r; - local = strjoina(i->image_root, "/", i->local, suffix); + local = strjoin(i->image_root, "/", i->local, suffix); + if (!local) + return log_oom(); - r = copy_file_atomic( - *path, - local, - 0644, - COPY_REFLINK | - (FLAGS_SET(i->flags, IMPORT_FORCE) ? COPY_REPLACE : 0) | - (FLAGS_SET(i->flags, IMPORT_SYNC) ? COPY_FSYNC_FULL : 0)); + if (FLAGS_SET(i->flags, IMPORT_PULL_KEEP_DOWNLOAD)) + r = copy_file_atomic( + *path, + local, + 0644, + COPY_REFLINK | + (FLAGS_SET(i->flags, IMPORT_FORCE) ? COPY_REPLACE : 0) | + (FLAGS_SET(i->flags, IMPORT_SYNC) ? COPY_FSYNC_FULL : 0)); + else + r = install_file(AT_FDCWD, *path, + AT_FDCWD, local, + (i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) | + (i->flags & IMPORT_SYNC ? INSTALL_SYNCFS : 0)); if (r == -EEXIST) log_warning_errno(r, "File %s already exists, not replacing.", local); else if (r == -ENOENT) log_debug_errno(r, "Skipping creation of auxiliary file, since none was found."); else if (r < 0) - log_warning_errno(r, "Failed to copy file %s, ignoring: %m", local); + log_warning_errno(r, "Failed to install file %s, ignoring: %m", local); else log_info("Created new file %s.", local); @@ -345,9 +353,7 @@ static int raw_pull_copy_auxiliary_file( static int raw_pull_make_local_copy(RawPull *i) { _cleanup_(unlink_and_freep) char *tp = NULL; - _cleanup_free_ char *f = NULL; - _cleanup_close_ int dfd = -EBADF; - const char *p; + _cleanup_free_ char *p = NULL; int r; assert(i); @@ -375,38 +381,49 @@ static int raw_pull_make_local_copy(RawPull *i) { return log_error_errno(errno, "Failed to seek to beginning of vendor image: %m"); } - p = strjoina(i->image_root, "/", i->local, ".raw"); - - r = tempfn_random(p, NULL, &f); - if (r < 0) + p = strjoin(i->image_root, "/", i->local, ".raw"); + if (!p) return log_oom(); - dfd = open(f, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); - if (dfd < 0) - return log_error_errno(errno, "Failed to create writable copy of image: %m"); + const char *source; + if (FLAGS_SET(i->flags, IMPORT_PULL_KEEP_DOWNLOAD)) { + _cleanup_close_ int dfd = -EBADF; + _cleanup_free_ char *f = NULL; - tp = TAKE_PTR(f); + r = tempfn_random(p, NULL, &f); + if (r < 0) + return log_oom(); - /* Turn off COW writing. This should greatly improve performance on COW file systems like btrfs, - * since it reduces fragmentation caused by not allowing in-place writes. */ - (void) import_set_nocow_and_log(dfd, tp); + dfd = open(f, O_WRONLY|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664); + if (dfd < 0) + return log_error_errno(errno, "Failed to create writable copy of image: %m"); - r = copy_bytes(i->raw_job->disk_fd, dfd, UINT64_MAX, COPY_REFLINK); - if (r < 0) - return log_error_errno(r, "Failed to make writable copy of image: %m"); + tp = TAKE_PTR(f); - (void) copy_times(i->raw_job->disk_fd, dfd, COPY_CRTIME); - (void) copy_xattr(i->raw_job->disk_fd, NULL, dfd, NULL, 0); + /* Turn off COW writing. This should greatly improve performance on COW file systems like btrfs, + * since it reduces fragmentation caused by not allowing in-place writes. */ + (void) import_set_nocow_and_log(dfd, tp); - dfd = safe_close(dfd); + r = copy_bytes(i->raw_job->disk_fd, dfd, UINT64_MAX, COPY_REFLINK); + if (r < 0) + return log_error_errno(r, "Failed to make writable copy of image: %m"); - r = install_file(AT_FDCWD, tp, + (void) copy_times(i->raw_job->disk_fd, dfd, COPY_CRTIME); + (void) copy_xattr(i->raw_job->disk_fd, NULL, dfd, NULL, 0); + + dfd = safe_close(dfd); + + source = tp; + } else + source = i->final_path; + + r = install_file(AT_FDCWD, source, AT_FDCWD, p, (i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) | (i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) | (i->flags & IMPORT_SYNC ? INSTALL_FSYNC_FULL : 0)); if (r < 0) - return log_error_errno(errno, "Failed to move local image into place '%s': %m", p); + return log_error_errno(r, "Failed to move local image into place '%s': %m", p); tp = mfree(tp); @@ -628,7 +645,7 @@ static void raw_pull_job_on_finished(PullJob *j) { r = install_file(AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path, - INSTALL_READ_ONLY| + (i->flags & IMPORT_PULL_KEEP_DOWNLOAD ? INSTALL_READ_ONLY : 0) | (i->flags & IMPORT_SYNC ? INSTALL_FSYNC_FULL : 0)); if (r < 0) { log_error_errno(r, "Failed to move raw file to '%s': %m", i->final_path); diff --git a/src/import/pull-tar.c b/src/import/pull-tar.c index ae573f1b54..7fc71fe6f3 100644 --- a/src/import/pull-tar.c +++ b/src/import/pull-tar.c @@ -221,6 +221,7 @@ static int tar_pull_determine_path( static int tar_pull_make_local_copy(TarPull *i) { _cleanup_(rm_rf_subvolume_and_freep) char *t = NULL; _cleanup_free_ char *p = NULL; + const char *source; int r; assert(i); @@ -235,24 +236,29 @@ static int tar_pull_make_local_copy(TarPull *i) { if (!p) return log_oom(); - r = tempfn_random(p, NULL, &t); - if (r < 0) - return log_error_errno(r, "Failed to generate temporary filename for %s: %m", p); + if (FLAGS_SET(i->flags, IMPORT_PULL_KEEP_DOWNLOAD)) { + r = tempfn_random(p, NULL, &t); + if (r < 0) + return log_error_errno(r, "Failed to generate temporary filename for %s: %m", p); - if (i->flags & IMPORT_BTRFS_SUBVOL) - r = btrfs_subvol_snapshot_at( - AT_FDCWD, i->final_path, - AT_FDCWD, t, - (i->flags & IMPORT_BTRFS_QUOTA ? BTRFS_SNAPSHOT_QUOTA : 0)| - BTRFS_SNAPSHOT_FALLBACK_COPY| - BTRFS_SNAPSHOT_FALLBACK_DIRECTORY| - BTRFS_SNAPSHOT_RECURSIVE); - else - r = copy_tree(i->final_path, t, UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_HARDLINKS, NULL, NULL); - if (r < 0) - return log_error_errno(r, "Failed to create local image: %m"); + if (i->flags & IMPORT_BTRFS_SUBVOL) + r = btrfs_subvol_snapshot_at( + AT_FDCWD, i->final_path, + AT_FDCWD, t, + (i->flags & IMPORT_BTRFS_QUOTA ? BTRFS_SNAPSHOT_QUOTA : 0)| + BTRFS_SNAPSHOT_FALLBACK_COPY| + BTRFS_SNAPSHOT_FALLBACK_DIRECTORY| + BTRFS_SNAPSHOT_RECURSIVE); + else + r = copy_tree(i->final_path, t, UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_HARDLINKS, NULL, NULL); + if (r < 0) + return log_error_errno(r, "Failed to create local image: %m"); - r = install_file(AT_FDCWD, t, + source = t; + } else + source = i->final_path; + + r = install_file(AT_FDCWD, source, AT_FDCWD, p, (i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) | (i->flags & IMPORT_READ_ONLY ? INSTALL_READ_ONLY : 0) | @@ -265,28 +271,36 @@ static int tar_pull_make_local_copy(TarPull *i) { log_info("Created new local image '%s'.", i->local); if (FLAGS_SET(i->flags, IMPORT_PULL_SETTINGS)) { - const char *local_settings; + _cleanup_free_ char *local_settings = NULL; assert(i->settings_job); r = tar_pull_determine_path(i, ".nspawn", &i->settings_path); if (r < 0) return r; - local_settings = strjoina(i->image_root, "/", i->local, ".nspawn"); + local_settings = strjoin(i->image_root, "/", i->local, ".nspawn"); + if (!local_settings) + return log_oom(); - r = copy_file_atomic( - i->settings_path, - local_settings, - 0664, - COPY_REFLINK | - (FLAGS_SET(i->flags, IMPORT_FORCE) ? COPY_REPLACE : 0) | - (FLAGS_SET(i->flags, IMPORT_SYNC) ? COPY_FSYNC_FULL : 0)); + if (FLAGS_SET(i->flags, IMPORT_PULL_KEEP_DOWNLOAD)) + r = copy_file_atomic( + i->settings_path, + local_settings, + 0664, + COPY_REFLINK | + (FLAGS_SET(i->flags, IMPORT_FORCE) ? COPY_REPLACE : 0) | + (FLAGS_SET(i->flags, IMPORT_SYNC) ? COPY_FSYNC_FULL : 0)); + else + r = install_file(AT_FDCWD, i->settings_path, + AT_FDCWD, local_settings, + (i->flags & IMPORT_FORCE ? INSTALL_REPLACE : 0) | + (i->flags & IMPORT_SYNC ? INSTALL_SYNCFS : 0)); if (r == -EEXIST) log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings); else if (r == -ENOENT) log_debug_errno(r, "Skipping creation of settings file, since none was found."); else if (r < 0) - log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings); + log_warning_errno(r, "Failed to install settings files %s, ignoring: %m", local_settings); else log_info("Created new settings file %s.", local_settings); } @@ -435,7 +449,7 @@ static void tar_pull_job_on_finished(PullJob *j) { r = install_file( AT_FDCWD, i->temp_path, AT_FDCWD, i->final_path, - INSTALL_READ_ONLY| + (i->flags & IMPORT_PULL_KEEP_DOWNLOAD ? INSTALL_READ_ONLY : 0) | (i->flags & IMPORT_SYNC ? INSTALL_SYNCFS : 0)); if (r < 0) { log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path); diff --git a/src/import/pull.c b/src/import/pull.c index 19cfbe0d8d..d4ad0b2ea3 100644 --- a/src/import/pull.c +++ b/src/import/pull.c @@ -270,7 +270,9 @@ static int help(int argc, char *argv[], void *userdata) { " --offset=BYTES Offset to seek to in destination\n" " --size-max=BYTES Maximum number of bytes to write to destination\n" " --class=CLASS Select image class (machine, sysext, confext,\n" - " portable)\n", + " portable)\n" + " --keep-download=BOOL Keep a copy pristine copy of the downloaded file\n" + " around\n", program_invocation_short_name, ansi_underline(), ansi_normal(), @@ -300,6 +302,7 @@ static int parse_argv(int argc, char *argv[]) { ARG_OFFSET, ARG_SIZE_MAX, ARG_CLASS, + ARG_KEEP_DOWNLOAD, }; static const struct option options[] = { @@ -321,11 +324,12 @@ static int parse_argv(int argc, char *argv[]) { { "offset", required_argument, NULL, ARG_OFFSET }, { "size-max", required_argument, NULL, ARG_SIZE_MAX }, { "class", required_argument, NULL, ARG_CLASS }, + { "keep-download", required_argument, NULL, ARG_KEEP_DOWNLOAD }, {} }; int c, r; - bool auto_settings = true; + bool auto_settings = true, auto_keep_download = true; assert(argc >= 0); assert(argv); @@ -492,6 +496,15 @@ static int parse_argv(int argc, char *argv[]) { break; + case ARG_KEEP_DOWNLOAD: + r = parse_boolean(optarg); + if (r < 0) + return log_error_errno(r, "Failed to parse --keep-download= argument: %s", optarg); + + SET_FLAG(arg_import_flags, IMPORT_PULL_KEEP_DOWNLOAD, r); + auto_keep_download = false; + break; + case '?': return -EINVAL; @@ -518,6 +531,11 @@ static int parse_argv(int argc, char *argv[]) { if (auto_settings && arg_class != IMAGE_MACHINE) arg_import_flags &= ~IMPORT_PULL_SETTINGS; + /* Keep the original pristine downloaded file as a copy only when dealing with machine images, + * because unlike sysext/confext/portable they are typically modified during runtime. */ + if (auto_keep_download) + SET_FLAG(arg_import_flags, IMPORT_PULL_KEEP_DOWNLOAD, arg_class == IMAGE_MACHINE); + return 1; } From 1a176d5bb730791f5a6a16a768323cc6f863666c Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 23 Feb 2024 21:50:29 +0100 Subject: [PATCH 17/26] importd: log the import callout that is going to be called --- src/import/importd.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/import/importd.c b/src/import/importd.c index a07ceb2288..5aeb0b1895 100644 --- a/src/import/importd.c +++ b/src/import/importd.c @@ -374,7 +374,7 @@ static int transfer_start(Transfer *t) { "(sd-transfer)", (int[]) { t->stdin_fd, t->stdout_fd < 0 ? pipefd[1] : t->stdout_fd, pipefd[1] }, /* except_fds= */ NULL, /* n_except_fds= */ 0, - FORK_RESET_SIGNALS|FORK_CLOSE_ALL_FDS|FORK_DEATHSIG_SIGTERM|FORK_REARRANGE_STDIO, + FORK_RESET_SIGNALS|FORK_CLOSE_ALL_FDS|FORK_DEATHSIG_SIGTERM|FORK_REARRANGE_STDIO|FORK_REOPEN_LOG, &t->pidref); if (r < 0) return r; @@ -494,6 +494,11 @@ static int transfer_start(Transfer *t) { assert(k < ELEMENTSOF(cmd)); + if (DEBUG_LOGGING) { + _cleanup_free_ char *joined = strv_join((char**) cmd, " "); + log_debug("Calling: %s", strnull(joined)); + } + r = invoke_callout_binary(cmd[0], (char * const *) cmd); log_error_errno(r, "Failed to execute %s tool: %m", cmd[0]); _exit(EXIT_FAILURE); From 8ce438bb711179816a15b9318658f56fcbb9c15b Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 22 Feb 2024 18:54:39 +0100 Subject: [PATCH 18/26] machinectl: chainload importctl for relevant verbs Now that "importctl" exists, let's chainload it from machinectl for the relevant verbs so that we only have a single implementation of the logic. --- src/machine/machinectl.c | 707 ++++----------------------------------- 1 file changed, 56 insertions(+), 651 deletions(-) diff --git a/src/machine/machinectl.c b/src/machine/machinectl.c index a7d04f2f4d..eced63a627 100644 --- a/src/machine/machinectl.c +++ b/src/machine/machinectl.c @@ -15,6 +15,7 @@ #include "alloc-util.h" #include "build.h" +#include "build-path.h" #include "bus-common-errors.h" #include "bus-error.h" #include "bus-locator.h" @@ -1879,633 +1880,6 @@ static int enable_machine(int argc, char *argv[], void *userdata) { return 0; } -static int match_log_message(sd_bus_message *m, void *userdata, sd_bus_error *error) { - const char **our_path = userdata, *line; - unsigned priority; - int r; - - assert(m); - assert(our_path); - - r = sd_bus_message_read(m, "us", &priority, &line); - if (r < 0) { - bus_log_parse_error(r); - return 0; - } - - if (!streq_ptr(*our_path, sd_bus_message_get_path(m))) - return 0; - - if (arg_quiet && LOG_PRI(priority) >= LOG_INFO) - return 0; - - log_full(priority, "%s", line); - return 0; -} - -static int match_transfer_removed(sd_bus_message *m, void *userdata, sd_bus_error *error) { - const char **our_path = userdata, *path, *result; - uint32_t id; - int r; - - assert(m); - assert(our_path); - - r = sd_bus_message_read(m, "uos", &id, &path, &result); - if (r < 0) { - bus_log_parse_error(r); - return 0; - } - - if (!streq_ptr(*our_path, path)) - return 0; - - sd_event_exit(sd_bus_get_event(sd_bus_message_get_bus(m)), !streq_ptr(result, "done")); - return 0; -} - -static int transfer_signal_handler(sd_event_source *s, const struct signalfd_siginfo *si, void *userdata) { - assert(s); - assert(si); - - if (!arg_quiet) - log_info("Continuing download in the background. Use \"machinectl cancel-transfer %" PRIu32 "\" to abort transfer.", PTR_TO_UINT32(userdata)); - - sd_event_exit(sd_event_source_get_event(s), EINTR); - return 0; -} - -static int transfer_image_common(sd_bus *bus, sd_bus_message *m) { - _cleanup_(sd_bus_slot_unrefp) sd_bus_slot *slot_job_removed = NULL, *slot_log_message = NULL; - _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL; - _cleanup_(sd_bus_message_unrefp) sd_bus_message *reply = NULL; - _cleanup_(sd_event_unrefp) sd_event* event = NULL; - const char *path = NULL; - uint32_t id; - int r; - - assert(bus); - assert(m); - - polkit_agent_open_if_enabled(arg_transport, arg_ask_password); - - r = sd_event_default(&event); - if (r < 0) - return log_error_errno(r, "Failed to get event loop: %m"); - - r = sd_bus_attach_event(bus, event, 0); - if (r < 0) - return log_error_errno(r, "Failed to attach bus to event loop: %m"); - - r = bus_match_signal_async( - bus, - &slot_job_removed, - bus_import_mgr, - "TransferRemoved", - match_transfer_removed, NULL, &path); - if (r < 0) - return log_error_errno(r, "Failed to request match: %m"); - - r = sd_bus_match_signal_async( - bus, - &slot_log_message, - "org.freedesktop.import1", - NULL, - "org.freedesktop.import1.Transfer", - "LogMessage", - match_log_message, NULL, &path); - if (r < 0) - return log_error_errno(r, "Failed to request match: %m"); - - r = sd_bus_call(bus, m, 0, &error, &reply); - if (r < 0) - return log_error_errno(r, "Failed to transfer image: %s", bus_error_message(&error, r)); - - r = sd_bus_message_read(reply, "uo", &id, &path); - if (r < 0) - return bus_log_parse_error(r); - - assert_se(sigprocmask_many(SIG_BLOCK, NULL, SIGTERM, SIGINT) >= 0); - - if (!arg_quiet) - log_info("Enqueued transfer job %u. Press C-c to continue download in background.", id); - - (void) sd_event_add_signal(event, NULL, SIGINT, transfer_signal_handler, UINT32_TO_PTR(id)); - (void) sd_event_add_signal(event, NULL, SIGTERM, transfer_signal_handler, UINT32_TO_PTR(id)); - - r = sd_event_loop(event); - if (r < 0) - return log_error_errno(r, "Failed to run event loop: %m"); - - return -r; -} - -static int import_tar(int argc, char *argv[], void *userdata) { - _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL; - _cleanup_free_ char *ll = NULL, *fn = NULL; - const char *local = NULL, *path = NULL; - _cleanup_close_ int fd = -EBADF; - sd_bus *bus = ASSERT_PTR(userdata); - int r; - - if (argc >= 2) - path = empty_or_dash_to_null(argv[1]); - - if (argc >= 3) - local = empty_or_dash_to_null(argv[2]); - else if (path) { - r = path_extract_filename(path, &fn); - if (r < 0) - return log_error_errno(r, "Cannot extract container name from filename: %m"); - if (r == O_DIRECTORY) - return log_error_errno(SYNTHETIC_ERRNO(EISDIR), - "Path '%s' refers to directory, but we need a regular file: %m", path); - - local = fn; - } - if (!local) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Need either path or local name."); - - r = tar_strip_suffixes(local, &ll); - if (r < 0) - return log_oom(); - - local = ll; - - if (!hostname_is_valid(local, 0)) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Local name %s is not a suitable machine name.", - local); - - if (path) { - fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); - if (fd < 0) - return log_error_errno(errno, "Failed to open %s: %m", path); - } - - r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ImportTar"); - if (r < 0) - return bus_log_create_error(r); - - r = sd_bus_message_append( - m, - "hsbb", - fd >= 0 ? fd : STDIN_FILENO, - local, - arg_force, - arg_read_only); - if (r < 0) - return bus_log_create_error(r); - - return transfer_image_common(bus, m); -} - -static int import_raw(int argc, char *argv[], void *userdata) { - _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL; - _cleanup_free_ char *ll = NULL, *fn = NULL; - const char *local = NULL, *path = NULL; - _cleanup_close_ int fd = -EBADF; - sd_bus *bus = ASSERT_PTR(userdata); - int r; - - if (argc >= 2) - path = empty_or_dash_to_null(argv[1]); - - if (argc >= 3) - local = empty_or_dash_to_null(argv[2]); - else if (path) { - r = path_extract_filename(path, &fn); - if (r < 0) - return log_error_errno(r, "Cannot extract container name from filename: %m"); - if (r == O_DIRECTORY) - return log_error_errno(SYNTHETIC_ERRNO(EISDIR), - "Path '%s' refers to directory, but we need a regular file: %m", path); - - local = fn; - } - if (!local) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Need either path or local name."); - - r = raw_strip_suffixes(local, &ll); - if (r < 0) - return log_oom(); - - local = ll; - - if (!hostname_is_valid(local, 0)) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Local name %s is not a suitable machine name.", - local); - - if (path) { - fd = open(path, O_RDONLY|O_CLOEXEC|O_NOCTTY); - if (fd < 0) - return log_error_errno(errno, "Failed to open %s: %m", path); - } - - r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ImportRaw"); - if (r < 0) - return bus_log_create_error(r); - - r = sd_bus_message_append( - m, - "hsbb", - fd >= 0 ? fd : STDIN_FILENO, - local, - arg_force, - arg_read_only); - if (r < 0) - return bus_log_create_error(r); - - return transfer_image_common(bus, m); -} - -static int import_fs(int argc, char *argv[], void *userdata) { - _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL; - const char *local = NULL, *path = NULL; - _cleanup_free_ char *fn = NULL; - _cleanup_close_ int fd = -EBADF; - sd_bus *bus = ASSERT_PTR(userdata); - int r; - - if (argc >= 2) - path = empty_or_dash_to_null(argv[1]); - - if (argc >= 3) - local = empty_or_dash_to_null(argv[2]); - else if (path) { - r = path_extract_filename(path, &fn); - if (r < 0) - return log_error_errno(r, "Cannot extract container name from filename: %m"); - - local = fn; - } - if (!local) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Need either path or local name."); - - if (!hostname_is_valid(local, 0)) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Local name %s is not a suitable machine name.", - local); - - if (path) { - fd = open(path, O_DIRECTORY|O_RDONLY|O_CLOEXEC); - if (fd < 0) - return log_error_errno(errno, "Failed to open directory '%s': %m", path); - } - - r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ImportFileSystem"); - if (r < 0) - return bus_log_create_error(r); - - r = sd_bus_message_append( - m, - "hsbb", - fd >= 0 ? fd : STDIN_FILENO, - local, - arg_force, - arg_read_only); - if (r < 0) - return bus_log_create_error(r); - - return transfer_image_common(bus, m); -} - -static void determine_compression_from_filename(const char *p) { - if (arg_format) - return; - - if (!p) - return; - - if (endswith(p, ".xz")) - arg_format = "xz"; - else if (endswith(p, ".gz")) - arg_format = "gzip"; - else if (endswith(p, ".bz2")) - arg_format = "bzip2"; -} - -static int export_tar(int argc, char *argv[], void *userdata) { - _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL; - _cleanup_close_ int fd = -EBADF; - const char *local = NULL, *path = NULL; - sd_bus *bus = ASSERT_PTR(userdata); - int r; - - local = argv[1]; - if (!hostname_is_valid(local, 0)) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Machine name %s is not valid.", local); - - if (argc >= 3) - path = argv[2]; - path = empty_or_dash_to_null(path); - - if (path) { - determine_compression_from_filename(path); - - fd = open(path, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_NOCTTY, 0666); - if (fd < 0) - return log_error_errno(errno, "Failed to open %s: %m", path); - } - - r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ExportTar"); - if (r < 0) - return bus_log_create_error(r); - - r = sd_bus_message_append( - m, - "shs", - local, - fd >= 0 ? fd : STDOUT_FILENO, - arg_format); - if (r < 0) - return bus_log_create_error(r); - - return transfer_image_common(bus, m); -} - -static int export_raw(int argc, char *argv[], void *userdata) { - _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL; - _cleanup_close_ int fd = -EBADF; - const char *local = NULL, *path = NULL; - sd_bus *bus = ASSERT_PTR(userdata); - int r; - - local = argv[1]; - if (!hostname_is_valid(local, 0)) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Machine name %s is not valid.", local); - - if (argc >= 3) - path = argv[2]; - path = empty_or_dash_to_null(path); - - if (path) { - determine_compression_from_filename(path); - - fd = open(path, O_WRONLY|O_CREAT|O_TRUNC|O_CLOEXEC|O_NOCTTY, 0666); - if (fd < 0) - return log_error_errno(errno, "Failed to open %s: %m", path); - } - - r = bus_message_new_method_call(bus, &m, bus_import_mgr, "ExportRaw"); - if (r < 0) - return bus_log_create_error(r); - - r = sd_bus_message_append( - m, - "shs", - local, - fd >= 0 ? fd : STDOUT_FILENO, - arg_format); - if (r < 0) - return bus_log_create_error(r); - - return transfer_image_common(bus, m); -} - -static int pull_tar(int argc, char *argv[], void *userdata) { - _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL; - _cleanup_free_ char *l = NULL, *ll = NULL; - const char *local, *remote; - sd_bus *bus = ASSERT_PTR(userdata); - int r; - - remote = argv[1]; - if (!http_url_is_valid(remote) && !file_url_is_valid(remote)) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "URL '%s' is not valid.", remote); - - if (argc >= 3) - local = argv[2]; - else { - r = import_url_last_component(remote, &l); - if (r < 0) - return log_error_errno(r, "Failed to get final component of URL: %m"); - - local = l; - } - - local = empty_or_dash_to_null(local); - - if (local) { - r = tar_strip_suffixes(local, &ll); - if (r < 0) - return log_oom(); - - local = ll; - - if (!hostname_is_valid(local, 0)) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Local name %s is not a suitable machine name.", - local); - } - - r = bus_message_new_method_call(bus, &m, bus_import_mgr, "PullTar"); - if (r < 0) - return bus_log_create_error(r); - - r = sd_bus_message_append( - m, - "sssb", - remote, - local, - import_verify_to_string(arg_verify), - arg_force); - if (r < 0) - return bus_log_create_error(r); - - return transfer_image_common(bus, m); -} - -static int pull_raw(int argc, char *argv[], void *userdata) { - _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL; - _cleanup_free_ char *l = NULL, *ll = NULL; - const char *local, *remote; - sd_bus *bus = ASSERT_PTR(userdata); - int r; - - remote = argv[1]; - if (!http_url_is_valid(remote) && !file_url_is_valid(remote)) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "URL '%s' is not valid.", remote); - - if (argc >= 3) - local = argv[2]; - else { - r = import_url_last_component(remote, &l); - if (r < 0) - return log_error_errno(r, "Failed to get final component of URL: %m"); - - local = l; - } - - local = empty_or_dash_to_null(local); - - if (local) { - r = raw_strip_suffixes(local, &ll); - if (r < 0) - return log_oom(); - - local = ll; - - if (!hostname_is_valid(local, 0)) - return log_error_errno(SYNTHETIC_ERRNO(EINVAL), - "Local name %s is not a suitable machine name.", - local); - } - - r = bus_message_new_method_call(bus, &m, bus_import_mgr, "PullRaw"); - if (r < 0) - return bus_log_create_error(r); - - r = sd_bus_message_append( - m, - "sssb", - remote, - local, - import_verify_to_string(arg_verify), - arg_force); - if (r < 0) - return bus_log_create_error(r); - - return transfer_image_common(bus, m); -} - -typedef struct TransferInfo { - uint32_t id; - const char *type; - const char *remote; - const char *local; - double progress; -} TransferInfo; - -static int compare_transfer_info(const TransferInfo *a, const TransferInfo *b) { - return strcmp(a->local, b->local); -} - -static int list_transfers(int argc, char *argv[], void *userdata) { - size_t max_type = STRLEN("TYPE"), max_local = STRLEN("LOCAL"), max_remote = STRLEN("REMOTE"); - _cleanup_(sd_bus_message_unrefp) sd_bus_message *reply = NULL; - _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL; - _cleanup_free_ TransferInfo *transfers = NULL; - const char *type, *remote, *local; - sd_bus *bus = userdata; - uint32_t id, max_id = 0; - size_t n_transfers = 0; - double progress; - int r; - - pager_open(arg_pager_flags); - - r = bus_call_method(bus, bus_import_mgr, "ListTransfers", &error, &reply, NULL); - if (r < 0) - return log_error_errno(r, "Could not get transfers: %s", bus_error_message(&error, r)); - - r = sd_bus_message_enter_container(reply, 'a', "(usssdo)"); - if (r < 0) - return bus_log_parse_error(r); - - while ((r = sd_bus_message_read(reply, "(usssdo)", &id, &type, &remote, &local, &progress, NULL)) > 0) { - size_t l; - - if (!GREEDY_REALLOC(transfers, n_transfers + 1)) - return log_oom(); - - transfers[n_transfers].id = id; - transfers[n_transfers].type = type; - transfers[n_transfers].remote = remote; - transfers[n_transfers].local = local; - transfers[n_transfers].progress = progress; - - l = strlen(type); - if (l > max_type) - max_type = l; - - l = strlen(remote); - if (l > max_remote) - max_remote = l; - - l = strlen(local); - if (l > max_local) - max_local = l; - - if (id > max_id) - max_id = id; - - n_transfers++; - } - if (r < 0) - return bus_log_parse_error(r); - - r = sd_bus_message_exit_container(reply); - if (r < 0) - return bus_log_parse_error(r); - - typesafe_qsort(transfers, n_transfers, compare_transfer_info); - - if (arg_legend && n_transfers > 0) - printf("%-*s %-*s %-*s %-*s %-*s\n", - (int) MAX(2U, DECIMAL_STR_WIDTH(max_id)), "ID", - (int) 7, "PERCENT", - (int) max_type, "TYPE", - (int) max_local, "LOCAL", - (int) max_remote, "REMOTE"); - - for (size_t j = 0; j < n_transfers; j++) - - if (transfers[j].progress < 0) - printf("%*" PRIu32 " %*s %-*s %-*s %-*s\n", - (int) MAX(2U, DECIMAL_STR_WIDTH(max_id)), transfers[j].id, - (int) 7, "n/a", - (int) max_type, transfers[j].type, - (int) max_local, transfers[j].local, - (int) max_remote, transfers[j].remote); - else - printf("%*" PRIu32 " %*u%% %-*s %-*s %-*s\n", - (int) MAX(2U, DECIMAL_STR_WIDTH(max_id)), transfers[j].id, - (int) 6, (unsigned) (transfers[j].progress * 100), - (int) max_type, transfers[j].type, - (int) max_local, transfers[j].local, - (int) max_remote, transfers[j].remote); - - if (arg_legend) { - if (n_transfers > 0) - printf("\n%zu transfers listed.\n", n_transfers); - else - printf("No transfers.\n"); - } - - return 0; -} - -static int cancel_transfer(int argc, char *argv[], void *userdata) { - _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL; - sd_bus *bus = ASSERT_PTR(userdata); - int r; - - polkit_agent_open_if_enabled(arg_transport, arg_ask_password); - - for (int i = 1; i < argc; i++) { - uint32_t id; - - r = safe_atou32(argv[i], &id); - if (r < 0) - return log_error_errno(r, "Failed to parse transfer id: %s", argv[i]); - - r = bus_call_method(bus, bus_import_mgr, "CancelTransfer", &error, NULL, "u", id); - if (r < 0) - return log_error_errno(r, "Could not cancel transfer: %s", bus_error_message(&error, r)); - } - - return 0; -} - static int set_limit(int argc, char *argv[], void *userdata) { _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL; sd_bus *bus = userdata; @@ -2590,6 +1964,52 @@ static int clean_images(int argc, char *argv[], void *userdata) { return 0; } +static int chainload_importctl(int argc, char *argv[]) { + int r; + + log_notice("The 'machinectl %1$s' command has been replaced by 'importctl -m %1$s'. Redirecting invocation.", argv[optind]); + + _cleanup_strv_free_ char **c = + strv_new("importctl", "--class=machine"); + if (!c) + return log_oom(); + + if (FLAGS_SET(arg_pager_flags, PAGER_DISABLE)) + if (strv_extend(&c, "--no-pager") < 0) + return log_oom(); + if (!arg_legend) + if (strv_extend(&c, "--no-legend") < 0) + return log_oom(); + if (arg_read_only) + if (strv_extend(&c, "--read-only") < 0) + return log_oom(); + if (arg_force) + if (strv_extend(&c, "--force") < 0) + return log_oom(); + if (arg_quiet) + if (strv_extend(&c, "--quiet") < 0) + return log_oom(); + if (!arg_ask_password) + if (strv_extend(&c, "--no-ask-password") < 0) + return log_oom(); + if (strv_extend_many(&c, "--verify", import_verify_to_string(arg_verify)) < 0) + return log_oom(); + if (arg_format) + if (strv_extend_many(&c, "--format", arg_format) < 0) + return log_oom(); + + if (strv_extend_strv(&c, argv + optind, /* filter_duplicates= */ false) < 0) + return log_oom(); + + if (DEBUG_LOGGING) { + _cleanup_free_ char *joined = strv_join(c, " "); + log_debug("Chainloading: %s", joined); + } + + r = invoke_callout_binary(BINDIR "/importctl", c); + return log_error_errno(r, "Failed to invoke 'importctl': %m"); +} + static int help(int argc, char *argv[], void *userdata) { _cleanup_free_ char *link = NULL; int r; @@ -2634,16 +2054,6 @@ static int help(int argc, char *argv[], void *userdata) { " remove NAME... Remove an image\n" " set-limit [NAME] BYTES Set image or pool size limit (disk quota)\n" " clean Remove hidden (or all) images\n" - "\n%3$sImage Transfer Commands:%4$s\n" - " pull-tar URL [NAME] Download a TAR container image\n" - " pull-raw URL [NAME] Download a RAW container or VM image\n" - " import-tar FILE [NAME] Import a local TAR container image\n" - " import-raw FILE [NAME] Import a local RAW container or VM image\n" - " import-fs DIRECTORY [NAME] Import a local directory container image\n" - " export-tar NAME [FILE] Export a TAR container image locally\n" - " export-raw NAME [FILE] Export a RAW container or VM image locally\n" - " list-transfers Show list of downloads in progress\n" - " cancel-transfer Cancel a download\n" "\n%3$sOptions:%4$s\n" " -h --help Show this help\n" " --version Show package version\n" @@ -2661,7 +2071,7 @@ static int help(int argc, char *argv[], void *userdata) { " -s --signal=SIGNAL Which signal to send\n" " --uid=USER Specify user ID to invoke shell as\n" " -E --setenv=VAR[=VALUE] Add an environment variable for shell\n" - " --read-only Create read-only bind mount\n" + " --read-only Create read-only bind mount or clone\n" " --mkdir Create directory before bind mounting, if missing\n" " -n --lines=INTEGER Number of journal entries to show\n" " --max-addresses=INTEGER Number of internet addresses to show at most\n" @@ -2670,9 +2080,7 @@ static int help(int argc, char *argv[], void *userdata) { " short-monotonic, short-unix, short-delta,\n" " json, json-pretty, json-sse, json-seq, cat,\n" " verbose, export, with-unit)\n" - " --verify=MODE Verification mode for downloaded images (no,\n" - " checksum, signature)\n" - " --force Download image even if already exists\n" + " --force Replace target file when copying, if necessary\n" " --now Start or power off container after enabling or\n" " disabling it\n" " --runner=RUNNER Select between nspawn and vmspawn as the runner\n" @@ -3012,15 +2420,6 @@ static int machinectl_main(int argc, char *argv[], sd_bus *bus) { { "start", 2, VERB_ANY, 0, start_machine }, { "enable", 2, VERB_ANY, 0, enable_machine }, { "disable", 2, VERB_ANY, 0, enable_machine }, - { "import-tar", 2, 3, 0, import_tar }, - { "import-raw", 2, 3, 0, import_raw }, - { "import-fs", 2, 3, 0, import_fs }, - { "export-tar", 2, 3, 0, export_tar }, - { "export-raw", 2, 3, 0, export_raw }, - { "pull-tar", 2, 3, 0, pull_tar }, - { "pull-raw", 2, 3, 0, pull_raw }, - { "list-transfers", VERB_ANY, 1, 0, list_transfers }, - { "cancel-transfer", 2, VERB_ANY, 0, cancel_transfer }, { "set-limit", 2, 3, 0, set_limit }, { "clean", VERB_ANY, 1, 0, clean_images }, {} @@ -3038,13 +2437,19 @@ static int run(int argc, char *argv[]) { /* The journal merging logic potentially needs a lot of fds. */ (void) rlimit_nofile_bump(HIGH_RLIMIT_NOFILE); - sigbus_install(); r = parse_argv(argc, argv); if (r <= 0) return r; + if (STRPTR_IN_SET(argv[optind], + "import-tar", "import-raw", "import-fs", + "export-tar", "export-raw", + "pull-tar", "pull-raw", + "list-transfers", "cancel-transfer")) + return chainload_importctl(argc, argv); + r = bus_connect_transport(arg_transport, arg_host, RUNTIME_SCOPE_SYSTEM, &bus); if (r < 0) return bus_log_connect_error(r, arg_transport); From 71cb203a6ea6de409c6b9c042feda2ff655101e9 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Mon, 26 Feb 2024 15:46:50 +0100 Subject: [PATCH 19/26] importctl: draw a pretty progress bar while downloading Everybody loves pretty terminal progress bar. --- src/basic/glyph-util.c | 4 ++ src/basic/glyph-util.h | 2 + src/import/importctl.c | 91 ++++++++++++++++++++++++++++++------ src/import/importd.c | 33 ++++++++++++- src/shared/pretty-print.c | 72 ++++++++++++++++++++++++++++ src/shared/pretty-print.h | 3 ++ src/test/meson.build | 4 ++ src/test/test-locale-util.c | 2 + src/test/test-progress-bar.c | 34 ++++++++++++++ 9 files changed, 228 insertions(+), 17 deletions(-) create mode 100644 src/test/test-progress-bar.c diff --git a/src/basic/glyph-util.c b/src/basic/glyph-util.c index b6b0f40ca6..d37be3234a 100644 --- a/src/basic/glyph-util.c +++ b/src/basic/glyph-util.c @@ -41,6 +41,8 @@ const char *special_glyph_full(SpecialGlyph code, bool force_utf) { [SPECIAL_GLYPH_TREE_SPACE] = " ", [SPECIAL_GLYPH_TREE_TOP] = ",-", [SPECIAL_GLYPH_VERTICAL_DOTTED] = ":", + [SPECIAL_GLYPH_HORIZONTAL_DOTTED] = "-", + [SPECIAL_GLYPH_HORIZONTAL_FAT] = "=", [SPECIAL_GLYPH_TRIANGULAR_BULLET] = ">", [SPECIAL_GLYPH_BLACK_CIRCLE] = "*", [SPECIAL_GLYPH_WHITE_CIRCLE] = "*", @@ -91,6 +93,8 @@ const char *special_glyph_full(SpecialGlyph code, bool force_utf) { /* Single glyphs in both cases */ [SPECIAL_GLYPH_VERTICAL_DOTTED] = u8"┆", + [SPECIAL_GLYPH_HORIZONTAL_DOTTED] = u8"┄", + [SPECIAL_GLYPH_HORIZONTAL_FAT] = u8"━", [SPECIAL_GLYPH_TRIANGULAR_BULLET] = u8"‣", [SPECIAL_GLYPH_BLACK_CIRCLE] = u8"●", [SPECIAL_GLYPH_WHITE_CIRCLE] = u8"○", diff --git a/src/basic/glyph-util.h b/src/basic/glyph-util.h index 2f70b187fc..db8dbbff26 100644 --- a/src/basic/glyph-util.h +++ b/src/basic/glyph-util.h @@ -13,6 +13,8 @@ typedef enum SpecialGlyph { SPECIAL_GLYPH_TREE_SPACE, SPECIAL_GLYPH_TREE_TOP, SPECIAL_GLYPH_VERTICAL_DOTTED, + SPECIAL_GLYPH_HORIZONTAL_DOTTED, + SPECIAL_GLYPH_HORIZONTAL_FAT, SPECIAL_GLYPH_TRIANGULAR_BULLET, SPECIAL_GLYPH_BLACK_CIRCLE, SPECIAL_GLYPH_WHITE_CIRCLE, diff --git a/src/import/importctl.c b/src/import/importctl.c index 688e583d07..7ada0e51df 100644 --- a/src/import/importctl.c +++ b/src/import/importctl.c @@ -45,6 +45,8 @@ static const char* arg_format = NULL; static JsonFormatFlags arg_json_format_flags = JSON_FORMAT_OFF; static ImageClass arg_image_class = _IMAGE_CLASS_INVALID; +#define PROGRESS_PREFIX "Total: " + static int settle_image_class(void) { if (arg_image_class < 0) { @@ -68,13 +70,21 @@ static int settle_image_class(void) { return 0; } +typedef struct Context { + const char *object_path; + double progress; +} Context; + static int match_log_message(sd_bus_message *m, void *userdata, sd_bus_error *error) { - const char **our_path = userdata, *line; + Context *c = ASSERT_PTR(userdata); + const char *line; unsigned priority; int r; assert(m); - assert(our_path); + + if (!streq_ptr(c->object_path, sd_bus_message_get_path(m))) + return 0; r = sd_bus_message_read(m, "us", &priority, &line); if (r < 0) { @@ -82,23 +92,51 @@ static int match_log_message(sd_bus_message *m, void *userdata, sd_bus_error *er return 0; } - if (!streq_ptr(*our_path, sd_bus_message_get_path(m))) - return 0; - if (arg_quiet && LOG_PRI(priority) >= LOG_INFO) return 0; + if (!arg_quiet) + clear_progress_bar(PROGRESS_PREFIX); + log_full(priority, "%s", line); + + if (!arg_quiet) + draw_progress_bar(PROGRESS_PREFIX, c->progress * 100); + + return 0; +} + +static int match_progress_update(sd_bus_message *m, void *userdata, sd_bus_error *error) { + Context *c = ASSERT_PTR(userdata); + int r; + + assert(m); + + if (!streq_ptr(c->object_path, sd_bus_message_get_path(m))) + return 0; + + r = sd_bus_message_read(m, "d", &c->progress); + if (r < 0) { + bus_log_parse_error(r); + return 0; + } + + if (!arg_quiet) + draw_progress_bar(PROGRESS_PREFIX, c->progress * 100); + return 0; } static int match_transfer_removed(sd_bus_message *m, void *userdata, sd_bus_error *error) { - const char **our_path = userdata, *path, *result; + Context *c = ASSERT_PTR(userdata); + const char *path, *result; uint32_t id; int r; assert(m); - assert(our_path); + + if (!arg_quiet) + clear_progress_bar(PROGRESS_PREFIX); r = sd_bus_message_read(m, "uos", &id, &path, &result); if (r < 0) { @@ -106,7 +144,7 @@ static int match_transfer_removed(sd_bus_message *m, void *userdata, sd_bus_erro return 0; } - if (!streq_ptr(*our_path, path)) + if (!streq_ptr(c->object_path, path)) return 0; sd_event_exit(sd_bus_get_event(sd_bus_message_get_bus(m)), !streq_ptr(result, "done")); @@ -117,6 +155,9 @@ static int transfer_signal_handler(sd_event_source *s, const struct signalfd_sig assert(s); assert(si); + if (!arg_quiet) + clear_progress_bar(PROGRESS_PREFIX); + if (!arg_quiet) log_info("Continuing download in the background. Use \"%s cancel-transfer %" PRIu32 "\" to abort transfer.", program_invocation_short_name, @@ -127,11 +168,11 @@ static int transfer_signal_handler(sd_event_source *s, const struct signalfd_sig } static int transfer_image_common(sd_bus *bus, sd_bus_message *m) { - _cleanup_(sd_bus_slot_unrefp) sd_bus_slot *slot_job_removed = NULL, *slot_log_message = NULL; + _cleanup_(sd_bus_slot_unrefp) sd_bus_slot *slot_job_removed = NULL, *slot_log_message = NULL, *slot_progress_update = NULL; _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL; _cleanup_(sd_bus_message_unrefp) sd_bus_message *reply = NULL; _cleanup_(sd_event_unrefp) sd_event* event = NULL; - const char *path = NULL; + Context c = {}; uint32_t id; int r; @@ -153,7 +194,9 @@ static int transfer_image_common(sd_bus *bus, sd_bus_message *m) { &slot_job_removed, bus_import_mgr, "TransferRemoved", - match_transfer_removed, NULL, &path); + match_transfer_removed, + /* add_callback= */ NULL, + &c); if (r < 0) return log_error_errno(r, "Failed to request match: %m"); @@ -161,10 +204,25 @@ static int transfer_image_common(sd_bus *bus, sd_bus_message *m) { bus, &slot_log_message, "org.freedesktop.import1", - NULL, + /* object_path= */ NULL, "org.freedesktop.import1.Transfer", "LogMessage", - match_log_message, NULL, &path); + match_log_message, + /* add_callback= */ NULL, + &c); + if (r < 0) + return log_error_errno(r, "Failed to request match: %m"); + + r = sd_bus_match_signal_async( + bus, + &slot_progress_update, + "org.freedesktop.import1", + /* object_path= */ NULL, + "org.freedesktop.import1.Transfer", + "ProgressUpdate", + match_progress_update, + /* add_callback= */ NULL, + &c); if (r < 0) return log_error_errno(r, "Failed to request match: %m"); @@ -172,12 +230,15 @@ static int transfer_image_common(sd_bus *bus, sd_bus_message *m) { if (r < 0) return log_error_errno(r, "Failed to transfer image: %s", bus_error_message(&error, r)); - r = sd_bus_message_read(reply, "uo", &id, &path); + r = sd_bus_message_read(reply, "uo", &id, &c.object_path); if (r < 0) return bus_log_parse_error(r); - if (!arg_quiet) + if (!arg_quiet) { + clear_progress_bar(PROGRESS_PREFIX); log_info("Enqueued transfer job %u. Press C-c to continue download in background.", id); + draw_progress_bar(PROGRESS_PREFIX, c.progress); + } (void) sd_event_add_signal(event, NULL, SIGINT|SD_EVENT_SIGNAL_PROCMASK, transfer_signal_handler, UINT32_TO_PTR(id)); (void) sd_event_add_signal(event, NULL, SIGTERM|SD_EVENT_SIGNAL_PROCMASK, transfer_signal_handler, UINT32_TO_PTR(id)); diff --git a/src/import/importd.c b/src/import/importd.c index 5aeb0b1895..493e1991b2 100644 --- a/src/import/importd.c +++ b/src/import/importd.c @@ -83,6 +83,7 @@ struct Transfer { unsigned n_canceled; unsigned progress_percent; + unsigned progress_percent_sent; int stdin_fd; int stdout_fd; @@ -166,7 +167,8 @@ static int transfer_new(Manager *m, Transfer **ret) { .stdin_fd = -EBADF, .stdout_fd = -EBADF, .verify = _IMPORT_VERIFY_INVALID, - .progress_percent= UINT_MAX, + .progress_percent = UINT_MAX, + .progress_percent_sent = UINT_MAX, }; id = m->current_transfer_id + 1; @@ -217,7 +219,28 @@ static void transfer_send_log_line(Transfer *t, const char *line) { line); if (r < 0) log_warning_errno(r, "Cannot emit log message signal, ignoring: %m"); - } +} + +static void transfer_send_progress_update(Transfer *t) { + int r; + + assert(t); + + if (t->progress_percent_sent == t->progress_percent) + return; + + r = sd_bus_emit_signal( + t->manager->bus, + t->object_path, + "org.freedesktop.import1.Transfer", + "ProgressUpdate", + "d", + transfer_percent_as_double(t)); + if (r < 0) + log_warning_errno(r, "Cannot emit progress update signal, ignoring: %m"); + + t->progress_percent_sent = t->progress_percent; +} static void transfer_send_logs(Transfer *t, bool flush) { assert(t); @@ -635,6 +658,8 @@ static int manager_on_notify(sd_event_source *s, int fd, uint32_t revents, void t->progress_percent = (unsigned) r; log_debug("Got percentage from client: %u%%", t->progress_percent); + + transfer_send_progress_update(t); return 0; } @@ -1369,6 +1394,10 @@ static const sd_bus_vtable transfer_vtable[] = { SD_BUS_PARAM(priority) SD_BUS_PARAM(line), 0), + SD_BUS_SIGNAL_WITH_NAMES("ProgressUpdate", + "d", + SD_BUS_PARAM(progress), + 0), SD_BUS_VTABLE_END, }; diff --git a/src/shared/pretty-print.c b/src/shared/pretty-print.c index a4e5809446..543a9ebdcb 100644 --- a/src/shared/pretty-print.c +++ b/src/shared/pretty-print.c @@ -462,3 +462,75 @@ int terminal_tint_color(double hue, char **ret) { return 0; } + +void draw_progress_bar(const char *prefix, double percentage) { + + fputs("\r", stderr); + if (prefix) + fputs(prefix, stderr); + + if (!terminal_is_dumb()) { + size_t cols = columns(); + size_t prefix_length = strlen_ptr(prefix); + size_t length = cols > prefix_length + 6 ? cols - prefix_length - 6 : 0; + + fputs(ansi_highlight_green(), stderr); + + if (length > 5 && percentage >= 0.0 && percentage <= 100.0) { + size_t p = (size_t) (length * percentage / 100.0); + bool separator_done = false; + + for (size_t i = 0; i < length; i++) { + + if (i <= p) { + if (get_color_mode() == COLOR_24BIT) { + uint8_t r8, g8, b8; + double z = i == 0 ? 0 : (((double) i / p) * 100); + hsv_to_rgb(145 /* green */, z, 33 + z*2/3, &r8, &g8, &b8); + fprintf(stderr, "\x1B[38;2;%u;%u;%um", r8, g8, b8); + } + + fputs(special_glyph(SPECIAL_GLYPH_HORIZONTAL_FAT), stderr); + } else if (i+1 < length && !separator_done) { + fputs(ansi_normal(), stderr); + fputc(' ', stderr); + separator_done = true; + fputs(ansi_grey(), stderr); + } else + fputs(special_glyph(SPECIAL_GLYPH_HORIZONTAL_DOTTED), stderr); + } + + fputs(ansi_normal(), stderr); + fputc(' ', stderr); + } + } + + fprintf(stderr, + "%s%3.0f%%%s", + ansi_highlight(), + percentage, + ansi_normal()); + + if (!terminal_is_dumb()) + fputs(ANSI_ERASE_TO_END_OF_LINE, stderr); + + fputc('\r', stderr); + fflush(stderr); +} + +void clear_progress_bar(const char *prefix) { + + fputc('\r', stderr); + + if (terminal_is_dumb()) { + size_t l = strlen_ptr(prefix); + for (size_t i = 0; i < l; i ++) + fputc(' ', stderr); + + fputs(" ", stderr); + } else + fputs(ANSI_ERASE_TO_END_OF_LINE, stderr); + + fputc('\r', stderr); + fflush(stderr); +} diff --git a/src/shared/pretty-print.h b/src/shared/pretty-print.h index 6069318be7..940d34775b 100644 --- a/src/shared/pretty-print.h +++ b/src/shared/pretty-print.h @@ -49,3 +49,6 @@ static inline const char *green_check_mark_internal(char buffer[static GREEN_CHE #define COLOR_MARK_BOOL(b) ((b) ? GREEN_CHECK_MARK() : RED_CROSS_MARK()) int terminal_tint_color(double hue, char **ret); + +void draw_progress_bar(const char *prefix, double percentage); +void clear_progress_bar(const char *prefix); diff --git a/src/test/meson.build b/src/test/meson.build index c628eaa7db..a0f3882480 100644 --- a/src/test/meson.build +++ b/src/test/meson.build @@ -380,6 +380,10 @@ executables += [ 'sources' : files('test-process-util.c'), 'dependencies' : threads, }, + test_template + { + 'sources' : files('test-progress-bar.c'), + 'type' : 'manual', + }, test_template + { 'sources' : files('test-qrcode-util.c'), 'dependencies' : libdl, diff --git a/src/test/test-locale-util.c b/src/test/test-locale-util.c index 67d9c7e65c..ab2d1f5746 100644 --- a/src/test/test-locale-util.c +++ b/src/test/test-locale-util.c @@ -92,6 +92,8 @@ TEST(dump_special_glyphs) { dump_glyph(SPECIAL_GLYPH_TREE_SPACE); dump_glyph(SPECIAL_GLYPH_TREE_TOP); dump_glyph(SPECIAL_GLYPH_VERTICAL_DOTTED); + dump_glyph(SPECIAL_GLYPH_HORIZONTAL_DOTTED); + dump_glyph(SPECIAL_GLYPH_HORIZONTAL_FAT); dump_glyph(SPECIAL_GLYPH_TRIANGULAR_BULLET); dump_glyph(SPECIAL_GLYPH_BLACK_CIRCLE); dump_glyph(SPECIAL_GLYPH_WHITE_CIRCLE); diff --git a/src/test/test-progress-bar.c b/src/test/test-progress-bar.c new file mode 100644 index 0000000000..b47adf0c28 --- /dev/null +++ b/src/test/test-progress-bar.c @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include "pretty-print.h" +#include "random-util.h" +#include "tests.h" + +#define PROGRESS_PREFIX "test: " + +TEST(progress_bar) { + + draw_progress_bar(PROGRESS_PREFIX, 0); + + bool paused = false; + + for (double d = 0; d <= 100; d += 0.5) { + usleep_safe(random_u64_range(20 * USEC_PER_MSEC)); + draw_progress_bar(PROGRESS_PREFIX, d); + + if (!paused && d >= 50) { + clear_progress_bar(PROGRESS_PREFIX); + fputs("Sleeping for 1s...", stdout); + fflush(stdout); + usleep_safe(USEC_PER_SEC); + paused = true; + } + } + + draw_progress_bar(PROGRESS_PREFIX, 100); + usleep_safe(300 * MSEC_PER_SEC); + clear_progress_bar(PROGRESS_PREFIX); + fputs("Done.\n", stdout); +} + +DEFINE_TEST_MAIN(LOG_INFO); From d304686ca39603016044bf8fe839c54007e6756e Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Mon, 26 Feb 2024 15:47:40 +0100 Subject: [PATCH 20/26] import: downgrade HTTP error code log message levels Let's downgrade log levels a bit on HTTP error codes. After all we gracefully handle many of them, and we do generated an extra message for the ones which are fatal anyway, hence there's no point in emphasizing the HTTP erro message levels as we currently do. --- src/import/pull-job.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/import/pull-job.c b/src/import/pull-job.c index bed7e64030..a4feda4081 100644 --- a/src/import/pull-job.c +++ b/src/import/pull-job.c @@ -187,7 +187,7 @@ void pull_job_curl_on_finished(CurlGlue *g, CURL *curl, CURLcode result) { } } - r = log_error_errno( + r = log_notice_errno( status == 404 ? SYNTHETIC_ERRNO(ENOMEDIUM) : SYNTHETIC_ERRNO(EIO), /* Make the most common error recognizable */ "HTTP request to %s failed with code %li.", j->url, status); goto finish; From ed459ec0c0c997734447985532c5683b50fc46bd Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Mon, 26 Feb 2024 15:53:35 +0100 Subject: [PATCH 21/26] import: mention explicitly which image directory we operate on Also, let's move the similar message about sync() mode to more common code. --- src/import/import-fs.c | 2 ++ src/import/import.c | 12 ++++++------ src/import/pull.c | 12 ++++++------ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/import/import-fs.c b/src/import/import-fs.c index b1619ef9ea..44fc5be8a5 100644 --- a/src/import/import-fs.c +++ b/src/import/import-fs.c @@ -171,6 +171,8 @@ static int import_fs(int argc, char *argv[], void *userdata) { log_info("Importing '%s', saving as '%s'.", strempty(pretty), local); } + log_info("Operating on image directory '%s'.", arg_image_root); + if (!arg_sync) log_info("File system synchronization on completion is off."); diff --git a/src/import/import.c b/src/import/import.c index 37dd0b2078..889cd63ff0 100644 --- a/src/import/import.c +++ b/src/import/import.c @@ -114,6 +114,12 @@ static int open_source(const char *path, const char *local, int *ret_open_fd) { log_info("Importing '%s', saving as '%s'.", strempty(pretty), local); } + if (!FLAGS_SET(arg_import_flags, IMPORT_DIRECT)) + log_info("Operating on image directory '%s'.", arg_image_root); + + if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC)) + log_info("File system synchronization on completion is off."); + *ret_open_fd = TAKE_FD(open_fd); return retval; } @@ -167,9 +173,6 @@ static int import_tar(int argc, char *argv[], void *userdata) { if (r < 0) return r; - if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC)) - log_info("File system synchronization on completion is off."); - r = tar_import_new(&import, event, arg_image_root, on_tar_finished, event); if (r < 0) return log_error_errno(r, "Failed to allocate importer: %m"); @@ -239,9 +242,6 @@ static int import_raw(int argc, char *argv[], void *userdata) { if (r < 0) return r; - if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC)) - log_info("File system synchronization on completion is off."); - r = raw_import_new(&import, event, arg_image_root, on_raw_finished, event); if (r < 0) return log_error_errno(r, "Failed to allocate importer: %m"); diff --git a/src/import/pull.c b/src/import/pull.c index d4ad0b2ea3..7c838a5307 100644 --- a/src/import/pull.c +++ b/src/import/pull.c @@ -90,6 +90,12 @@ static int normalize_local(const char *local, const char *url, char **ret) { } else log_info("Pulling '%s'.", url); + if (!FLAGS_SET(arg_import_flags, IMPORT_DIRECT)) + log_info("Operating on image directory '%s'.", arg_image_root); + + if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC)) + log_info("File system synchronization on completion is off."); + *ret = TAKE_PTR(ll); return 0; } @@ -142,9 +148,6 @@ static int pull_tar(int argc, char *argv[], void *userdata) { if (r < 0) return r; - if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC)) - log_info("File system synchronization on completion is off."); - r = tar_pull_new(&pull, event, arg_image_root, on_tar_finished, event); if (r < 0) return log_error_errno(r, "Failed to allocate puller: %m"); @@ -212,9 +215,6 @@ static int pull_raw(int argc, char *argv[], void *userdata) { if (r < 0) return r; - if (!FLAGS_SET(arg_import_flags, IMPORT_SYNC)) - log_info("File system synchronization on completion is off."); - r = raw_pull_new(&pull, event, arg_image_root, on_raw_finished, event); if (r < 0) return log_error_errno(r, "Failed to allocate puller: %m"); From 5a985dd05a00aba3b65b851deeebb330abefb955 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Mon, 26 Feb 2024 19:04:48 +0100 Subject: [PATCH 22/26] importd: pass log level to invoked child --- src/import/importd.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/import/importd.c b/src/import/importd.c index 493e1991b2..36d327bcf6 100644 --- a/src/import/importd.c +++ b/src/import/importd.c @@ -429,6 +429,10 @@ static int transfer_start(Transfer *t) { _exit(EXIT_FAILURE); } + r = setenv_systemd_log_level(); + if (r < 0) + log_warning_errno(r, "Failed to update $SYSTEMD_LOG_LEVEL, ignoring: %m"); + r = setenv_systemd_exec_pid(true); if (r < 0) log_warning_errno(r, "Failed to update $SYSTEMD_EXEC_PID, ignoring: %m"); From 5b7bfe0637a5550b2b81d4f0e716b9cb6f989c5d Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Tue, 27 Feb 2024 11:08:49 +0100 Subject: [PATCH 23/26] importd: add command to list downloaded images It's a bit weird we allow importing/pulling/exporting images, but we have no scheme for showing what#s already downloaded. Hence let's add this, it's easy to add after all. --- src/import/importctl.c | 94 +++++++++++++++++++++++++ src/import/importd.c | 88 +++++++++++++++++++++++ src/import/org.freedesktop.import1.conf | 4 ++ 3 files changed, 186 insertions(+) diff --git a/src/import/importctl.c b/src/import/importctl.c index 7ada0e51df..5f6f926135 100644 --- a/src/import/importctl.c +++ b/src/import/importctl.c @@ -884,6 +884,98 @@ static int cancel_transfer(int argc, char *argv[], void *userdata) { return 0; } +static int list_images(int argc, char *argv[], void *userdata) { + _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL; + _cleanup_(sd_bus_message_unrefp) sd_bus_message *reply = NULL; + _cleanup_(table_unrefp) Table *t = NULL; + sd_bus *bus = ASSERT_PTR(userdata); + int r; + + pager_open(arg_pager_flags); + + r = bus_call_method(bus, bus_import_mgr, "ListImages", &error, &reply, "st", image_class_to_string(arg_image_class), UINT64_C(0)); + if (r < 0) + return log_error_errno(r, "Could not list images: %s", bus_error_message(&error, r)); + + r = sd_bus_message_enter_container(reply, 'a', "(ssssbtttttt)"); + if (r < 0) + return bus_log_parse_error(r); + + t = table_new("class", "name", "type", "path", "ro", "crtime", "mtime", "usage", "usage-exclusive", "limit", "limit-exclusive"); + if (!t) + return log_oom(); + + (void) table_set_sort(t, (size_t) 0, (size_t) 1); + table_set_ersatz_string(t, TABLE_ERSATZ_DASH); + + /* Hide the exclusive columns for now */ + (void) table_hide_column_from_display(t, 8); + (void) table_hide_column_from_display(t, 10); + + for (;;) { + uint64_t crtime, mtime, usage, usage_exclusive, limit, limit_exclusive; + const char *class, *name, *type, *path; + int read_only; + + r = sd_bus_message_read(reply, "(ssssbtttttt)", &class, &name, &type, &path, &read_only, &crtime, &mtime, &usage, &usage_exclusive, &limit, &limit_exclusive); + if (r < 0) + return bus_log_parse_error(r); + if (r == 0) + break; + + r = table_add_many( + t, + TABLE_STRING, class, + TABLE_STRING, name, + TABLE_STRING, type, + TABLE_PATH, path); + if (r < 0) + return table_log_add_error(r); + + if (FLAGS_SET(arg_json_format_flags, JSON_FORMAT_OFF)) + r = table_add_many( + t, + TABLE_STRING, read_only ? "ro" : "rw", + TABLE_SET_COLOR, read_only ? ANSI_HIGHLIGHT_RED : ANSI_HIGHLIGHT_GREEN); + else + r = table_add_many( + t, + TABLE_BOOLEAN, read_only); + if (r < 0) + return table_log_add_error(r); + + r = table_add_many( + t, + TABLE_TIMESTAMP, crtime, + TABLE_TIMESTAMP, mtime, + TABLE_SIZE, usage, + TABLE_SIZE, usage_exclusive, + TABLE_SIZE, limit, + TABLE_SIZE, limit_exclusive); + if (r < 0) + return table_log_add_error(r); + } + + r = sd_bus_message_exit_container(reply); + if (r < 0) + return bus_log_parse_error(r); + + if (!table_isempty(t)) { + r = table_print_with_pager(t, arg_json_format_flags, arg_pager_flags, arg_legend); + if (r < 0) + return log_error_errno(r, "Failed to output table: %m"); + } + + if (arg_legend) { + if (!table_isempty(t)) + printf("\n%zu images listed.\n", table_get_rows(t) - 1); + else + printf("No images.\n"); + } + + return 0; +} + static int help(int argc, char *argv[], void *userdata) { _cleanup_free_ char *link = NULL; int r; @@ -906,6 +998,7 @@ static int help(int argc, char *argv[], void *userdata) { " export-raw NAME [FILE] Export a RAW container or VM image locally\n" " list-transfers Show list of transfers in progress\n" " cancel-transfer [ID...] Cancel a transfer\n" + " list-images Show list of installed images\n" "\n%3$sOptions:%4$s\n" " -h --help Show this help\n" " --version Show package version\n" @@ -1122,6 +1215,7 @@ static int importctl_main(int argc, char *argv[], sd_bus *bus) { { "pull-raw", 2, 3, 0, pull_raw }, { "list-transfers", VERB_ANY, 1, VERB_DEFAULT, list_transfers }, { "cancel-transfer", 2, VERB_ANY, 0, cancel_transfer }, + { "list-images", VERB_ANY, 1, 0, list_images }, {} }; diff --git a/src/import/importd.c b/src/import/importd.c index 36d327bcf6..4bba46984a 100644 --- a/src/import/importd.c +++ b/src/import/importd.c @@ -1296,6 +1296,86 @@ static int method_cancel_transfer(sd_bus_message *msg, void *userdata, sd_bus_er return sd_bus_reply_method_return(msg, NULL); } +static int method_list_images(sd_bus_message *msg, void *userdata, sd_bus_error *error) { + _cleanup_(sd_bus_message_unrefp) sd_bus_message *reply = NULL; + ImageClass class = _IMAGE_CLASS_INVALID; + int r; + + assert(msg); + + const char *sclass; + uint64_t flags; + + r = sd_bus_message_read(msg, "st", &sclass, &flags); + if (r < 0) + return r; + + if (!isempty(sclass)) { + class = image_class_from_string(sclass); + if (class < 0) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Image class '%s' not known", sclass); + } + + if (flags != 0) + return sd_bus_error_setf(error, SD_BUS_ERROR_INVALID_ARGS, + "Flags 0x%" PRIx64 " invalid", flags); + + r = sd_bus_message_new_method_return(msg, &reply); + if (r < 0) + return r; + + r = sd_bus_message_open_container(reply, 'a', "(ssssbtttttt)"); + if (r < 0) + return r; + + for (ImageClass c = class < 0 ? 0 : class; + class < 0 ? (c < _IMAGE_CLASS_MAX) : (c == class); + c++) { + + _cleanup_(hashmap_freep) Hashmap *h = NULL; + + h = hashmap_new(&image_hash_ops); + if (!h) + return -ENOMEM; + + r = image_discover(c, /* root= */ NULL, h); + if (r < 0) { + if (class >= 0) + return r; + + log_warning_errno(r, "Failed to discover images of type %s: %m", image_class_to_string(c)); + continue; + } + + Image *i; + HASHMAP_FOREACH(i, h) { + r = sd_bus_message_append( + reply, + "(ssssbtttttt)", + image_class_to_string(i->class), + i->name, + image_type_to_string(i->type), + i->path, + i->read_only, + i->crtime, + i->mtime, + i->usage, + i->usage_exclusive, + i->limit, + i->limit_exclusive); + if (r < 0) + return r; + } + } + + r = sd_bus_message_close_container(reply); + if (r < 0) + return r; + + return sd_bus_send(NULL, reply, NULL); +} + static int property_get_progress( sd_bus *bus, const char *path, @@ -1592,6 +1672,14 @@ static const sd_bus_vtable manager_vtable[] = { NULL,, method_cancel_transfer, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD_WITH_NAMES("ListImages", + "st", + SD_BUS_PARAM(class) + SD_BUS_PARAM(flags), + "a(ssssbtttttt)", + SD_BUS_PARAM(images), + method_list_images, + SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_SIGNAL_WITH_NAMES("TransferNew", "uo", diff --git a/src/import/org.freedesktop.import1.conf b/src/import/org.freedesktop.import1.conf index f775aa061c..f99ec56c79 100644 --- a/src/import/org.freedesktop.import1.conf +++ b/src/import/org.freedesktop.import1.conf @@ -110,6 +110,10 @@ send_interface="org.freedesktop.import1.Transfer" send_member="Cancel"/> + + From fd571c9df0ba86636401582c4ea0a8cb608e1216 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Tue, 27 Feb 2024 09:39:57 +0100 Subject: [PATCH 24/26] man: document new importctl/importd functionality This also replaces the Fedora download example with another one from Ubuntu, since Fedora's images these days no longer qualify as DDIs, they have no distinctive partition type UUIDs set for multiple of their partitions, hence the images cannot be booted. A bit sad. Let's provide a command that just works in its place. --- man/importctl.xml | 442 ++++++++++++++++++++++++++++++++ man/machinectl.xml | 298 +-------------------- man/org.freedesktop.import1.xml | 267 ++++++++++++++----- man/portablectl.xml | 1 + man/rules/meson.build | 1 + man/systemd-importd.service.xml | 12 +- man/systemd-nspawn.xml | 15 +- man/systemd-sysext.xml | 1 + man/systemd-vmspawn.xml | 2 + 9 files changed, 665 insertions(+), 374 deletions(-) create mode 100644 man/importctl.xml diff --git a/man/importctl.xml b/man/importctl.xml new file mode 100644 index 0000000000..25c49fc5dc --- /dev/null +++ b/man/importctl.xml @@ -0,0 +1,442 @@ + + +%entities; +]> + + + + + + importctl + systemd + + + + importctl + 1 + + + + importctl + Download, import or export disk images + + + + + importctl + OPTIONS + COMMAND + NAME + + + + + Description + + importctl may be used to download, import export disk images via + systemd-importd.service8. + + importctl operates both on block-level disk images (such as DDIs) as well as + file-system-level images (tarballs). It supports disk images are one of the four following + classes: + + + VM images or full OS container images, that may be run via + systemd-vmspawn1 or + systemd-nspawn1, and + managed via + machinectl1. + + Portable service images, that may be attached an managed via + portablectl1. + + System extension (sysext) images, that may be activated via + systemd-sysext8. + + Configuration extension (confext) images, that may be activated via + systemd-confext8. + + + When images are downloaded or imported they are placed in the following directories, depending on + the parameter: + + + Classes and Directories + + + + + + Class + Directory + + + + + machine + /var/lib/machines/ + + + portable + /var/lib/portables/ + + + sysext + /var/lib/extensions/ + + + confext + /var/lib/confexts/ + + + +
+ + + + Commands + + The following commands are understood: + + + + pull-tar URL [NAME] + + Downloads a .tar image from the specified URL, and makes it + available under the specified local name in the image directory for the selected + . The URL must be of type http:// or + https://, and must refer to a .tar, + .tar.gz, .tar.xz or .tar.bz2 archive + file. If the local image name is omitted, it is automatically derived from the last component of the + URL, with its suffix removed. + + The image is verified before it is made available, unless is + specified. Verification is done either via an inline signed file with the name of the image and the + suffix .sha256 or via separate SHA256SUMS and + SHA256SUMS.gpg files. The signature files need to be made available on the same + web server, under the same URL as the .tar file. With + , only the SHA256 checksum for the file is verified, based on the + .sha256 suffixed file or the SHA256SUMS file. With + , the sha checksum file is first verified with the inline + signature in the .sha256 file or the detached GPG signature file + SHA256SUMS.gpg. The public key for this verification step needs to be available + in /usr/lib/systemd/import-pubring.gpg or + /etc/systemd/import-pubring.gpg. + + If is specified the image will be downloaded and stored in + a read-only subvolume/directory in the image directory that is named after the specified URL and its + HTTP etag. A writable snapshot is then taken from this subvolume, and named after the specified local + name. This behavior ensures that creating multiple instances of the same URL is efficient, as + multiple downloads are not necessary. In order to create only the read-only image, and avoid creating + its writable snapshot, specify - as local name. + + Note that pressing C-c during execution of this command will not abort the download. Use + cancel-transfer, described below. + + + + + + pull-raw URL [NAME] + + Downloads a .raw disk image from the specified URL, and makes it + available under the specified local name in the image directory for the selected + . The URL must be of type http:// or + https://. The image must either be a .qcow2 or raw disk + image, optionally compressed as .gz, .xz, or + .bz2. If the local name is omitted, it is automatically derived from the last + component of the URL, with its suffix removed. + + Image verification is identical for raw and tar images (see above). + + If the downloaded image is in .qcow2 format it is converted into a raw + image file before it is made available. + + If is specified the image will be downloaded and stored in + a read-only file in the image directory that is named after the specified URL and its HTTP etag. A + writable copy is then made from this file, and named after the specified local name. This behavior + ensures that creating multiple instances of the same URL is efficient, as multiple downloads are not + necessary. In order to create only the read-only image, and avoid creating its writable copy, + specify - as local name. + + Note that pressing C-c during execution of this command will not abort the download. Use + cancel-transfer, described below. + + + + + + import-tar FILE [NAME] + import-raw FILE [NAME] + + Imports a TAR or RAW image, and places it under the specified name in the image + directory for the image class selected via . When + import-tar is used, the file specified as the first argument should be a tar + archive, possibly compressed with xz, gzip or bzip2. It will then be unpacked into its own + subvolume/directory. When import-raw is used, the file should be a qcow2 or raw + disk image, possibly compressed with xz, gzip or bzip2. If the second argument (the resulting image + name) is not specified, it is automatically derived from the file name. If the filename is passed as + -, the image is read from standard input, in which case the second argument is + mandatory. + + No cryptographic validation is done when importing the images. + + Much like image downloads, ongoing imports may be listed with list + and aborted with cancel-transfer. + + + + + + import-fs DIRECTORY [NAME] + + Imports an image stored in a local directory into the image directory for the image + class selected via and operates similarly to import-tar + or import-raw, but the first argument is the source directory. If supported, this + command will create a btrfs snapshot or subvolume for the new image. + + + + + + export-tar NAME [FILE] + export-raw NAME [FILE] + + Exports a TAR or RAW image and stores it in the specified file. The first parameter + should be an image name. The second parameter should be a file path the TAR or RAW + image is written to. If the path ends in .gz, the file is compressed with gzip, if + it ends in .xz, with xz, and if it ends in .bz2, with bzip2. If + the path ends in neither, the file is left uncompressed. If the second argument is missing, the image + is written to standard output. The compression may also be explicitly selected with the + switch. This is in particular useful if the second parameter is left + unspecified. + + Much like image downloads and imports, ongoing exports may be listed with + list and aborted with cancel-transfer. + + Note that, currently, only directory and subvolume images may be exported as TAR images, and + only raw disk images as RAW images. + + + + + + list-transfer + + Shows a list of image downloads, imports and exports that are currently in + progress. + + + + + + cancel-transfer ID… + + Aborts a download, import or export of the image with the specified ID. To list + ongoing transfers and their IDs, use list. + + + + + + list-images + + Shows a list of already downloaded/imported images. + + + + + + + + + Options + + The following options are understood: + + + + + + + When used with pull-raw, pull-tar, + import-raw, import-tar or import-fs a + read-only image is created. + + + + + + + + + When downloading an image, specify whether the image shall be verified before it is + made available. Takes one of no, checksum and + signature. If no, no verification is done. If + checksum is specified, the download is checked for integrity after the transfer is + complete, but no signatures are verified. If signature is specified, the checksum + is verified and the image's signature is checked against a local keyring of trustable vendors. It is + strongly recommended to set this option to signature if the server and protocol + support this. Defaults to signature. + + + + + + + + When downloading an image, and a local copy by the specified local name already + exists, delete it first and replace it by the newly downloaded image. + + + + + + + + When used with the or + commands, specifies the compression format to use for the resulting file. Takes one of + uncompressed, xz, gzip, + bzip2. By default, the format is determined automatically from the output image + file name passed. + + + + + + + + + Suppresses additional informational output while running. + + + + + + + + + + + Connect to + systemd-import.service8 + running in a local container, to perform the specified operation within the container. + + + + + + + + + + + + Selects the image class for the downloaded images. This primarily selects the + directory to download into. The switch takes machine, + portable, sysext or confext as argument. The + short options , , , are + shortcuts for , , + , . + + Note that defaults to true for + and false otherwise, see below. + + + + + + + + + Takes a boolean argument. When specified with pull-raw or + pull-tar, selects whether to download directly into the specified local image + name, or whether to download into a read-only copy first of which to make a writable copy after the + download is completed. Defaults to true for , false otherwise. + + The switch is a shortcut for . + + + + + + + + + + + + + + + Examples + + Download an Ubuntu TAR image and open a shell in it + + # importctl pull-tar -mN https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64-root.tar.xz +# systemd-nspawn -M jammy-server-cloudimg-amd64-root + + This downloads and verifies the specified .tar image, and then uses + systemd-nspawn1 to + open a shell in it. + + + + Download an Ubuntu RAW image, set a root password in it, start + it as a service + + # importctl pull-raw -mN \ + https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64-disk-kvm.img \ + jammy +# systemd-firstboot --image=/var/lib/machines/jammy.raw --prompt-root-password --force +# machinectl start jammy +# machinectl login jammy + + This downloads the specified .raw image and makes it available under the + local name jammy. Then, a root password is set with + systemd-firstboot1. Afterwards + the machine is started as system service. With the last command a login prompt into the container is + requested. + + + + Exports a container image as tar file + + # importctl export-tar -m fedora myfedora.tar.xz + + Exports the container fedora as an xz-compressed tar file + myfedora.tar.xz into the current directory. + + + + + Exit status + + On success, 0 is returned, a non-zero failure code + otherwise. + + + + + + See Also + + systemd1 + systemd-importd.service8 + systemd-nspawn1 + systemd-vmspawn1 + machinectl1 + portablectl1 + systemd-sysext8 + systemd-confext8 + tar1 + xz1 + gzip1 + bzip21 + + + + diff --git a/man/machinectl.xml b/man/machinectl.xml index a4d5317543..319942207b 100644 --- a/man/machinectl.xml +++ b/man/machinectl.xml @@ -80,6 +80,9 @@ The file system tree of the host OS itself. + Images may be downloaded, imported and exported via the + importctl1 + tool. @@ -87,8 +90,9 @@ The following commands are understood: - Machine Commands + Machine Commands + list @@ -361,8 +365,9 @@ - Image Commands + Image Commands + list-images @@ -515,7 +520,7 @@ When combined with the switch removes all images, not just hidden ones. This command effectively empties /var/lib/machines/. - Note that commands such as machinectl pull-tar or machinectl + Note that commands such as importctl pull-tar or importctl pull-raw usually create hidden, read-only, unmodified machine images from the downloaded image first, before cloning a writable working copy of it, in order to avoid duplicate downloads in case of images that are reused multiple times. Use machinectl clean to remove old, hidden images created this @@ -525,197 +530,6 @@ - - Image Transfer Commands - - - pull-tar URL [NAME] - - Downloads a .tar - container image from the specified URL, and makes it available - under the specified local machine name. The URL must be of - type http:// or - https://, and must refer to a - .tar, .tar.gz, - .tar.xz or .tar.bz2 - archive file. If the local machine name is omitted, it - is automatically derived from the last component of the URL, - with its suffix removed. - - The image is verified before it is made available, unless - is specified. - Verification is done either via an inline signed file with the name - of the image and the suffix .sha256 or via - separate SHA256SUMS and - SHA256SUMS.gpg files. - The signature files need to be made available on the same web - server, under the same URL as the .tar file. - With , only the SHA256 checksum - for the file is verified, based on the .sha256 - suffixed file or the SHA256SUMS file. - With , the sha checksum file is - first verified with the inline signature in the - .sha256 file or the detached GPG signature file - SHA256SUMS.gpg. - The public key for this verification step needs to be available in - /usr/lib/systemd/import-pubring.gpg or - /etc/systemd/import-pubring.gpg. - - The container image will be downloaded and stored in a - read-only subvolume in - /var/lib/machines/ that is named after - the specified URL and its HTTP etag. A writable snapshot is - then taken from this subvolume, and named after the specified - local name. This behavior ensures that creating multiple - container instances of the same URL is efficient, as multiple - downloads are not necessary. In order to create only the - read-only image, and avoid creating its writable snapshot, - specify - as local machine name. - - Note that the read-only subvolume is prefixed with - .tar-, and is thus not shown by - list-images, unless - is passed. - - Note that pressing C-c during execution of this command - will not abort the download. Use - cancel-transfer, described - below. - - - - - - pull-raw URL [NAME] - - Downloads a .raw - container or VM disk image from the specified URL, and makes - it available under the specified local machine name. The URL - must be of type http:// or - https://. The container image must either - be a .qcow2 or raw disk image, optionally - compressed as .gz, - .xz, or .bz2. If the - local machine name is omitted, it is automatically - derived from the last component of the URL, with its suffix - removed. - - Image verification is identical for raw and tar images - (see above). - - If the downloaded image is in - .qcow2 format it is converted into a raw - image file before it is made available. - - Downloaded images of this type will be placed as - read-only .raw file in - /var/lib/machines/. A local, writable - (reflinked) copy is then made under the specified local - machine name. To omit creation of the local, writable copy - pass - as local machine name. - - Similarly to the behavior of pull-tar, the read-only image is prefixed with - .raw-, and thus not shown by list-images, unless - is passed. - - Note that pressing C-c during execution of this command - will not abort the download. Use - cancel-transfer, described - below. - - - - - - import-tar FILE [NAME] - import-raw FILE [NAME] - Imports a TAR or RAW container or VM image, - and places it under the specified name in - /var/lib/machines/. When - import-tar is used, the file specified as - the first argument should be a tar archive, possibly compressed - with xz, gzip or bzip2. It will then be unpacked into its own - subvolume in /var/lib/machines/. When - import-raw is used, the file should be a - qcow2 or raw disk image, possibly compressed with xz, gzip or - bzip2. If the second argument (the resulting image name) is - not specified, it is automatically derived from the file - name. If the filename is passed as -, the - image is read from standard input, in which case the second - argument is mandatory. - - Optionally, the switch may be used to create a read-only container or VM - image. No cryptographic validation is done when importing the images. - - Much like image downloads, ongoing imports may be listed - with list-transfers and aborted with - cancel-transfer. - - - - - - import-fs DIRECTORY [NAME] - - Imports a container image stored in a local directory into - /var/lib/machines/, operates similarly to import-tar or - import-raw, but the first argument is the source directory. If supported, this - command will create a btrfs snapshot or subvolume for the new image. - - - - - - export-tar NAME [FILE] - export-raw NAME [FILE] - Exports a TAR or RAW container or VM image and - stores it in the specified file. The first parameter should be - a VM or container image name. The second parameter should be a - file path the TAR or RAW image is written to. If the path ends - in .gz, the file is compressed with gzip, if - it ends in .xz, with xz, and if it ends in - .bz2, with bzip2. If the path ends in - neither, the file is left uncompressed. If the second argument - is missing, the image is written to standard output. The - compression may also be explicitly selected with the - switch. This is in particular - useful if the second parameter is left unspecified. - - Much like image downloads and imports, ongoing exports - may be listed with list-transfers and - aborted with - cancel-transfer. - - Note that, currently, only directory and subvolume images - may be exported as TAR images, and only raw disk images as RAW - images. - - - - - - list-transfers - - Shows a list of container or VM image - downloads, imports and exports that are currently in - progress. - - - - - - cancel-transfer ID… - - Aborts a download, import or export of the - container or VM image with the specified ID. To list ongoing - transfers and their IDs, use - list-transfers. - - - - - - @@ -834,8 +648,7 @@ When used with bind, creates a read-only bind mount. - When used with clone, import-raw or import-tar a - read-only container or VM image is created. + When used with clone a read-only container or VM image is created. @@ -866,27 +679,6 @@ - - - - When downloading a container or VM image, - specify whether the image shall be verified before it is made - available. Takes one of no, - checksum and signature. - If no, no verification is done. If - checksum is specified, the download is - checked for integrity after the transfer is complete, but no - signatures are verified. If signature is - specified, the checksum is verified and the image's signature - is checked against a local keyring of trustable vendors. It is - strongly recommended to set this option to - signature if the server and protocol - support this. Defaults to - signature. - - - - | @@ -926,28 +718,11 @@ - When downloading a container or VM image, and - a local copy by the specified local machine name already - exists, delete it first and replace it by the newly downloaded - image. + Replace target file when copying files. - - - - When used with the - or commands, specifies the - compression format to use for the resulting file. Takes one of - uncompressed, xz, - gzip, bzip2. By default, - the format is determined automatically from the image file - name passed. - - - - @@ -1069,58 +844,8 @@ Examples - - Download a Ubuntu image and open a shell in it - # machinectl pull-tar https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-root.tar.gz -# systemd-nspawn -M trusty-server-cloudimg-amd64-root - - This downloads and verifies the specified - .tar image, and then uses - systemd-nspawn1 - to open a shell in it. - - - - Download a Fedora image, set a root password in it, start - it as a service - - # machinectl pull-raw --verify=no \ - https://download.fedoraproject.org/pub/fedora/linux/releases/&fedora_latest_version;/Cloud/x86_64/images/Fedora-Cloud-Base-&fedora_latest_version;-&fedora_cloud_release;.x86_64.raw.xz \ - Fedora-Cloud-Base-&fedora_latest_version;-&fedora_cloud_release;.x86-64 -# systemd-nspawn -M Fedora-Cloud-Base-&fedora_latest_version;-&fedora_cloud_release;.x86-64 -# passwd -# exit -# machinectl start Fedora-Cloud-Base-&fedora_latest_version;-&fedora_cloud_release;.x86-64 -# machinectl login Fedora-Cloud-Base-&fedora_latest_version;-&fedora_cloud_release;.x86-64 - - This downloads the specified .raw - image with verification disabled. Then, a shell is opened in it - and a root password is set. Afterwards the shell is left, and - the machine started as system service. With the last command a - login prompt into the container is requested. - - - - Exports a container image as tar file - - # machinectl export-tar fedora myfedora.tar.xz - - Exports the container fedora as an - xz-compressed tar file myfedora.tar.xz into the - current directory. - - - - Create a new shell session - - # machinectl shell --uid=lennart - - This creates a new shell session on the local host for - the user ID lennart, in a su1-like - fashion. - + @@ -1140,6 +865,7 @@ systemd-machined.service8 systemd-nspawn1 systemd.special7 + importctl1 tar1 xz1 gzip1 diff --git a/man/org.freedesktop.import1.xml b/man/org.freedesktop.import1.xml index e230a69f4c..2486eea4b3 100644 --- a/man/org.freedesktop.import1.xml +++ b/man/org.freedesktop.import1.xml @@ -25,11 +25,11 @@ systemd-importd.service8 - is a system service which may be used to import, export and download additional system images. These - images can be used by tools such as - systemd-nspawn1 - to run local containers. The service is used as the backend for machinectl pull-raw, - machinectl pull-tar and related commands. This page describes the D-Bus interface. + is a system service which may be used to import, export and download disk images. These images can be + used by tools such as + systemd-nspawn1 to run + local containers. The service is used as the backend for importctl pull-raw, + importctl pull-tar and related commands. This page describes the D-Bus interface. Note that @@ -56,42 +56,94 @@ node /org/freedesktop/import1 { in b read_only, out u transfer_id, out o transfer_path); + ImportTarEx(in h fd, + in s local_name, + in s class, + in t flags, + out u transfer_id, + out o transfer_path); ImportRaw(in h fd, in s local_name, in b force, in b read_only, out u transfer_id, out o transfer_path); + ImportRawEx(in h fd, + in s local_name, + in s class, + in t flags, + out u transfer_id, + out o transfer_path); ImportFileSystem(in h fd, in s local_name, in b force, in b read_only, out u transfer_id, out o transfer_path); + ImportFileSystemEx(in h fd, + in s local_name, + in s class, + in t flags, + out u transfer_id, + out o transfer_path); ExportTar(in s local_name, in h fd, in s format, out u transfer_id, out o transfer_path); + ExportTarEx(in s local_name, + in s class, + in h fd, + in s format, + in t flags, + out u transfer_id, + out o transfer_path); ExportRaw(in s local_name, in h fd, in s format, out u transfer_id, out o transfer_path); + ExportRawEx(in s local_name, + in s class, + in h fd, + in s format, + in t flags, + out u transfer_id, + out o transfer_path); PullTar(in s url, in s local_name, in s verify_mode, in b force, out u transfer_id, out o transfer_path); + PullTarEx(in s url, + in s local_name, + in s class, + in s verify_mode, + in t flags, + out u transfer_id, + out o transfer_path); PullRaw(in s url, in s local_name, in s verify_mode, in b force, out u transfer_id, out o transfer_path); + PullRawEx(in s url, + in s local_name, + in s class, + in s verify_mode, + in t flags, + out u transfer_id, + out o transfer_path); ListTransfers(out a(usssdo) transfers); + ListTransfersEx(in s class, + in t flags, + out a(ussssdo) transfers); CancelTransfer(in u transfer_id); + ListImages(in s class, + in t flags, + out a(ssssbtttttt) images); signals: TransferNew(u transfer_id, o transfer_path); @@ -105,8 +157,6 @@ node /org/freedesktop/import1 { }; - - @@ -115,22 +165,40 @@ node /org/freedesktop/import1 { + + + + + + + + + + + + + + + + + + @@ -140,81 +208,114 @@ node /org/freedesktop/import1 { Methods - ImportTar() and ImportRaw() import a system image and - place it into /var/lib/machines/. The first argument should be a file descriptor - (opened for reading) referring to the tar or raw file to import. It should reference a file on disk, - a pipe or a socket. When ImportTar() is used the file descriptor should - refer to a tar file, optionally compressed with - gzip1, - bzip21, - or + ImportTar()/ImportTarEx() and + ImportRaw()/ImportRawEx() import a disk image and place it + into the image directory. The first argument should be a file descriptor (opened for reading) referring + to the tar or raw file to import. It should reference a file on disk, a pipe or a socket. When + ImportTar()/ImportTarEx() is used the file descriptor should + refer to a tar file, optionally compressed with gzip1, + bzip21, or xz1. systemd-importd will detect the used compression scheme (if any) automatically. When - ImportRaw() is used the file descriptor should refer to a raw or qcow2 disk image - containing an MBR or GPT disk label, also optionally compressed with gzip, bzip2 or xz. In either case, - if the file is specified as a file descriptor on disk, progress information is generated for the import - operation (as in that case we know the total size on disk). If a socket or pipe is specified, progress information is not - available. The file descriptor argument is followed by a local name for the image. This should be a - name suitable as a hostname and will be used to name the imported image below - /var/lib/machines/. A tar import is placed as a directory tree or a - btrfs8 - subvolume below /var/lib/machines/ under the specified name with no suffix - appended. A raw import is placed as a file in /var/lib/machines/ with the - .raw suffix appended. If the argument is true, any - pre-existing image with the same name is removed before starting the operation. Otherwise, the - operation fails if an image with the same name already exists. Finally, the - argument controls - whether to create a writable or read-only image. Both methods return immediately after starting the import, - with the import transfer ongoing. They return a pair of transfer identifier and object path, which may - be used to retrieve progress information about the transfer or to cancel it. The transfer identifier is a - simple numeric identifier, the object path references an + ImportRaw()/ImportRawEx() is used the file descriptor should + refer to a raw or qcow2 disk image containing an MBR or GPT disk label, also optionally compressed with + gzip, bzip2 or xz. In either case, if the file is specified as a file descriptor on disk, progress + information is generated for the import operation (as in that case we know the total size on disk). If + a socket or pipe is specified, progress information is not available. The file descriptor argument is + followed by a local name for the image. This should be a name suitable as a hostname and will be used + to name the imported image below /var/lib/machines/. A tar import is placed as a + directory tree or a btrfs8 + subvolume below the image directory under the specified name with no suffix appended. A raw import is + placed as a file in the image directory with the .raw suffix appended. In case of + ImportTar()/ImportRaw(), if the + argument is true, any pre-existing image with the same name is removed before starting the + operation. Otherwise, the operation fails if an image with the same name already exists. The + argument controls whether to create a writable or read-only image. In case + of ImportTarEx()/ImportRawEx() these boolean flags are + provided via a 64bit flags parameter instead, with bit 0 mapping to the + parameter, and bit 1 mapping to . The parameter + specifies the image class, and takes one of machine, portable, + sysext, confext. All four methods return immediately after + starting the import, with the import transfer ongoing. They return a pair of transfer identifier and + object path, which may be used to retrieve progress information about the transfer or to cancel it. The + transfer identifier is a simple numeric identifier, the object path references an org.freedesktop.import1.Transfer object, see below. Listen for a TransferRemoved() signal for the transfer ID in order to detect when a transfer is complete. The returned transfer object is useful to determine the current progress or log output of the ongoing import operation. - ExportTar() and ExportRaw() implement the reverse - operation, and may be used to export a system image in order to place it in a tar or raw image. They - take the machine name to export as their first parameter, followed by a file descriptor (opened for writing) - where the tar or raw file will be written. It may either reference a file on disk or a pipe/socket. The - third argument specifies in which compression format to write the image. It takes one of + ExportTar()/ExportTarEx() and + ExportRaw()/ExportRaw() implement the reverse operation, and + may be used to export a system image in order to place it in a tar or raw image. They take the machine + name to export as their first parameter, followed by a file descriptor (opened for writing) where the + tar or raw file will be written. It may either reference a file on disk or a pipe/socket. The third + argument specifies in which compression format to write the image. It takes one of uncompressed, xz, bzip2 or gzip, depending on which compression scheme is required. The image written to the - specified file descriptor will be a tar file in case of ExportTar() or a raw disk - image in case of ExportRaw(). Note that currently raw disk images may not be - exported as tar files, and vice versa. This restriction might be lifted eventually. The method - returns a transfer identifier and object path for cancelling or tracking the export operation, similarly - to ImportTar() or ImportRaw() as described above. + specified file descriptor will be a tar file in case of + ExportTar()/ExportTarEx() or a raw disk image in case of + ExportRaw()/ExportRawEx(). Note that currently raw disk + images may not be exported as tar files, and vice versa. This restriction might be lifted + eventually. The method returns a transfer identifier and object path for cancelling or tracking the + export operation, similarly to ImportTar()/ImportTarEx() or + ImportRaw()/ImportRawEx() as described + above. ExportTarEx()/ExportRawEx() expect the image class as + additional parameter, as well as a 64bit flags parameter that currently must be specified as + zero. - PullTar() and PullRaw() may be used to download, verify - and import a system image from a URL. They take a URL argument which should point to a tar or - raw file on the http:// or https:// protocols, possibly - compressed with xz, bzip2 or gzip. The second argument is a local name for the image. It should be - suitable as a hostname, similarly to the matching argument of the ImportTar() and - ImportRaw() methods above. The third argument indicates the verification mode for - the image. It may be one of no, checksum, - signature. no turns off any kind of verification of the image; - checksum looks for a SHA256SUM file next to the downloaded - image and verifies any SHA256 hash value in that file against the image; signature - does the same but also tries to authenticate the SHA256SUM file via - gpg8 - first. The last argument indicates whether to replace a possibly pre-existing image with the same local - name (if true), or whether to fail (if false). Like the import - and export calls above, these calls return a pair of transfer identifier and object path for the ongoing + PullTar()/PullTarEx() and + PullRaw()/PullRawEx() may be used to download, verify and + import a system image from a URL. They take a URL argument which should point to a tar or raw file on + the http:// or https:// protocols, possibly compressed with xz, + bzip2 or gzip. The second argument is a local name for the image. It should be suitable as a hostname, + similarly to the matching argument of the + ImportTar()/ImportTarEx() and + ImportRaw()/ImportRawEx() methods above. The third argument + indicates the verification mode for the image. It may be one of no, + checksum, signature. no turns off any kind of + verification of the image; checksum looks for a SHA256SUM file + next to the downloaded image and verifies any SHA256 hash value in that file against the image; + signature does the same but also tries to authenticate the + SHA256SUM file via gpg8 first. In + case of PullTar()/PullRaw() the last argument indicates + whether to replace a possibly pre-existing image with the same local name (if true), + or whether to fail (if false). In case of + PullTarEx()/PullRawEx() the last argument is a 64bit flags + parameter, where bit 0 controls the force flag, bit 1 is a + read_only flag that controls whether the created image shall be marked read-only, + and bit 2 is a keep_download flag that indicates whether a pristine, read-only copy + of the downloaded image shell be kept, in addition for the local copy of the image. The + …_Ex() variants also expect an image class string (as above). Like the import and + export calls above, these calls return a pair of transfer identifier and object path for the ongoing download. - ListTransfers() returns a list of ongoing import, export or download - operations as created with the six calls described above. It returns an array of structures which - consist of the numeric transfer identifier, a string indicating the operation (one of - import-tar, import-raw, export-tar, - export-raw, pull-tar or pull-raw), a string - describing the remote file (in case of download operations this is the source URL, in case of - import/export operations this is a short string describing the file descriptor passed in), a string - with the local machine image name, a progress value between 0.0 (for 0%) and 1.0 (for 100%), as well as - the transfer object path. + ImportFileSystem()/ImportFileSystemEx() are similar to + ImportTar()/ImportTarEx() but import a directory tree. The + first argument must refer to a directory file descriptor for the source hierarchy to import. + + ListTransfers()/ListTransfersEx() return a list of + ongoing import, export or download operations as created with the six calls described above. They + return an array of structures which consist of the numeric transfer identifier, a string indicating the + operation (one of import-tar, import-raw, + export-tar, export-raw, pull-tar or + pull-raw), a string describing the remote file (in case of download operations this + is the source URL, in case of import/export operations this is a short string describing the file + descriptor passed in), a string with the local machine image name, the image class (only in case of + ListTransfersEx(); one of machine, portable, + sysext, confext), a progress value between 0.0 (for 0%) and 1.0 + (for 100%), as well as the transfer object path. CancelTransfer() may be used to cancel an ongoing import, export or download operation. Simply specify the transfer identifier to cancel the ongoing operation. + + ListImages() returns a list of currently installed images. It takes a image + class string and a flags parameter. The image class is either the empty string or specifies one of the + four image classes, by which it will then filter. The flags parameter must be zero at this time. It + returns an array of items, each describing one image. The item fields are in order: the image class, + the local image name, the image type, the image path, the read-only flag, the creation and modification + times (in microseconds since the UNIX epoch), as well as the current disk usage in bytes (both overall, + and exclusive), as well as any size limit in bytes set on the image (both overall and + exclusive). @@ -242,6 +343,7 @@ node /org/freedesktop/import1/transfer/_1 { signals: LogMessage(u priority, s line); + ProgressUpdate(d progress); properties: @org.freedesktop.DBus.Property.EmitsChangedSignal("const") readonly u Id = ...; @@ -262,8 +364,6 @@ node /org/freedesktop/import1/transfer/_1 { }; - - @@ -274,6 +374,8 @@ node /org/freedesktop/import1/transfer/_1 { + + @@ -315,6 +417,17 @@ node /org/freedesktop/import1/transfer/_1 { between 0.0 and 1.0. To show a progress bar on screen we recommend to query this value in regular intervals, for example every 500 ms or so. + + + Signals + + The LogMessage() signal is emitted for log messages generated by a transfer. It + carries a pair of syslog log level integer and log string. + + The ProgressUpdate() signal is emitted in regular intervals when new + download progress information is available for a transfer. It carries a double precision floating + pointer number between 0.0 and 1.0 indicating the transfer progress. + @@ -340,4 +453,20 @@ node /org/freedesktop/import1/transfer/_1 { + + History + + The Manager Object + ImportTarEx(), ImportRawEx(), + ImportFileSystemEx(), ExportTarEx(), + ExportRawEx(), PullTarEx(), PullRawEx(), + ListTransfersEx(), ListImages() were added in version + 256. + + + Transfer Objects + ProgressUpdate() was added in version 256. + + + diff --git a/man/portablectl.xml b/man/portablectl.xml index 12953bc139..5223131368 100644 --- a/man/portablectl.xml +++ b/man/portablectl.xml @@ -511,6 +511,7 @@ systemd-sysext8 org.freedesktop.portable15 systemd-portabled.service8 + importctl1 diff --git a/man/rules/meson.build b/man/rules/meson.build index 15de3fa576..e6d714ba7d 100644 --- a/man/rules/meson.build +++ b/man/rules/meson.build @@ -23,6 +23,7 @@ manpages = [ ['hostname', '5', [], ''], ['hostnamectl', '1', [], 'ENABLE_HOSTNAMED'], ['hwdb', '7', [], 'ENABLE_HWDB'], + ['importctl', '1', [], 'ENABLE_MACHINED'], ['integritytab', '5', [], 'HAVE_LIBCRYPTSETUP'], ['iocost.conf', '5', [], ''], ['journal-remote.conf', '5', ['journal-remote.conf.d'], 'HAVE_MICROHTTPD'], diff --git a/man/systemd-importd.service.xml b/man/systemd-importd.service.xml index 7e617cfc67..dfeaff4fd7 100644 --- a/man/systemd-importd.service.xml +++ b/man/systemd-importd.service.xml @@ -29,12 +29,12 @@ Description - systemd-importd is a system service that allows importing, exporting and downloading of - system images suitable for running as VM or containers. It is a companion service for - systemd-machined.service8, and provides the implementation for - machinectl1's + systemd-importd is a system service that allows importing, exporting and + downloading of disk images. It provides the implementation for + importctl1's pull-raw, pull-tar, import-raw, - import-tar, import-fs, export-raw, and export-tar commands. + import-tar, import-fs, export-raw, and + export-tar commands. See org.freedesktop.import15 @@ -47,7 +47,7 @@ See Also systemd1 - machinectl1 + importctl1 systemd-machined.service8 systemd-nspawn1 diff --git a/man/systemd-nspawn.xml b/man/systemd-nspawn.xml index 29edbba4cc..8f7e6d103d 100644 --- a/man/systemd-nspawn.xml +++ b/man/systemd-nspawn.xml @@ -1800,19 +1800,7 @@ After=sys-subsystem-net-devices-ens1.device Examples - - Download a - <ulink url="https://getfedora.org">Fedora</ulink> image and start a shell in it - - # machinectl pull-raw --verify=no \ - https://download.fedoraproject.org/pub/fedora/linux/releases/&fedora_latest_version;/Cloud/x86_64/images/Fedora-Cloud-Base-&fedora_latest_version;-&fedora_cloud_release;.x86_64.raw.xz \ - Fedora-Cloud-Base-&fedora_latest_version;-&fedora_cloud_release;.x86-64 -# systemd-nspawn -M Fedora-Cloud-Base-&fedora_latest_version;-&fedora_cloud_release;.x86-64 - - This downloads an image using - machinectl1 - and opens a shell in it. - + Build and boot a minimal Fedora distribution in a container @@ -1921,6 +1909,7 @@ After=sys-subsystem-net-devices-ens1.device zypper8 systemd.slice5 machinectl1 + importctl1 btrfs8 diff --git a/man/systemd-sysext.xml b/man/systemd-sysext.xml index ea1af001a7..8e5a6a6aa1 100644 --- a/man/systemd-sysext.xml +++ b/man/systemd-sysext.xml @@ -449,6 +449,7 @@ systemd1 systemd-nspawn1 systemd-stub7 + importctl1 diff --git a/man/systemd-vmspawn.xml b/man/systemd-vmspawn.xml index c9ad82b2f0..ef4d3ca2b6 100644 --- a/man/systemd-vmspawn.xml +++ b/man/systemd-vmspawn.xml @@ -485,6 +485,8 @@ $ systemd-vmspawn --image=image.raw systemd1 mkosi1 + machinectl1 + importctl1 From befbfca70f37e7e5ed62af17e56b55f7b5292826 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 23 Feb 2024 22:50:30 +0100 Subject: [PATCH 25/26] test: add integration test for importctl (these are explicit tests, on top of the existing machinectl tests that are now chainload importctl) --- test/units/testsuite-13.importctl.sh | 66 +++++++++++++++++++++++++++ test/units/testsuite-13.machinectl.sh | 3 ++ 2 files changed, 69 insertions(+) create mode 100755 test/units/testsuite-13.importctl.sh diff --git a/test/units/testsuite-13.importctl.sh b/test/units/testsuite-13.importctl.sh new file mode 100755 index 0000000000..a13e3fd1fd --- /dev/null +++ b/test/units/testsuite-13.importctl.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: LGPL-2.1-or-later +# shellcheck disable=SC2016 +set -eux +set -o pipefail + +# shellcheck source=test/units/util.sh +. "$(dirname "$0")"/util.sh + +export PAGER= + +at_exit() { + set +e + umount -l -R /var/lib/confexts + rm -f /var/tmp/importtest /var/tmp/importtest2 /var/tmp/importtest.tar.gz /var/tmp/importtest2.tar.gz +} + +trap at_exit EXIT + +systemctl service-log-level systemd-importd debug + +# Mount tmpfs over /var/lib/confexts to not pollute the image +mkdir -p /var/lib/confexts +mount -t tmpfs tmpfs /var/lib/confexts -o mode=755 + +importctl +importctl --no-pager --help +importctl --version +importctl list-transfers +importctl list-transfers --no-legend --no-ask-password +importctl list-transfers -j +importctl list-images +importctl list-images --no-legend --no-ask-password +importctl list-images -j + +(! importctl cancel-transfer 4711) + +dd if=/dev/urandom of=/var/tmp/importtest bs=4096 count=10 + +importctl import-raw --class=confext /var/tmp/importtest +cmp /var/tmp/importtest /var/lib/confexts/importtest.raw +importctl export-raw --class=confext importtest /var/tmp/importtest2 +cmp /var/tmp/importtest /var/tmp/importtest2 + +(! importctl pull-raw --class=confext file:///var/tmp/importtest) +importctl pull-raw --verify=no --class=confext file:///var/tmp/importtest importtest3 +cmp /var/tmp/importtest /var/lib/confexts/importtest3.raw + +tar czf /var/tmp/importtest.tar.gz -C /var/tmp importtest + +importctl import-tar --class=confext /var/tmp/importtest.tar.gz importtest4 +cmp /var/tmp/importtest /var/lib/confexts/importtest4/importtest + +importctl export-tar --class=confext importtest4 /var/tmp/importtest2.tar.gz +importctl import-tar --class=confext /var/tmp/importtest2.tar.gz importtest5 +cmp /var/tmp/importtest /var/lib/confexts/importtest5/importtest + +importctl import-fs --class=confext /var/lib/confexts/importtest5 importtest6 +cmp /var/tmp/importtest /var/lib/confexts/importtest6/importtest + +(! importctl pull-tar --class=confext file:///var/tmp/importtest.tar.gz importtest7) +importctl pull-tar --class=confext --verify=no file:///var/tmp/importtest.tar.gz importtest7 +cmp /var/tmp/importtest /var/lib/confexts/importtest7/importtest + +importctl list-images +importctl list-images -j diff --git a/test/units/testsuite-13.machinectl.sh b/test/units/testsuite-13.machinectl.sh index b5f90f6d0c..c3bfde2768 100755 --- a/test/units/testsuite-13.machinectl.sh +++ b/test/units/testsuite-13.machinectl.sh @@ -20,6 +20,9 @@ at_exit() { trap at_exit EXIT +systemctl service-log-level systemd-machined debug +systemctl service-log-level systemd-importd debug + # Mount tmpfs over /var/lib/machines to not pollute the image mkdir -p /var/lib/machines mount -t tmpfs tmpfs /var/lib/machines From 7e9cdafad349714913538f8adba038640bf1b577 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Tue, 27 Feb 2024 15:05:26 +0100 Subject: [PATCH 26/26] update TODO --- TODO | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/TODO b/TODO index 80a2cd60b2..e6a785655f 100644 --- a/TODO +++ b/TODO @@ -133,6 +133,12 @@ Features: * add a new specifier to unit files that figures out the DDI the unit file is from, tracing through overlayfs, DM, loopback block device. +* importd/importctl + - import generator + - port tar handling to libarchive + - add varlink interface + - download images into .v/ dirs + * in os-release define a field that can be initialized at build time from SOURCE_DATE_EPOCH (maybe even under that name?). Would then be used to initialize the timestamp logic of ConditionNeedsUpdate=. @@ -153,10 +159,6 @@ Features: pidfd, so that we can reasonably robustly do this. Would only cover the execution environment like namespaces, but not the privilege settings. -* add a generic client to importd called importctl, then make machinectl just - chain-exec() it. Make sure importd/importctl can be used for sysext images, - portable images too. - * varlink: extend varlink IDL macros to include documentation strings * Introduce a CGroupRef structure, inspired by PidRef. Should contain cgroup @@ -1541,8 +1543,6 @@ Features: that our log messages could contain clickable links for example for unit files and suchlike we operate on. -* importd: add ability download images for portabled + sysext - * add support for "portablectl attach http://foobar.com/waaa.raw (i.e. importd integration) * sync dynamic uids/gids between host+portable srvice (i.e. if DynamicUser=1 is set for a service, make sure that the