From 05f47839e1feca299191aaa1251c041d61ada1fc Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 29 May 2024 09:33:27 +0200 Subject: [PATCH 1/7] varlink: add varlink_many_*() helpers that send messages to a set of connections at once If we have a Varlink interface that allows man subscribers to the same source of events we typically want to notify all of them at once with the same message. Let's add some helpers for this purpose: varlink_many_*() can be used to send messages, similar to the corresponding varlink_*() calls, but they take a Set of Varlink objects and we issue the operation on every connection once. This ports over one existing case where this is beneficial, but the main user for this is supposed to be importd later. --- src/resolve/resolved-manager.c | 53 ++++++++++++++++------------------ src/shared/varlink.c | 48 ++++++++++++++++++++++++++++++ src/shared/varlink.h | 9 +++++- 3 files changed, 81 insertions(+), 29 deletions(-) diff --git a/src/resolve/resolved-manager.c b/src/resolve/resolved-manager.c index 2eee418adf..9bbdc648fc 100644 --- a/src/resolve/resolved-manager.c +++ b/src/resolve/resolved-manager.c @@ -1165,7 +1165,6 @@ static int dns_question_to_json(DnsQuestion *q, sd_json_variant **ret) { int manager_monitor_send(Manager *m, DnsQuery *q) { _cleanup_(sd_json_variant_unrefp) sd_json_variant *jquestion = NULL, *jcollected_questions = NULL, *janswer = NULL; _cleanup_(dns_question_unrefp) DnsQuestion *merged = NULL; - Varlink *connection; DnsAnswerItem *rri; int r; @@ -1220,34 +1219,32 @@ int manager_monitor_send(Manager *m, DnsQuery *q) { return log_debug_errno(r, "Failed to append notification entry to array: %m"); } - SET_FOREACH(connection, m->varlink_subscription) { - r = varlink_notifybo( - connection, - SD_JSON_BUILD_PAIR("state", SD_JSON_BUILD_STRING(dns_transaction_state_to_string(q->state))), - SD_JSON_BUILD_PAIR_CONDITION(q->state == DNS_TRANSACTION_DNSSEC_FAILED, - "result", SD_JSON_BUILD_STRING(dnssec_result_to_string(q->answer_dnssec_result))), - SD_JSON_BUILD_PAIR_CONDITION(q->state == DNS_TRANSACTION_RCODE_FAILURE, - "rcode", SD_JSON_BUILD_INTEGER(q->answer_rcode)), - SD_JSON_BUILD_PAIR_CONDITION(q->state == DNS_TRANSACTION_ERRNO, - "errno", SD_JSON_BUILD_INTEGER(q->answer_errno)), - SD_JSON_BUILD_PAIR_CONDITION(IN_SET(q->state, - DNS_TRANSACTION_DNSSEC_FAILED, - DNS_TRANSACTION_RCODE_FAILURE) && - q->answer_ede_rcode >= 0, - "extendedDNSErrorCode", SD_JSON_BUILD_INTEGER(q->answer_ede_rcode)), - SD_JSON_BUILD_PAIR_CONDITION(IN_SET(q->state, - DNS_TRANSACTION_DNSSEC_FAILED, - DNS_TRANSACTION_RCODE_FAILURE) && - q->answer_ede_rcode >= 0 && !isempty(q->answer_ede_msg), - "extendedDNSErrorMessage", SD_JSON_BUILD_STRING(q->answer_ede_msg)), - SD_JSON_BUILD_PAIR("question", SD_JSON_BUILD_VARIANT(jquestion)), - SD_JSON_BUILD_PAIR_CONDITION(!!jcollected_questions, - "collectedQuestions", SD_JSON_BUILD_VARIANT(jcollected_questions)), - SD_JSON_BUILD_PAIR_CONDITION(!!janswer, + r = varlink_many_notifybo( + m->varlink_subscription, + SD_JSON_BUILD_PAIR("state", SD_JSON_BUILD_STRING(dns_transaction_state_to_string(q->state))), + SD_JSON_BUILD_PAIR_CONDITION(q->state == DNS_TRANSACTION_DNSSEC_FAILED, + "result", SD_JSON_BUILD_STRING(dnssec_result_to_string(q->answer_dnssec_result))), + SD_JSON_BUILD_PAIR_CONDITION(q->state == DNS_TRANSACTION_RCODE_FAILURE, + "rcode", SD_JSON_BUILD_INTEGER(q->answer_rcode)), + SD_JSON_BUILD_PAIR_CONDITION(q->state == DNS_TRANSACTION_ERRNO, + "errno", SD_JSON_BUILD_INTEGER(q->answer_errno)), + SD_JSON_BUILD_PAIR_CONDITION(IN_SET(q->state, + DNS_TRANSACTION_DNSSEC_FAILED, + DNS_TRANSACTION_RCODE_FAILURE) && + q->answer_ede_rcode >= 0, + "extendedDNSErrorCode", SD_JSON_BUILD_INTEGER(q->answer_ede_rcode)), + SD_JSON_BUILD_PAIR_CONDITION(IN_SET(q->state, + DNS_TRANSACTION_DNSSEC_FAILED, + DNS_TRANSACTION_RCODE_FAILURE) && + q->answer_ede_rcode >= 0 && !isempty(q->answer_ede_msg), + "extendedDNSErrorMessage", SD_JSON_BUILD_STRING(q->answer_ede_msg)), + SD_JSON_BUILD_PAIR("question", SD_JSON_BUILD_VARIANT(jquestion)), + SD_JSON_BUILD_PAIR_CONDITION(!!jcollected_questions, + "collectedQuestions", SD_JSON_BUILD_VARIANT(jcollected_questions)), + SD_JSON_BUILD_PAIR_CONDITION(!!janswer, "answer", SD_JSON_BUILD_VARIANT(janswer))); - if (r < 0) - log_debug_errno(r, "Failed to send monitor event, ignoring: %m"); - } + if (r < 0) + log_debug_errno(r, "Failed to send monitor event, ignoring: %m"); return 0; } diff --git a/src/shared/varlink.c b/src/shared/varlink.c index 8329f39021..d0a86157a2 100644 --- a/src/shared/varlink.c +++ b/src/shared/varlink.c @@ -4162,3 +4162,51 @@ int varlink_error_to_errno(const char *error, sd_json_variant *parameters) { return -EBADR; /* Catch-all */ } + +int varlink_many_notifyb(Set *s, ...) { + int r; + + /* Equivalent to varlink_notifyb(), but does this for each entry of the supplied set of Varlink connections */ + + if (set_isempty(s)) + return 0; + + _cleanup_(sd_json_variant_unrefp) sd_json_variant *parameters = NULL; + va_list ap; + va_start(ap, s); + r = sd_json_buildv(¶meters, ap); + va_end(ap); + if (r < 0) + return r; + + int ret = 1; + Varlink *link; + SET_FOREACH(link, s) + RET_GATHER(ret, varlink_notify(link, parameters)); + + return ret; +} + +int varlink_many_reply(Set *s, sd_json_variant *parameters) { + if (set_isempty(s)) + return 0; + + int ret = 1; + Varlink *link; + SET_FOREACH(link, s) + RET_GATHER(ret, varlink_reply(link, parameters)); + + return ret; +} + +int varlink_many_error(Set *s, const char *error_id, sd_json_variant *parameters) { + if (set_isempty(s)) + return 0; + + int ret = 1; + Varlink *link; + SET_FOREACH(link, s) + RET_GATHER(ret, varlink_error(link, error_id, parameters)); + + return ret; +} diff --git a/src/shared/varlink.h b/src/shared/varlink.h index 739786b975..40a8fd086f 100644 --- a/src/shared/varlink.h +++ b/src/shared/varlink.h @@ -2,9 +2,10 @@ #pragma once #include "sd-event.h" - #include "sd-json.h" + #include "pidref.h" +#include "set.h" #include "time-util.h" #include "varlink-idl.h" @@ -264,6 +265,12 @@ int varlink_invocation(VarlinkInvocationFlags flags); int varlink_error_to_errno(const char *error, sd_json_variant *parameters); +int varlink_many_notifyb(Set *s, ...); +#define varlink_many_notifybo(s, ...) \ + varlink_many_notifyb((s), SD_JSON_BUILD_OBJECT(__VA_ARGS__)) +int varlink_many_reply(Set *s, sd_json_variant *parameters); +int varlink_many_error(Set *s, const char *error_id, sd_json_variant *parameters); + DEFINE_TRIVIAL_CLEANUP_FUNC(Varlink *, varlink_unref); DEFINE_TRIVIAL_CLEANUP_FUNC(Varlink *, varlink_close_unref); DEFINE_TRIVIAL_CLEANUP_FUNC(Varlink *, varlink_flush_close_unref); From 71613cd59a890514c8dddf27659912b58435c1fe Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Sat, 11 May 2024 14:00:25 +0200 Subject: [PATCH 2/7] import: add generic enum for choosing between tar + raw downloads --- src/shared/import-util.c | 11 +++++++++-- src/shared/import-util.h | 10 ++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/shared/import-util.c b/src/shared/import-util.c index 9057b78d28..bcdc21f79e 100644 --- a/src/shared/import-util.c +++ b/src/shared/import-util.c @@ -126,9 +126,16 @@ int import_url_change_suffix( return 0; } +static const char* const import_type_table[_IMPORT_TYPE_MAX] = { + [IMPORT_RAW] = "raw", + [IMPORT_TAR] = "tar", +}; + +DEFINE_STRING_TABLE_LOOKUP(import_type, ImportType); + static const char* const import_verify_table[_IMPORT_VERIFY_MAX] = { - [IMPORT_VERIFY_NO] = "no", - [IMPORT_VERIFY_CHECKSUM] = "checksum", + [IMPORT_VERIFY_NO] = "no", + [IMPORT_VERIFY_CHECKSUM] = "checksum", [IMPORT_VERIFY_SIGNATURE] = "signature", }; diff --git a/src/shared/import-util.h b/src/shared/import-util.h index 3b2425b916..98b99e371f 100644 --- a/src/shared/import-util.h +++ b/src/shared/import-util.h @@ -5,6 +5,13 @@ #include "macro.h" +typedef enum ImportType { + IMPORT_RAW, + IMPORT_TAR, + _IMPORT_TYPE_MAX, + _IMPORT_TYPE_INVALID = -EINVAL, +} ImportType; + typedef enum ImportVerify { IMPORT_VERIFY_NO, IMPORT_VERIFY_CHECKSUM, @@ -25,6 +32,9 @@ static inline int import_url_append_component(const char *url, const char *suffi return import_url_change_suffix(url, 0, suffix, ret); } +const char* import_type_to_string(ImportType v) _const_; +ImportType import_type_from_string(const char *s) _pure_; + const char* import_verify_to_string(ImportVerify v) _const_; ImportVerify import_verify_from_string(const char *s) _pure_; From 17a6043a145f19f4f5b020fa4bb9c82c1ce160e7 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 8 May 2024 10:42:48 +0200 Subject: [PATCH 3/7] importd: add simple varlink API For now, let's just support Pull() and ListTransfers(), but this is just a start. --- src/import/importd.c | 288 ++++++++++++++++++++++++- src/shared/meson.build | 1 + src/shared/varlink-io.systemd.Import.c | 129 +++++++++++ src/shared/varlink-io.systemd.Import.h | 6 + src/test/test-varlink-idl.c | 3 + 5 files changed, 422 insertions(+), 5 deletions(-) create mode 100644 src/shared/varlink-io.systemd.Import.c create mode 100644 src/shared/varlink-io.systemd.Import.h diff --git a/src/import/importd.c b/src/import/importd.c index c6ca31556a..5ef89ac884 100644 --- a/src/import/importd.c +++ b/src/import/importd.c @@ -22,6 +22,7 @@ #include "hostname-util.h" #include "import-common.h" #include "import-util.h" +#include "json-util.h" #include "machine-pool.h" #include "main-func.h" #include "missing_capability.h" @@ -39,6 +40,8 @@ #include "strv.h" #include "syslog-util.h" #include "user-util.h" +#include "varlink.h" +#include "varlink-io.systemd.Import.h" #include "web-util.h" typedef struct Transfer Transfer; @@ -87,11 +90,14 @@ struct Transfer { int stdin_fd; int stdout_fd; + + Set *varlink_subscribed; }; struct Manager { sd_event *event; sd_bus *bus; + VarlinkServer *varlink_server; uint32_t current_transfer_id; Hashmap *transfers; @@ -120,6 +126,8 @@ static const char* const transfer_type_table[_TRANSFER_TYPE_MAX] = { DEFINE_PRIVATE_STRING_TABLE_LOOKUP_TO_STRING(transfer_type, TransferType); +DEFINE_PRIVATE_HASH_OPS_WITH_VALUE_DESTRUCTOR(varlink_hash_ops, void, trivial_hash_func, trivial_compare_func, Varlink, varlink_unref); + static Transfer *transfer_unref(Transfer *t) { if (!t) return NULL; @@ -141,6 +149,8 @@ static Transfer *transfer_unref(Transfer *t) { safe_close(t->stdin_fd); safe_close(t->stdout_fd); + set_free(t->varlink_subscribed); + return mfree(t); } @@ -218,7 +228,16 @@ static void transfer_send_log_line(Transfer *t, const char *line) { priority, line); if (r < 0) - log_warning_errno(r, "Cannot emit log message signal, ignoring: %m"); + log_warning_errno(r, "Cannot emit log message bus signal, ignoring: %m"); + + r = varlink_many_notifybo( + t->varlink_subscribed, + SD_JSON_BUILD_PAIR("log", + SD_JSON_BUILD_OBJECT( + SD_JSON_BUILD_PAIR_UNSIGNED("priority", priority), + SD_JSON_BUILD_PAIR_STRING("message", line)))); + if (r < 0) + log_warning_errno(r, "Cannot emit log message varlink message, ignoring: %m"); } static void transfer_send_progress_update(Transfer *t) { @@ -229,15 +248,23 @@ static void transfer_send_progress_update(Transfer *t) { if (t->progress_percent_sent == t->progress_percent) return; + double progress = transfer_percent_as_double(t); + r = sd_bus_emit_signal( t->manager->bus, t->object_path, "org.freedesktop.import1.Transfer", "ProgressUpdate", "d", - transfer_percent_as_double(t)); + progress); if (r < 0) - log_warning_errno(r, "Cannot emit progress update signal, ignoring: %m"); + log_warning_errno(r, "Cannot emit progress update bus signal, ignoring: %m"); + + r = varlink_many_notifybo( + t->varlink_subscribed, + SD_JSON_BUILD_PAIR_REAL("progress", progress)); + if (r < 0) + log_warning_errno(r, "Cannot emit progress update varlink message, ignoring: %m"); t->progress_percent_sent = t->progress_percent; } @@ -314,10 +341,18 @@ static int transfer_finalize(Transfer *t, bool success) { t->object_path, success ? "done" : t->n_canceled > 0 ? "canceled" : "failed"); - if (r < 0) log_error_errno(r, "Cannot emit message: %m"); + if (success) + r = varlink_many_reply(t->varlink_subscribed, NULL); + else if (t->n_canceled > 0) + r = varlink_many_error(t->varlink_subscribed, "io.systemd.Import.TransferCancelled", NULL); + else + r = varlink_many_error(t->varlink_subscribed, "io.systemd.Import.TransferFailed", NULL); + if (r < 0) + log_warning_errno(r, "Cannot emit varlink reply, ignoring: %m"); + transfer_unref(t); return 0; } @@ -587,6 +622,8 @@ static Manager *manager_unref(Manager *m) { hashmap_free(m->polkit_registry); m->bus = sd_bus_flush_close_unref(m->bus); + m->varlink_server = varlink_server_unref(m->varlink_server); + sd_event_unref(m->event); return mfree(m); @@ -1729,11 +1766,248 @@ static int manager_connect_bus(Manager *m) { return 0; } +static JSON_DISPATCH_ENUM_DEFINE(json_dispatch_image_class, ImageClass, image_class_from_string); + +static int make_transfer_json(Transfer *t, sd_json_variant **ret) { + int r; + + assert(t); + + r = sd_json_buildo(ret, + SD_JSON_BUILD_PAIR("id", SD_JSON_BUILD_UNSIGNED(t->id)), + SD_JSON_BUILD_PAIR("type", JSON_BUILD_STRING_UNDERSCORIFY(transfer_type_to_string(t->type))), + SD_JSON_BUILD_PAIR("remote", SD_JSON_BUILD_STRING(t->remote)), + SD_JSON_BUILD_PAIR("local", SD_JSON_BUILD_STRING(t->local)), + SD_JSON_BUILD_PAIR("class", JSON_BUILD_STRING_UNDERSCORIFY(image_class_to_string(t->class))), + SD_JSON_BUILD_PAIR("percent", SD_JSON_BUILD_REAL(transfer_percent_as_double(t)))); + if (r < 0) + return log_error_errno(r, "Failed to build transfer JSON data: %m"); + + return 0; +} + +static int vl_method_list_transfers(Varlink *link, sd_json_variant *parameters, VarlinkMethodFlags flags, void *userdata) { + + struct p { + ImageClass class; + } p = { + .class = _IMAGE_CLASS_INVALID, + }; + + static const sd_json_dispatch_field dispatch_table[] = { + { "class", SD_JSON_VARIANT_STRING, json_dispatch_image_class, offsetof(struct p, class), 0 }, + {}, + }; + + Manager *m = ASSERT_PTR(userdata); + int r; + + assert(link); + assert(parameters); + + r = varlink_dispatch(link, parameters, dispatch_table, &p); + if (r != 0) + return r; + + if (!FLAGS_SET(flags, VARLINK_METHOD_MORE)) + return varlink_error(link, VARLINK_ERROR_EXPECTED_MORE, NULL); + + Transfer *previous = NULL, *t; + HASHMAP_FOREACH(t, m->transfers) { + + if (p.class >= 0 && p.class != t->class) + continue; + + if (previous) { + _cleanup_(sd_json_variant_unrefp) sd_json_variant *v = NULL; + + r = make_transfer_json(previous, &v); + if (r < 0) + return r; + + r = varlink_notify(link, v); + if (r < 0) + return r; + } + + previous = t; + } + + if (previous) { + _cleanup_(sd_json_variant_unrefp) sd_json_variant *v = NULL; + + r = make_transfer_json(previous, &v); + if (r < 0) + return r; + + return varlink_reply(link, v); + } + + return varlink_error(link, "io.systemd.Import.NoTransfers", NULL); +} + +static JSON_DISPATCH_ENUM_DEFINE(json_dispatch_import_verify, ImportVerify, import_verify_from_string); +static JSON_DISPATCH_ENUM_DEFINE(json_dispatch_import_type, ImportType, import_type_from_string); + +static int vl_method_pull(Varlink *link, sd_json_variant *parameters, VarlinkMethodFlags flags, void *userdata) { + + struct p { + const char *remote, *local; + ImageClass class; + ImportType type; + ImportVerify verify; + bool force; + bool read_only; + bool keep_download; + } p = { + .class = _IMAGE_CLASS_INVALID, + .verify = IMPORT_VERIFY_SIGNATURE, + }; + + static const sd_json_dispatch_field dispatch_table[] = { + { "remote", SD_JSON_VARIANT_STRING, sd_json_dispatch_const_string, offsetof(struct p, remote), SD_JSON_MANDATORY }, + { "local", SD_JSON_VARIANT_STRING, sd_json_dispatch_const_string, offsetof(struct p, local), 0 }, + { "class", SD_JSON_VARIANT_STRING, json_dispatch_image_class, offsetof(struct p, class), SD_JSON_MANDATORY }, + { "type", SD_JSON_VARIANT_STRING, json_dispatch_import_type, offsetof(struct p, type), SD_JSON_MANDATORY }, + { "verify", SD_JSON_VARIANT_STRING, json_dispatch_import_verify, offsetof(struct p, verify), SD_JSON_STRICT }, + { "force", SD_JSON_VARIANT_BOOLEAN, sd_json_dispatch_stdbool, offsetof(struct p, force), 0 }, + { "readOnly", SD_JSON_VARIANT_BOOLEAN, sd_json_dispatch_stdbool, offsetof(struct p, read_only), 0 }, + { "keepDownload", SD_JSON_VARIANT_BOOLEAN, sd_json_dispatch_stdbool, offsetof(struct p, keep_download), 0 }, + VARLINK_DISPATCH_POLKIT_FIELD, + {}, + }; + + Manager *m = ASSERT_PTR(userdata); + int r; + + assert(link); + assert(parameters); + + r = varlink_dispatch(link, parameters, dispatch_table, &p); + if (r != 0) + return r; + + if (!http_url_is_valid(p.remote) && !file_url_is_valid(p.remote)) + return varlink_error_invalid_parameter_name(link, "remote"); + + if (p.local && !image_name_is_valid(p.local)) + return varlink_error_invalid_parameter_name(link, "local"); + + uint64_t transfer_flags = (p.force * IMPORT_FORCE) | (p.read_only * IMPORT_READ_ONLY) | (p.keep_download * IMPORT_PULL_KEEP_DOWNLOAD); + + TransferType tt = + p.type == IMPORT_TAR ? TRANSFER_PULL_TAR : + p.type == IMPORT_RAW ? TRANSFER_PULL_RAW : _TRANSFER_TYPE_INVALID; + + assert(tt >= 0); + + if (manager_find(m, tt, p.remote)) + return varlink_errorbo(link, "io.systemd.Import.AlreadyInProgress", SD_JSON_BUILD_PAIR_STRING("remote", p.remote)); + + r = varlink_verify_polkit_async( + link, + m->bus, + "org.freedesktop.import1.pull", + (const char**) STRV_MAKE( + "remote", p.remote, + "local", p.local, + "class", image_class_to_string(p.class), + "type", import_type_to_string(p.type), + "verify", import_verify_to_string(p.verify)), + &m->polkit_registry); + if (r <= 0) + return r; + + _cleanup_(transfer_unrefp) Transfer *t = NULL; + + r = transfer_new(m, &t); + if (r < 0) + return r; + + t->type = tt; + t->verify = p.verify; + t->flags = transfer_flags; + t->class = p.class; + + t->remote = strdup(p.remote); + if (!t->remote) + return -ENOMEM; + + if (p.local) { + t->local = strdup(p.local); + if (!t->local) + return -ENOMEM; + } + + r = transfer_start(t); + if (r < 0) + return r; + + /* If more was not set, just return the download id, and be done with it */ + if (!FLAGS_SET(flags, VARLINK_METHOD_MORE)) + return varlink_replybo(link, SD_JSON_BUILD_PAIR("id", SD_JSON_BUILD_UNSIGNED(t->id))); + + /* Otherwise add this connection to the set of subscriptions, return the id, but keep the thing running */ + r = set_ensure_put(&t->varlink_subscribed, &varlink_hash_ops, link); + if (r < 0) + return r; + + varlink_ref(link); + + r = varlink_notifybo(link, SD_JSON_BUILD_PAIR("id", SD_JSON_BUILD_UNSIGNED(t->id))); + if (r < 0) + return r; + + TAKE_PTR(t); + return 0; +} + +static int manager_connect_varlink(Manager *m) { + int r; + + assert(m); + assert(m->event); + assert(!m->varlink_server); + + r = varlink_server_new(&m->varlink_server, VARLINK_SERVER_ACCOUNT_UID|VARLINK_SERVER_INHERIT_USERDATA); + if (r < 0) + return log_error_errno(r, "Failed to allocate Varlink server: %m"); + + varlink_server_set_userdata(m->varlink_server, m); + + r = varlink_server_add_interface(m->varlink_server, &vl_interface_io_systemd_Import); + if (r < 0) + return log_error_errno(r, "Failed to add Import interface to varlink server: %m"); + + r = varlink_server_bind_method_many( + m->varlink_server, + "io.systemd.Import.ListTransfers", vl_method_list_transfers, + "io.systemd.Import.Pull", vl_method_pull); + if (r < 0) + return log_error_errno(r, "Failed to bind Varlink method calls: %m"); + + r = varlink_server_attach_event(m->varlink_server, m->event, SD_EVENT_PRIORITY_NORMAL); + if (r < 0) + return log_error_errno(r, "Failed to attach Varlink server to event loop: %m"); + + r = varlink_server_listen_auto(m->varlink_server); + if (r < 0) + return log_error_errno(r, "Failed to bind to passed Varlink sockets: %m"); + if (r == 0) { + r = varlink_server_listen_address(m->varlink_server, "/run/systemd/io.systemd.Import", 0666); + if (r < 0) + return log_error_errno(r, "Failed to bind to Varlink socket: %m"); + } + + return 0; +} + static bool manager_check_idle(void *userdata) { Manager *m = ASSERT_PTR(userdata); return hashmap_isempty(m->transfers) && - hashmap_isempty(m->polkit_registry); + hashmap_isempty(m->polkit_registry) && + varlink_server_current_connections(m->varlink_server) == 0; } static void manager_parse_env(Manager *m) { @@ -1786,6 +2060,10 @@ static int run(int argc, char *argv[]) { if (r < 0) return r; + r = manager_connect_varlink(m); + if (r < 0) + return r; + r = sd_notify(false, NOTIFY_READY); if (r < 0) log_warning_errno(r, "Failed to send readiness notification, ignoring: %m"); diff --git a/src/shared/meson.build b/src/shared/meson.build index e33e924b55..7da8f4f96d 100644 --- a/src/shared/meson.build +++ b/src/shared/meson.build @@ -180,6 +180,7 @@ shared_sources = files( 'varlink-io.systemd.BootControl.c', 'varlink-io.systemd.Credentials.c', 'varlink-io.systemd.Hostname.c', + 'varlink-io.systemd.Import.c', 'varlink-io.systemd.Journal.c', 'varlink-io.systemd.Machine.c', 'varlink-io.systemd.ManagedOOM.c', diff --git a/src/shared/varlink-io.systemd.Import.c b/src/shared/varlink-io.systemd.Import.c new file mode 100644 index 0000000000..4b2f3dcdea --- /dev/null +++ b/src/shared/varlink-io.systemd.Import.c @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include "varlink-io.systemd.Import.h" + +static VARLINK_DEFINE_ENUM_TYPE( + ImageClass, + VARLINK_FIELD_COMMENT("An image to boot as a system on baremetal, in a VM or as a container"), + VARLINK_DEFINE_ENUM_VALUE(machine), + VARLINK_FIELD_COMMENT("An portable service image"), + VARLINK_DEFINE_ENUM_VALUE(portable), + VARLINK_FIELD_COMMENT("A system extension image"), + VARLINK_DEFINE_ENUM_VALUE(sysext), + VARLINK_FIELD_COMMENT("A configuration extension image"), + VARLINK_DEFINE_ENUM_VALUE(confext)); + +static VARLINK_DEFINE_ENUM_TYPE( + RemoteType, + VARLINK_FIELD_COMMENT("Raw binary disk images, typically in a GPT envelope"), + VARLINK_DEFINE_ENUM_VALUE(raw), + VARLINK_FIELD_COMMENT("A tarball, optionally compressed"), + VARLINK_DEFINE_ENUM_VALUE(tar)); + +static VARLINK_DEFINE_ENUM_TYPE( + TransferType, + VARLINK_FIELD_COMMENT("A local import of a tarball"), + VARLINK_DEFINE_ENUM_VALUE(import_tar), + VARLINK_FIELD_COMMENT("A local import of a raw disk image"), + VARLINK_DEFINE_ENUM_VALUE(import_raw), + VARLINK_FIELD_COMMENT("A local import of a file system tree"), + VARLINK_DEFINE_ENUM_VALUE(import_fs), + VARLINK_FIELD_COMMENT("A local export of a tarball"), + VARLINK_DEFINE_ENUM_VALUE(export_tar), + VARLINK_FIELD_COMMENT("A local export of a raw disk image"), + VARLINK_DEFINE_ENUM_VALUE(export_raw), + VARLINK_FIELD_COMMENT("A download of a tarball"), + VARLINK_DEFINE_ENUM_VALUE(pull_tar), + VARLINK_FIELD_COMMENT("A download of a raw disk image"), + VARLINK_DEFINE_ENUM_VALUE(pull_raw)); + +static VARLINK_DEFINE_ENUM_TYPE( + ImageVerify, + VARLINK_FIELD_COMMENT("No verification"), + VARLINK_DEFINE_ENUM_VALUE(no), + VARLINK_FIELD_COMMENT("Verify that downloads match checksum file (SHA256SUMS), but do not check signature of checksum file"), + VARLINK_DEFINE_ENUM_VALUE(checksum), + VARLINK_FIELD_COMMENT("Verify that downloads match checksum file (SHA256SUMS), and check signature of checksum file."), + VARLINK_DEFINE_ENUM_VALUE(signature)); + +static VARLINK_DEFINE_STRUCT_TYPE( + LogMessage, + VARLINK_FIELD_COMMENT("The log message"), + VARLINK_DEFINE_FIELD(message, VARLINK_STRING, 0), + VARLINK_FIELD_COMMENT("The priority of the log message, using the BSD syslog priority levels"), + VARLINK_DEFINE_FIELD(priority, VARLINK_INT, 0)); + +static VARLINK_DEFINE_METHOD( + ListTransfers, + VARLINK_FIELD_COMMENT("Image class to filter by"), + VARLINK_DEFINE_INPUT_BY_TYPE(class, ImageClass, VARLINK_NULLABLE), + VARLINK_FIELD_COMMENT("A unique numeric identifier for the ongoing transfer"), + VARLINK_DEFINE_OUTPUT(id, VARLINK_INT, 0), + VARLINK_FIELD_COMMENT("The type of transfer"), + VARLINK_DEFINE_OUTPUT_BY_TYPE(type, TransferType, 0), + VARLINK_FIELD_COMMENT("The remote URL"), + VARLINK_DEFINE_OUTPUT(remote, VARLINK_STRING, 0), + VARLINK_FIELD_COMMENT("The local image name"), + VARLINK_DEFINE_OUTPUT(local, VARLINK_STRING, 0), + VARLINK_FIELD_COMMENT("The class of the image"), + VARLINK_DEFINE_OUTPUT_BY_TYPE(class, ImageClass, 0), + VARLINK_FIELD_COMMENT("Progress in percent"), + VARLINK_DEFINE_OUTPUT(percent, VARLINK_FLOAT, 0)); + +static VARLINK_DEFINE_METHOD( + Pull, + VARLINK_FIELD_COMMENT("The remote URL to download from"), + VARLINK_DEFINE_INPUT(remote, VARLINK_STRING, 0), + VARLINK_FIELD_COMMENT("The local image name to download to"), + VARLINK_DEFINE_INPUT(local, VARLINK_STRING, VARLINK_NULLABLE), + VARLINK_FIELD_COMMENT("The type of the resource"), + VARLINK_DEFINE_INPUT_BY_TYPE(type, RemoteType, 0), + VARLINK_FIELD_COMMENT("The image class"), + VARLINK_DEFINE_INPUT_BY_TYPE(class, ImageClass, 0), + VARLINK_FIELD_COMMENT("The whether and how thoroughly to verify the download before installing it locally. Defauts to 'signature'."), + VARLINK_DEFINE_INPUT_BY_TYPE(verify, ImageVerify, VARLINK_NULLABLE), + VARLINK_FIELD_COMMENT("If true, an existing image by the local name is deleted. Defaults to false."), + VARLINK_DEFINE_INPUT(force, VARLINK_BOOL, VARLINK_NULLABLE), + VARLINK_FIELD_COMMENT("Whether to make the image read-only after downloading. Defaults ot false."), + VARLINK_DEFINE_INPUT(readOnly, VARLINK_BOOL, VARLINK_NULLABLE), + VARLINK_FIELD_COMMENT("Whether to keep a pristine copy of the download separate from the locally installed image. Defaults to false."), + VARLINK_DEFINE_INPUT(keepDownload, VARLINK_BOOL, VARLINK_NULLABLE), + VARLINK_FIELD_COMMENT("Whether to permit interactive authentication. Defaults to false."), + VARLINK_DEFINE_INPUT(allowInteractiveAuthentication, VARLINK_BOOL, VARLINK_NULLABLE), + VARLINK_FIELD_COMMENT("A progress update, as percent value"), + VARLINK_DEFINE_OUTPUT(progress, VARLINK_FLOAT, VARLINK_NULLABLE), + VARLINK_FIELD_COMMENT("A log message about the ongoing transfer"), + VARLINK_DEFINE_OUTPUT_BY_TYPE(log, LogMessage, VARLINK_NULLABLE), + VARLINK_FIELD_COMMENT("The numeric ID of this download"), + VARLINK_DEFINE_OUTPUT(id, VARLINK_INT, VARLINK_NULLABLE)); + +static VARLINK_DEFINE_ERROR(AlreadyInProgress); +static VARLINK_DEFINE_ERROR(TransferCancelled); +static VARLINK_DEFINE_ERROR(TransferFailed); +static VARLINK_DEFINE_ERROR(NoTransfers); + +VARLINK_DEFINE_INTERFACE( + io_systemd_Import, + "io.systemd.Import", + VARLINK_SYMBOL_COMMENT("Describes the class of images"), + &vl_type_ImageClass, + VARLINK_SYMBOL_COMMENT("Describes the type of a images to transfer"), + &vl_type_RemoteType, + VARLINK_SYMBOL_COMMENT("Describes the type of a transfer"), + &vl_type_TransferType, + VARLINK_SYMBOL_COMMENT("Describes whether and how thoroughly to verify the download before installing it locally"), + &vl_type_ImageVerify, + VARLINK_SYMBOL_COMMENT("Structure for log messages associated with a transfer operation"), + &vl_type_LogMessage, + VARLINK_SYMBOL_COMMENT("List ongoing transfers, or query details about specific transfers"), + &vl_method_ListTransfers, + VARLINK_SYMBOL_COMMENT("Download a .tar or .raw file. This must be called with the 'more' flag enabled. It will immediately return the numeric ID of the transfer, and then follow up with progress and log message updates, until the transfer is complete."), + &vl_method_Pull, + VARLINK_SYMBOL_COMMENT("A transfer for the specified file is already ongoing"), + &vl_error_AlreadyInProgress, + VARLINK_SYMBOL_COMMENT("The transfer has been cancelled on user request"), + &vl_error_TransferCancelled, + VARLINK_SYMBOL_COMMENT("The transfer failed"), + &vl_error_TransferFailed, + VARLINK_SYMBOL_COMMENT("No currently ongoing transfer"), + &vl_error_NoTransfers); diff --git a/src/shared/varlink-io.systemd.Import.h b/src/shared/varlink-io.systemd.Import.h new file mode 100644 index 0000000000..6579c29b39 --- /dev/null +++ b/src/shared/varlink-io.systemd.Import.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include "varlink-idl.h" + +extern const VarlinkInterface vl_interface_io_systemd_Import; diff --git a/src/test/test-varlink-idl.c b/src/test/test-varlink-idl.c index 92a714b5bd..c2d385eb56 100644 --- a/src/test/test-varlink-idl.c +++ b/src/test/test-varlink-idl.c @@ -10,6 +10,7 @@ #include "varlink-io.systemd.h" #include "varlink-io.systemd.BootControl.h" #include "varlink-io.systemd.Credentials.h" +#include "varlink-io.systemd.Import.h" #include "varlink-io.systemd.Journal.h" #include "varlink-io.systemd.ManagedOOM.h" #include "varlink-io.systemd.MountFileSystem.h" @@ -182,6 +183,8 @@ TEST(parse_format) { print_separator(); test_parse_format_one(&vl_interface_io_systemd_BootControl); print_separator(); + test_parse_format_one(&vl_interface_io_systemd_Import); + print_separator(); test_parse_format_one(&vl_interface_xyz_test); } From f59665881191b7439bdbb5c7795f5ab6f2df1506 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 8 May 2024 11:11:43 +0200 Subject: [PATCH 4/7] importd: allow activation in early boot, and make it socket activatable Previously, importd was only accessible via D-Bus, which required it to be a late boot service. Now that we have Varlink we can rearrange things to become early-boot activated, just after the image directories are mounted. This will later allow us to have generator that auto-downloads images on boot. --- src/import/importd.c | 2 +- units/meson.build | 5 +++++ units/systemd-importd.service.in | 8 +++++++- units/systemd-importd.socket | 24 ++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 units/systemd-importd.socket diff --git a/src/import/importd.c b/src/import/importd.c index 5ef89ac884..112180c846 100644 --- a/src/import/importd.c +++ b/src/import/importd.c @@ -1743,7 +1743,7 @@ static int manager_connect_bus(Manager *m) { assert(m->event); assert(!m->bus); - r = sd_bus_default_system(&m->bus); + r = bus_open_system_watch_bind(&m->bus); if (r < 0) return log_error_errno(r, "Failed to get system bus connection: %m"); diff --git a/units/meson.build b/units/meson.build index b231341a1f..bdc34e6f2c 100644 --- a/units/meson.build +++ b/units/meson.build @@ -361,6 +361,11 @@ units = [ 'conditions' : ['ENABLE_IMPORTD'], 'symlinks' : ['dbus-org.freedesktop.import1.service'], }, + { + 'file' : 'systemd-importd.socket', + 'conditions' : ['ENABLE_IMPORTD'], + 'symlinks' : ['sockets.target.wants/'], + }, { 'file' : 'systemd-initctl.service.in', 'conditions' : ['HAVE_SYSV_COMPAT'], diff --git a/units/systemd-importd.service.in b/units/systemd-importd.service.in index daa93776e1..e119d40929 100644 --- a/units/systemd-importd.service.in +++ b/units/systemd-importd.service.in @@ -8,9 +8,15 @@ # (at your option) any later version. [Unit] -Description=Virtual Machine and Container Download Service +Description=Disk Image Download Service Documentation=man:systemd-importd.service(8) Documentation=man:org.freedesktop.import1(5) +DefaultDependencies=no +After=systemd-importd.socket +WantsMountsFor=/var/lib/machines /var/lib/portables /var/lib/extensions /var/lib/confexts +After=systemd-remount-fs.service +Before=shutdown.target +Conflicts=shutdown.target [Service] Type=notify diff --git a/units/systemd-importd.socket b/units/systemd-importd.socket new file mode 100644 index 0000000000..d1c9c22a4c --- /dev/null +++ b/units/systemd-importd.socket @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +# +# This file is part of systemd. +# +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=Disk Image Download Service Socket +Documentation=man:systemd-importd.service(8) +Documentation=man:org.freedesktop.import1(5) +DefaultDependencies=no +WantsMountsFor=/var/lib/machines /var/lib/portables /var/lib/extensions /var/lib/confexts +After=systemd-remount-fs.service +Before=sockets.target +Conflicts=shutdown.target +Before=shutdown.target + +[Socket] +ListenStream=/run/systemd/io.systemd.Import +FileDescriptorName=varlink +SocketMode=0666 From 5f87b035fad1682a32806d5b3597e9f742a5d286 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Tue, 25 Jun 2024 09:55:16 +0200 Subject: [PATCH 5/7] import: add generator that synthesizes download jobs from kernel cmdline --- man/rules/meson.build | 1 + man/systemd-import-generator.xml | 194 +++++++++++++++++++ man/systemd.system-credentials.xml | 10 + src/import/import-generator.c | 288 +++++++++++++++++++++++++++++ src/import/meson.build | 5 + 5 files changed, 498 insertions(+) create mode 100644 man/systemd-import-generator.xml create mode 100644 src/import/import-generator.c diff --git a/man/rules/meson.build b/man/rules/meson.build index 9b8a29c564..fda14d55bd 100644 --- a/man/rules/meson.build +++ b/man/rules/meson.build @@ -953,6 +953,7 @@ manpages = [ ['systemd-hostnamed.service', '8', ['systemd-hostnamed'], 'ENABLE_HOSTNAMED'], ['systemd-hwdb', '8', [], 'ENABLE_HWDB'], ['systemd-id128', '1', [], ''], + ['systemd-import-generator', '8', [], ''], ['systemd-importd.service', '8', ['systemd-importd'], 'ENABLE_IMPORTD'], ['systemd-inhibit', '1', [], ''], ['systemd-initctl.service', diff --git a/man/systemd-import-generator.xml b/man/systemd-import-generator.xml new file mode 100644 index 0000000000..108509d7d4 --- /dev/null +++ b/man/systemd-import-generator.xml @@ -0,0 +1,194 @@ + + + +%entities; +]> + + + + + systemd-import-generator + systemd + + + + systemd-import-generator + 8 + + + + systemd-import-generator + Generator for automatically downloading disk images at boot + + + + /usr/lib/systemd/system-generators/systemd-import-generator + + + + Description + + systemd-import-generator may be used to automatically download disk images + (tarballs or DDIs) via + systemd-importd.service8 + at boot, based on parameters on the kernel command line or via system credentials. This is useful for + automatically deploying an + systemd-confext8, + systemd-sysext8, + systemd-nspawn1/ + systemd-vmspawn1 or + systemd-portabled.service8 + image at boot. This provides functionality equivalent to + importctl1, but + accessible via the kernel command line and system credentials. + + systemd-import-generator implements + systemd.generator7. + + + + Kernel Command Line + + systemd-import-generator understands the following + kernel-command-line7 + parameters: + + + + systemd.pull= + + This option takes a colon separate triplet of option string, local target image name + and remote URL. The local target image name can be specified as an empty string, in which case the + name is derived from the specified remote URL. The remote URL must using the + http://, https://, file:// schemes. The + option string itself is a comma separated list of options: + + + + rw + ro + + Controls whether to mark the local image as read-only. If not + specified read-only defaults to off. + + + + + + verify= + + Controls whether to cryptographically validate the download before installing it + in place. Takes one of no, checksum or + signature (the latter being the default if not specified). For details see the + of + importctl1 + + + + + + sysext + confext + machine + portable + + Controls the image class to download, and thus ultimately the target directory + for the image, depending on this choice the target directory + /var/lib/extensions/, /var/lib/confexts/, + /var/lib/machines/ or /var/lib/portables/ is + selected. + + Specification of exactly one of these options is mandatory. + + + + + + tar + raw + + Controls the type of resource to download, i.e. a (possibly compressed) tarball + that needs to be unpacked into a file system tree, or (possibly compressed) raw disk image (DDI). + + Specification of exactly one of these options is mandatory. + + + + + + + + + + systemd.pull.success_action= + systemd.pull.failure_action= + + Controls whether to execute an action such as reboot, power-off and similar after + completing the download successfully, or unsuccessfully. See + SuccessAction=/FailureAction= on + systemd.unit5 for + details about the available actions. If not specified no action is taken, and the system will + continue to boot normally. + + + + + + + + Credentials + + systemd-import-generator supports the system credentials logic. The following + credentials are used when passed in: + + + + import.pull + + This credential should be a text file, with each line referencing one download + operation. Each line should follow the same format as the value of the + systemd.pull= kernel command line option described above. + + + + + + + + Examples + + + Download Configuration Extension + + systemd.pull=raw,confext::https://example.com/myconfext.raw.gz + + With a kernel command line option like the above a configuration extension DDI is downloaded + automatically at boot from the specified URL, validated cryptographically, uncompressed and installed. + + + + Download System Extension (Without Validation) + + systemd.pull=tar,sysext,verify=no::https://example.com/mysysext.tar.gz + + With a kernel command line option like the above a system extension tarball is downloaded + automatically at boot from the specified URL, uncompressed and installed – without any cryptographic + validation. This is useful for development purposes in virtual machines and containers. Warning: do not + deploy a system with validation disabled like this! + + + + + See Also + + systemd1 + systemd-importd.service8 + kernel-command-line7 + systemd.system-credentials7 + importctl1 + + + diff --git a/man/systemd.system-credentials.xml b/man/systemd.system-credentials.xml index d9fbae25ee..f8c27d04ac 100644 --- a/man/systemd.system-credentials.xml +++ b/man/systemd.system-credentials.xml @@ -415,6 +415,16 @@ + + + import.pull + + Specified disk images (tarballs and DDIs) to automatically download and install at boot. For details see + systemd-import-generator8. + + + + diff --git a/src/import/import-generator.c b/src/import/import-generator.c new file mode 100644 index 0000000000..29fa5aad32 --- /dev/null +++ b/src/import/import-generator.c @@ -0,0 +1,288 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include "sd-json.h" + +#include "creds-util.h" +#include "discover-image.h" +#include "fd-util.h" +#include "fileio.h" +#include "generator.h" +#include "import-util.h" +#include "json-util.h" +#include "proc-cmdline.h" +#include "specifier.h" +#include "web-util.h" + +static const char *arg_dest = NULL; +static char *arg_success_action = NULL; +static char *arg_failure_action = NULL; +static sd_json_variant **arg_transfers = NULL; +static size_t arg_n_transfers = 0; + +STATIC_DESTRUCTOR_REGISTER(arg_success_action, freep); +STATIC_DESTRUCTOR_REGISTER(arg_failure_action, freep); +STATIC_ARRAY_DESTRUCTOR_REGISTER(arg_transfers, arg_n_transfers, sd_json_variant_unref_many); + +static int parse_pull_expression(const char *v) { + const char *p = v; + int r; + + assert(v); + + _cleanup_free_ char *options = NULL; + r = extract_first_word(&p, &options, ":", EXTRACT_DONT_COALESCE_SEPARATORS); + if (r < 0) + return log_error_errno(r, "Failed to extract option string from pull expression '%s': %m", v); + if (r == 0) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No option string in pull expression '%s': %m", v); + + _cleanup_free_ char *local = NULL; + r = extract_first_word(&p, &local, ":", EXTRACT_DONT_COALESCE_SEPARATORS); + if (r < 0) + return log_error_errno(r, "Failed to extract local name from pull expression '%s': %m", v); + if (r == 0) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No local string in pull expression '%s': %m", v); + + if (!http_url_is_valid(p) && !file_url_is_valid(p)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Not a valid URL, refusing: %s", p); + _cleanup_free_ char *remote = strdup(p); + if (!remote) + return log_oom(); + + if (isempty(local)) + local = mfree(local); + else if (!image_name_is_valid(local)) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Not a valid image name, refusing: %s", local); + + ImportType type = _IMPORT_TYPE_INVALID; + ImageClass class = _IMAGE_CLASS_INVALID; + ImportVerify verify = IMPORT_VERIFY_SIGNATURE; + bool ro = false; + + const char *o = options; + for (;;) { + _cleanup_free_ char *opt = NULL; + + r = extract_first_word(&o, &opt, ",", EXTRACT_DONT_COALESCE_SEPARATORS); + if (r < 0) + return log_error_errno(r, "Failed to extract option from pull option expression '%s': %m", options); + if (r == 0) + break; + + const char *suffix; + + if (streq(opt, "ro")) + ro = true; + else if (streq(opt, "rw")) + ro = false; + else if ((suffix = startswith(opt, "verify="))) { + + ImportVerify w = import_verify_from_string(suffix); + + if (w < 0) + log_warning_errno(w, "Unknown verification mode, ignoring: %s", suffix); + else + verify = w; + } else { + ImageClass c; + + c = image_class_from_string(opt); + if (c < 0) { + ImportType t; + + t = import_type_from_string(opt); + if (t < 0) + log_warning_errno(c, "Unknown pull option, ignoring: %s", opt); + else + type = t; + } else + class = c; + } + } + + if (type < 0) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No image type (raw, tar) specified in pull expression, refusing: %s", v); + if (class < 0) + return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No image class (machine, portable, sysext, confext) specified in pull expression, refusing: %s", v); + + if (!GREEDY_REALLOC(arg_transfers, arg_n_transfers + 1)) + return log_oom(); + + _cleanup_(sd_json_variant_unrefp) sd_json_variant *j = NULL; + + r = sd_json_buildo( + &j, + SD_JSON_BUILD_PAIR("remote", SD_JSON_BUILD_STRING(remote)), + SD_JSON_BUILD_PAIR_CONDITION(!!local, "local", SD_JSON_BUILD_STRING(local)), + SD_JSON_BUILD_PAIR("class", JSON_BUILD_STRING_UNDERSCORIFY(image_class_to_string(class))), + SD_JSON_BUILD_PAIR("type", JSON_BUILD_STRING_UNDERSCORIFY(import_type_to_string(type))), + SD_JSON_BUILD_PAIR("readOnly", SD_JSON_BUILD_BOOLEAN(ro)), + SD_JSON_BUILD_PAIR("verify", JSON_BUILD_STRING_UNDERSCORIFY(import_verify_to_string(verify)))); + if (r < 0) + return log_error_errno(r, "Failed to build import JSON object: %m"); + + arg_transfers[arg_n_transfers++] = TAKE_PTR(j); + return 0; +} + +static int parse_proc_cmdline_item(const char *key, const char *value, void *data) { + int r; + + if (proc_cmdline_key_streq(key, "systemd.pull")) { + + if (proc_cmdline_value_missing(key, value)) + return 0; + + r = parse_pull_expression(value); + if (r < 0) + log_warning_errno(r, "Failed to parse %s expression, ignoring: %s", key, value); + + } else if (proc_cmdline_key_streq(key, "systemd.pull.success_action")) { + + if (proc_cmdline_value_missing(key, value)) + return 0; + + return free_and_strdup_warn(&arg_success_action, value); + + } else if (proc_cmdline_key_streq(key, "systemd.pull.failure_action")) { + + if (proc_cmdline_value_missing(key, value)) + return 0; + + return free_and_strdup_warn(&arg_failure_action, value); + } + + return 0; +} + +static int parse_credentials(void) { + _cleanup_free_ char *b = NULL; + size_t sz = 0; + int r; + + r = read_credential_with_decryption("import.pull", (void**) &b, &sz); + if (r <= 0) + return r; + + _cleanup_fclose_ FILE *f = NULL; + f = fmemopen_unlocked(b, sz, "r"); + if (!f) + return log_oom(); + + for (;;) { + _cleanup_free_ char *item = NULL; + + r = read_stripped_line(f, LINE_MAX, &item); + if (r == 0) + break; + if (r < 0) { + log_error_errno(r, "Failed to parse credential 'ssh.listen': %m"); + break; + } + + if (startswith(item, "#")) + continue; + + r = parse_pull_expression(item); + if (r < 0) + log_warning_errno(r, "Failed to parse expression, ignoring: %s", item); + } + + return 0; +} + +static int transfer_generate(sd_json_variant *v, size_t c) { + int r; + + assert(v); + + _cleanup_free_ char *service = NULL; + if (asprintf(&service, "import%zu.service", c) < 0) + return log_oom(); + + _cleanup_fclose_ FILE *f = NULL; + r = generator_open_unit_file(arg_dest, /* source = */ NULL, service, &f); + if (r < 0) + return r; + + const char *remote = sd_json_variant_string(sd_json_variant_by_key(v, "remote")); + + fprintf(f, + "[Unit]\n" + "Description=Download of %s\n" + "Documentation=man:systemd-import-generator(8)\n" + "SourcePath=/proc/cmdline\n" + "Requires=systemd-importd.socket\n" + "After=systemd-importd.socket\n" + "Conflicts=shutdown.target\n" + "Before=shutdown.target\n" + "DefaultDependencies=no\n", + remote); + + if (arg_success_action) + fprintf(f, "SuccessAction=%s\n", + arg_success_action); + + if (arg_failure_action) + fprintf(f, "FailureAction=%s\n", + arg_failure_action); + + const char *class = sd_json_variant_string(sd_json_variant_by_key(v, "class")); + if (streq_ptr(class, "sysext")) + fputs("Before=systemd-sysext.service\n", f); + else if (streq_ptr(class, "confext")) + fputs("Before=systemd-confext.service\n", f); + + /* Assume network resource unless URL is file:// */ + if (!file_url_is_valid(remote)) + fputs("Wants=network-online.target\n" + "After=network-online.target\n", f); + + fputs("\n" + "[Service]\n" + "Type=oneshot\n", f); + + _cleanup_free_ char *formatted = NULL; + r = sd_json_variant_format(v, /* flags= */ 0, &formatted); + if (r < 0) + return log_error_errno(r, "Failed to format import JSON data: %m"); + + _cleanup_free_ char *escaped = specifier_escape(formatted); + if (!escaped) + return log_oom(); + + fprintf(f, "ExecStart=:varlinkctl call -q --more /run/systemd/io.systemd.Import io.systemd.Import.Pull '%s'\n", + escaped); + + r = fflush_and_check(f); + if (r < 0) + return log_error_errno(r, "Failed to write unit %s: %m", service); + + return generator_add_symlink(arg_dest, "multi-user.target", "wants", service); +} + +static int generate(void) { + size_t c = 0; + int r = 0; + + FOREACH_ARRAY(i, arg_transfers, arg_n_transfers) + RET_GATHER(r, transfer_generate(*i, c++)); + + return r; +} + +static int run(const char *dest, const char *dest_early, const char *dest_late) { + int r; + + assert_se(arg_dest = dest); + + r = proc_cmdline_parse(parse_proc_cmdline_item, NULL, PROC_CMDLINE_RD_STRICT|PROC_CMDLINE_STRIP_RD_PREFIX); + if (r < 0) + log_warning_errno(r, "Failed to parse kernel command line, ignoring: %m"); + + (void) parse_credentials(); + + return generate(); +} + +DEFINE_MAIN_GENERATOR_FUNCTION(run); diff --git a/src/import/meson.build b/src/import/meson.build index 184dd7bbf2..45500edb43 100644 --- a/src/import/meson.build +++ b/src/import/meson.build @@ -110,6 +110,11 @@ executables += [ 'conditions' : ['ENABLE_IMPORTD'], 'sources' : files('importctl.c'), }, + generator_template + { + 'name' : 'systemd-import-generator', + 'sources' : files('import-generator.c'), + 'conditions' : ['ENABLE_IMPORTD'], + }, test_template + { 'sources' : files( 'test-qcow2.c', From d7afdf36ae877dc68e9fdc7f2d337fd00fb5576e Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Mon, 27 May 2024 19:05:48 +0200 Subject: [PATCH 6/7] ci: add test for importd varlink interface + import generator --- test/units/TEST-13-NSPAWN.importctl.sh | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/test/units/TEST-13-NSPAWN.importctl.sh b/test/units/TEST-13-NSPAWN.importctl.sh index a13e3fd1fd..be31837447 100755 --- a/test/units/TEST-13-NSPAWN.importctl.sh +++ b/test/units/TEST-13-NSPAWN.importctl.sh @@ -9,10 +9,13 @@ set -o pipefail export PAGER= +TEST_CMDLINE="/tmp/proc-cmdline.$RANDOM" + at_exit() { set +e umount -l -R /var/lib/confexts - rm -f /var/tmp/importtest /var/tmp/importtest2 /var/tmp/importtest.tar.gz /var/tmp/importtest2.tar.gz + rm -f /var/tmp/importtest /var/tmp/importtest2 /var/tmp/importtest.tar.gz /var/tmp/importtest2.tar.gz "$TEST_CMDLINE" + mountpoint -q /proc/cmdline && umount /proc/cmdline } trap at_exit EXIT @@ -64,3 +67,19 @@ cmp /var/tmp/importtest /var/lib/confexts/importtest7/importtest importctl list-images importctl list-images -j + +varlinkctl call --more /run/systemd/io.systemd.Import io.systemd.Import.ListTransfers '{}' --graceful=io.systemd.Import.NoTransfers + +varlinkctl call --more /run/systemd/io.systemd.Import io.systemd.Import.Pull '{"class":"confext","remote":"file:///var/tmp/importtest.tar.gz","local":"importtest8","type":"tar","verify":"no"}' +cmp /var/tmp/importtest /var/lib/confexts/importtest8/importtest + +echo -n "systemd.pull=tar,confext,verify=no:importtest9:file:///var/tmp/importtest.tar.gz " > "$TEST_CMDLINE" +cat /proc/cmdline >> "$TEST_CMDLINE" +mount --bind "$TEST_CMDLINE" /proc/cmdline + +cat /proc/cmdline + +systemctl daemon-reload + +systemctl start import0.service +cmp /var/tmp/importtest /var/lib/confexts/importtest9/importtest From eac0cbed8ce27615a44b888232fc10b4cd5926d5 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Sat, 11 May 2024 17:02:26 +0200 Subject: [PATCH 7/7] update TODO --- TODO | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/TODO b/TODO index 76f86b8000..2ee21e7357 100644 --- a/TODO +++ b/TODO @@ -248,9 +248,8 @@ Features: from, tracing through overlayfs, DM, loopback block device. * importd/importctl - - import generator - port tar handling to libarchive - - add varlink interface + - complete varlink interface - download images into .v/ dirs * in os-release define a field that can be initialized at build time from