From 15d43e30650529f27e450f703b82671aa67c2ca7 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Tue, 11 Aug 2020 14:50:36 +0200 Subject: [PATCH 01/20] repart: don't unload data we configured explicitly, and fully free all data we match to disk The context_unload_partition_table() call is supposed to remove all data from the loaded partitions about how we mapped it to existing partitions on disk, but it should leave everything we parsed from the definition files in place. We mostly got this right, except for two cases: 1. new_uuid is parsed from the definition files and should stay 2. current_label is read from the existing partition table and should be freed --- src/partition/repart.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/partition/repart.c b/src/partition/repart.c index 572db3bdc4..4768600a8d 100644 --- a/src/partition/repart.c +++ b/src/partition/repart.c @@ -1563,7 +1563,8 @@ static void context_unload_partition_table(Context *context) { p->padding_area = NULL; p->allocated_to_area = NULL; - p->current_uuid = p->new_uuid = SD_ID128_NULL; + p->current_uuid = SD_ID128_NULL; + p->current_label = mfree(p->current_label); } context->start = UINT64_MAX; From 0de3c4f4b7b47481fb78e68f6110a167b27b43c9 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Mon, 3 Aug 2020 12:05:37 +0200 Subject: [PATCH 02/20] mkdir: add new mkdir_p_root() helper --- src/basic/mkdir.c | 52 +++++++++++++++++++++++++++++++++++++++++++++++ src/basic/mkdir.h | 2 ++ 2 files changed, 54 insertions(+) diff --git a/src/basic/mkdir.c b/src/basic/mkdir.c index d591e65e41..eea8f1c9aa 100644 --- a/src/basic/mkdir.c +++ b/src/basic/mkdir.c @@ -5,6 +5,7 @@ #include #include "alloc-util.h" +#include "fd-util.h" #include "format-util.h" #include "fs-util.h" #include "macro.h" @@ -187,3 +188,54 @@ int mkdir_p(const char *path, mode_t mode) { int mkdir_p_safe(const char *prefix, const char *path, mode_t mode, uid_t uid, gid_t gid, MkdirFlags flags) { return mkdir_p_internal(prefix, path, mode, uid, gid, flags, mkdir_errno_wrapper); } + +int mkdir_p_root(const char *root, const char *p, uid_t uid, gid_t gid, mode_t m) { + _cleanup_free_ char *pp = NULL; + _cleanup_close_ int dfd = -1; + const char *bn; + int r; + + pp = dirname_malloc(p); + if (!pp) + return -ENOMEM; + + /* Not top-level? */ + if (!(path_equal(pp, "/") || isempty(pp) || path_equal(pp, "."))) { + + /* Recurse up */ + r = mkdir_p_root(root, pp, uid, gid, m); + if (r < 0) + return r; + } + + bn = basename(p); + if (path_equal(bn, "/") || isempty(bn) || path_equal(bn, ".")) + return 0; + + if (!filename_is_valid(bn)) + return -EINVAL; + + dfd = chase_symlinks_and_open(pp, root, CHASE_PREFIX_ROOT, O_RDONLY|O_CLOEXEC|O_DIRECTORY, NULL); + if (dfd < 0) + return dfd; + + if (mkdirat(dfd, bn, m) < 0) { + if (errno == EEXIST) + return 0; + + return -errno; + } + + if (uid_is_valid(uid) || gid_is_valid(gid)) { + _cleanup_close_ int nfd = -1; + + nfd = openat(dfd, bn, O_RDONLY|O_CLOEXEC|O_DIRECTORY); + if (nfd < 0) + return -errno; + + if (fchown(nfd, uid, gid) < 0) + return -errno; + } + + return 1; +} diff --git a/src/basic/mkdir.h b/src/basic/mkdir.h index 8bfaaf405b..fc66a7d365 100644 --- a/src/basic/mkdir.h +++ b/src/basic/mkdir.h @@ -26,3 +26,5 @@ typedef int (*mkdir_func_t)(const char *pathname, mode_t mode); int mkdir_safe_internal(const char *path, mode_t mode, uid_t uid, gid_t gid, MkdirFlags flags, mkdir_func_t _mkdir); int mkdir_parents_internal(const char *prefix, const char *path, mode_t mode, uid_t uid, gid_t gid, MkdirFlags flags, mkdir_func_t _mkdir); int mkdir_p_internal(const char *prefix, const char *path, mode_t mode, uid_t uid, gid_t gid, MkdirFlags flags, mkdir_func_t _mkdir); + +int mkdir_p_root(const char *root, const char *p, uid_t uid, gid_t gid, mode_t m); From 8dbc208cc1a7bbd9af44d98df4a4eb472266699a Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 30 Jul 2020 18:48:52 +0200 Subject: [PATCH 03/20] loop-util: define API for syncing loopback device --- src/shared/loop-util.c | 15 +++++++++++++++ src/shared/loop-util.h | 1 + 2 files changed, 16 insertions(+) diff --git a/src/shared/loop-util.c b/src/shared/loop-util.c index 8db2fed66f..fb88f51849 100644 --- a/src/shared/loop-util.c +++ b/src/shared/loop-util.c @@ -472,3 +472,18 @@ int loop_device_flock(LoopDevice *d, int operation) { return 0; } + +int loop_device_sync(LoopDevice *d) { + assert(d); + + /* We also do this implicitly in loop_device_unref(). Doing this explicitly here has the benefit that + * we can check the return value though. */ + + if (d->fd < 0) + return -EBADF; + + if (fsync(d->fd) < 0) + return -errno; + + return 0; +} diff --git a/src/shared/loop-util.h b/src/shared/loop-util.h index 73f82a6d30..16f215b059 100644 --- a/src/shared/loop-util.h +++ b/src/shared/loop-util.h @@ -26,3 +26,4 @@ void loop_device_relinquish(LoopDevice *d); int loop_device_refresh_size(LoopDevice *d, uint64_t offset, uint64_t size); int loop_device_flock(LoopDevice *d, int operation); +int loop_device_sync(LoopDevice *d); From c95f9a2351ffd63b53a4c33fb683606e43677c89 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 29 Jul 2020 18:36:26 +0200 Subject: [PATCH 04/20] shared: introduce mkfs-util.c/.h Let's move the "mkfs" code from homed there, plus other related code. This way we can easily reuse it from other places. --- src/basic/path-util.c | 12 ----- src/basic/path-util.h | 1 - src/home/homework-luks.c | 68 +---------------------- src/shared/meson.build | 2 + src/shared/mkfs-util.c | 114 +++++++++++++++++++++++++++++++++++++++ src/shared/mkfs-util.h | 10 ++++ 6 files changed, 128 insertions(+), 79 deletions(-) create mode 100644 src/shared/mkfs-util.c create mode 100644 src/shared/mkfs-util.h diff --git a/src/basic/path-util.c b/src/basic/path-util.c index c4e022b3a1..c463ae23ab 100644 --- a/src/basic/path-util.c +++ b/src/basic/path-util.c @@ -726,18 +726,6 @@ int fsck_exists(const char *fstype) { return binary_is_good(checker); } -int mkfs_exists(const char *fstype) { - const char *mkfs; - - assert(fstype); - - if (streq(fstype, "auto")) - return -EINVAL; - - mkfs = strjoina("mkfs.", fstype); - return binary_is_good(mkfs); -} - int parse_path_argument_and_warn(const char *path, bool suppress_root, char **arg) { char *p; int r; diff --git a/src/basic/path-util.h b/src/basic/path-util.h index 30031fca8e..99c4e35782 100644 --- a/src/basic/path-util.h +++ b/src/basic/path-util.h @@ -85,7 +85,6 @@ int find_binary(const char *name, char **filename); bool paths_check_timestamp(const char* const* paths, usec_t *paths_ts_usec, bool update); int fsck_exists(const char *fstype); -int mkfs_exists(const char *fstype); /* Iterates through the path prefixes of the specified path, going up * the tree, to root. Also returns "" (and not "/"!) for the root diff --git a/src/home/homework-luks.c b/src/home/homework-luks.c index 1fcf17806e..1fe0465cfb 100644 --- a/src/home/homework-luks.c +++ b/src/home/homework-luks.c @@ -24,6 +24,7 @@ #include "memory-util.h" #include "missing_magic.h" #include "mkdir.h" +#include "mkfs-util.h" #include "mount-util.h" #include "openssl-util.h" #include "parse-util.h" @@ -1371,71 +1372,6 @@ int home_trim_luks(UserRecord *h) { return 0; } -static int run_mkfs( - const char *node, - const char *fstype, - const char *label, - sd_id128_t uuid, - bool discard) { - - int r; - - assert(node); - assert(fstype); - assert(label); - - r = mkfs_exists(fstype); - if (r < 0) - return log_error_errno(r, "Failed to check if mkfs for file system %s exists: %m", fstype); - if (r == 0) - return log_error_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT), "No mkfs for file system %s installed.", fstype); - - r = safe_fork("(mkfs)", FORK_RESET_SIGNALS|FORK_RLIMIT_NOFILE_SAFE|FORK_DEATHSIG|FORK_LOG|FORK_WAIT|FORK_STDOUT_TO_STDERR, NULL); - if (r < 0) - return r; - if (r == 0) { - const char *mkfs; - char suuid[37]; - - /* Child */ - - mkfs = strjoina("mkfs.", fstype); - id128_to_uuid_string(uuid, suuid); - - if (streq(fstype, "ext4")) - execlp(mkfs, mkfs, - "-L", label, - "-U", suuid, - "-I", "256", - "-O", "has_journal", - "-m", "0", - "-E", discard ? "lazy_itable_init=1,discard" : "lazy_itable_init=1,nodiscard", - node, NULL); - else if (streq(fstype, "btrfs")) { - if (discard) - execlp(mkfs, mkfs, "-L", label, "-U", suuid, node, NULL); - else - execlp(mkfs, mkfs, "-L", label, "-U", suuid, "--nodiscard", node, NULL); - } else if (streq(fstype, "xfs")) { - const char *j; - - j = strjoina("uuid=", suuid); - if (discard) - execlp(mkfs, mkfs, "-L", label, "-m", j, "-m", "reflink=1", node, NULL); - else - execlp(mkfs, mkfs, "-L", label, "-m", j, "-m", "reflink=1", "-K", node, NULL); - } else { - log_error("Cannot make file system: %s", fstype); - _exit(EXIT_FAILURE); - } - - log_error_errno(errno, "Failed to execute %s: %m", mkfs); - _exit(EXIT_FAILURE); - } - - return 0; -} - static struct crypt_pbkdf_type* build_good_pbkdf(struct crypt_pbkdf_type *buffer, UserRecord *hr) { assert(buffer); assert(hr); @@ -2083,7 +2019,7 @@ int home_create_luks( log_info("Setting up LUKS device %s completed.", dm_node); - r = run_mkfs(dm_node, fstype, user_record_user_name_and_realm(h), fs_uuid, user_record_luks_discard(h)); + r = make_filesystem(dm_node, fstype, user_record_user_name_and_realm(h), fs_uuid, user_record_luks_discard(h)); if (r < 0) goto fail; diff --git a/src/shared/meson.build b/src/shared/meson.build index d192a4d8d7..39bf5523d6 100644 --- a/src/shared/meson.build +++ b/src/shared/meson.build @@ -166,6 +166,8 @@ shared_sources = files(''' macvlan-util.c macvlan-util.h main-func.h + mkfs-util.c + mkfs-util.h module-util.h mount-util.c mount-util.h diff --git a/src/shared/mkfs-util.c b/src/shared/mkfs-util.c new file mode 100644 index 0000000000..a78f68e398 --- /dev/null +++ b/src/shared/mkfs-util.c @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ + +#include "id128-util.h" +#include "mkfs-util.h" +#include "path-util.h" +#include "process-util.h" +#include "string-util.h" + +int mkfs_exists(const char *fstype) { + const char *mkfs; + int r; + + assert(fstype); + + if (STR_IN_SET(fstype, "auto", "swap")) /* these aren't real file system types, refuse early */ + return -EINVAL; + + mkfs = strjoina("mkfs.", fstype); + if (!filename_is_valid(mkfs)) /* refuse file system types with slashes and similar */ + return -EINVAL; + + r = find_binary(mkfs, NULL); + if (r == -ENOENT) + return false; + if (r < 0) + return r; + + return true; +} + +int make_filesystem( + const char *node, + const char *fstype, + const char *label, + sd_id128_t uuid, + bool discard) { + + _cleanup_free_ char *mkfs = NULL; + int r; + + assert(node); + assert(fstype); + assert(label); + + if (streq(fstype, "swap")) { + r = find_binary("mkswap", &mkfs); + if (r == -ENOENT) + return log_error_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT), "mkswap binary not available."); + if (r < 0) + return log_error_errno(r, "Failed to determine whether mkswap binary exists: %m"); + } else { + r = mkfs_exists(fstype); + if (r < 0) + return log_error_errno(r, "Failed to determine whether mkfs binary for %s exists: %m", fstype); + if (r == 0) + return log_error_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT), "mkfs binary for %s is not available.", fstype); + + mkfs = strjoin("mkfs.", fstype); + if (!mkfs) + return log_oom(); + } + + r = safe_fork("(mkfs)", FORK_RESET_SIGNALS|FORK_RLIMIT_NOFILE_SAFE|FORK_DEATHSIG|FORK_LOG|FORK_WAIT|FORK_STDOUT_TO_STDERR, NULL); + if (r < 0) + return r; + if (r == 0) { + char suuid[ID128_UUID_STRING_MAX]; + + /* Child */ + id128_to_uuid_string(uuid, suuid); + + if (streq(fstype, "ext4")) + (void) execlp(mkfs, mkfs, + "-L", label, + "-U", suuid, + "-I", "256", + "-O", "has_journal", + "-m", "0", + "-E", discard ? "lazy_itable_init=1,discard" : "lazy_itable_init=1,nodiscard", + node, NULL); + + else if (streq(fstype, "btrfs")) { + if (discard) + (void) execlp(mkfs, mkfs, "-L", label, "-U", suuid, node, NULL); + else + (void) execlp(mkfs, mkfs, "-L", label, "-U", suuid, "--nodiscard", node, NULL); + + } else if (streq(fstype, "xfs")) { + const char *j; + + j = strjoina("uuid=", suuid); + if (discard) + (void) execlp(mkfs, mkfs, "-L", label, "-m", j, "-m", "reflink=1", node, NULL); + else + (void) execlp(mkfs, mkfs, "-L", label, "-m", j, "-m", "reflink=1", "-K", node, NULL); + + } else if (streq(fstype, "swap")) { + + (void) execlp(mkfs, mkfs, + "-L", label, + "-U", suuid, + node, NULL); + + } else + /* Generic fallback for all other file systems */ + (void) execlp(mkfs, mkfs, node, NULL); + + log_error_errno(errno, "Failed to execute %s: %m", mkfs); + + _exit(EXIT_FAILURE); + } + + return 0; +} diff --git a/src/shared/mkfs-util.h b/src/shared/mkfs-util.h new file mode 100644 index 0000000000..54e9b931e6 --- /dev/null +++ b/src/shared/mkfs-util.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +#pragma once + +#include + +#include "sd-id128.h" + +int mkfs_exists(const char *fstype); + +int make_filesystem(const char *node, const char *fstype, const char *label, sd_id128_t uuid, bool discard); From da49710ef25887491861405c5bda9ee38946746c Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 29 Jul 2020 19:10:33 +0200 Subject: [PATCH 05/20] makefs: port to generic make_filesystem() call --- src/partition/makefs.c | 41 ++++++++++------------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/src/partition/makefs.c b/src/partition/makefs.c index 97f50c9033..1ce6acd01e 100644 --- a/src/partition/makefs.c +++ b/src/partition/makefs.c @@ -11,40 +11,15 @@ #include "dissect-image.h" #include "fd-util.h" #include "main-func.h" +#include "mkfs-util.h" #include "process-util.h" #include "signal-util.h" #include "string-util.h" -static int makefs(const char *type, const char *device) { - const char *mkfs; - pid_t pid; - int r; - - if (streq(type, "swap")) - mkfs = "/sbin/mkswap"; - else - mkfs = strjoina("/sbin/mkfs.", type); - if (access(mkfs, X_OK) != 0) - return log_error_errno(errno, "%s is not executable: %m", mkfs); - - r = safe_fork("(mkfs)", FORK_RESET_SIGNALS|FORK_DEATHSIG|FORK_RLIMIT_NOFILE_SAFE|FORK_LOG, &pid); - if (r < 0) - return r; - if (r == 0) { - const char *cmdline[3] = { mkfs, device, NULL }; - - /* Child */ - - execv(cmdline[0], (char**) cmdline); - _exit(EXIT_FAILURE); - } - - return wait_for_terminate_and_check(mkfs, pid, WAIT_LOG); -} - static int run(int argc, char *argv[]) { - _cleanup_free_ char *device = NULL, *type = NULL, *detected = NULL; + _cleanup_free_ char *device = NULL, *fstype = NULL, *detected = NULL; _cleanup_close_ int lock_fd = -1; + sd_id128_t uuid; struct stat st; int r; @@ -55,8 +30,8 @@ static int run(int argc, char *argv[]) { "This program expects two arguments."); /* type and device must be copied because makefs calls safe_fork, which clears argv[] */ - type = strdup(argv[1]); - if (!type) + fstype = strdup(argv[1]); + if (!fstype) return log_oom(); device = strdup(argv[2]); @@ -85,7 +60,11 @@ static int run(int argc, char *argv[]) { return 0; } - return makefs(type, device); + r = sd_id128_randomize(&uuid); + if (r < 0) + return log_error_errno(r, "Failed to generate UUID for file system: %m"); + + return make_filesystem(device, fstype, basename(device), uuid, true); } DEFINE_MAIN_FUNCTION(run); From aa2a74ad9dbcecb1e5929d293c16445db0af5551 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Tue, 4 Aug 2020 15:54:25 +0200 Subject: [PATCH 06/20] repart: make error code when operating on non-existing file a bit more useful --- src/partition/repart.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/partition/repart.c b/src/partition/repart.c index 4768600a8d..fc207585b0 100644 --- a/src/partition/repart.c +++ b/src/partition/repart.c @@ -3202,7 +3202,7 @@ static int find_root(char **ret, int *ret_fd) { r = acquire_root_devno(arg_node, O_RDONLY|O_CLOEXEC, ret, ret_fd); if (r < 0) - return log_error_errno(r, "Failed to determine backing device of %s: %m", arg_node); + return log_error_errno(r, "Failed to open file or determine backing device of %s: %m", arg_node); return 0; } From 53171c0453b564fa0bdc59da9e1bd9060575cf59 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 30 Jul 2020 10:09:57 +0200 Subject: [PATCH 07/20] repart: add support for formatting newly created partitions --- src/partition/repart.c | 119 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 109 insertions(+), 10 deletions(-) diff --git a/src/partition/repart.c b/src/partition/repart.c index fc207585b0..f2abb28f25 100644 --- a/src/partition/repart.c +++ b/src/partition/repart.c @@ -36,7 +36,9 @@ #include "json.h" #include "list.h" #include "locale-util.h" +#include "loop-util.h" #include "main-func.h" +#include "mkfs-util.h" #include "parse-util.h" #include "path-util.h" #include "pretty-print.h" @@ -121,6 +123,8 @@ struct Partition { int copy_blocks_fd; uint64_t copy_blocks_size; + char *format; + LIST_FIELDS(Partition, partitions); }; @@ -204,6 +208,8 @@ static Partition* partition_free(Partition *p) { free(p->copy_blocks_path); safe_close(p->copy_blocks_fd); + free(p->format); + return mfree(p); } @@ -1000,6 +1006,30 @@ static int config_parse_size4096( return 0; } +static int config_parse_fstype( + const char *unit, + const char *filename, + unsigned line, + const char *section, + unsigned section_line, + const char *lvalue, + int ltype, + const char *rvalue, + void *data, + void *userdata) { + + char **fstype = data; + + assert(rvalue); + assert(data); + + if (!filename_is_valid(rvalue)) + return log_syntax(unit, LOG_ERR, filename, line, 0, + "File system type is not valid, refusing: %s", rvalue); + + return free_and_strdup_warn(fstype, rvalue); +} + static int partition_read_definition(Partition *p, const char *path) { ConfigTableItem table[] = { @@ -1015,6 +1045,7 @@ static int partition_read_definition(Partition *p, const char *path) { { "Partition", "PaddingMaxBytes", config_parse_size4096, -1, &p->padding_max }, { "Partition", "FactoryReset", config_parse_bool, 0, &p->factory_reset }, { "Partition", "CopyBlocks", config_parse_path, 0, &p->copy_blocks_path }, + { "Partition", "Format", config_parse_fstype, 0, &p->format }, {} }; int r; @@ -1040,6 +1071,10 @@ static int partition_read_definition(Partition *p, const char *path) { return log_syntax(NULL, LOG_ERR, path, 1, SYNTHETIC_ERRNO(EINVAL), "Type= not defined, refusing."); + if (p->copy_blocks_path && p->format) + return log_syntax(NULL, LOG_ERR, path, 1, SYNTHETIC_ERRNO(EINVAL), + "Format= and CopyBlocks= cannot be combined, refusing."); + return 0; } @@ -1191,23 +1226,23 @@ static int fdisk_set_disklabel_id_by_uuid(struct fdisk_context *c, sd_id128_t id return fdisk_set_ask(c, NULL, NULL); } -#define DISK_UUID_TOKEN "disk-uuid" - -static int disk_acquire_uuid(Context *context, sd_id128_t *ret) { +static int derive_uuid(sd_id128_t base, const char *token, sd_id128_t *ret) { union { unsigned char md[SHA256_DIGEST_LENGTH]; sd_id128_t id; } result; - assert(context); + assert(token); assert(ret); - /* Calculate the HMAC-SHA256 of the string "disk-uuid", keyed off the machine ID. We use the machine - * ID as key (and not as cleartext!) since it's the machine ID we don't want to leak. */ + /* Derive a new UUID from the specified UUID in a stable and reasonably safe way. Specifically, we + * calculate the HMAC-SHA256 of the specified token string, keyed by the supplied base (typically the + * machine ID). We use the machine ID as key (and not as cleartext!) of the HMAC operation since it's + * the machine ID we don't want to leak. */ if (!HMAC(EVP_sha256(), - &context->seed, sizeof(context->seed), - (const unsigned char*) DISK_UUID_TOKEN, strlen(DISK_UUID_TOKEN), + &base, sizeof(base), + (const unsigned char*) token, strlen(token), result.md, NULL)) return log_error_errno(SYNTHETIC_ERRNO(ENOTRECOVERABLE), "HMAC-SHA256 calculation failed."); @@ -1312,7 +1347,7 @@ static int context_load_partition_table( if (r < 0) return log_error_errno(r, "Failed to create GPT disk label: %m"); - r = disk_acquire_uuid(context, &disk_uuid); + r = derive_uuid(context->seed, "disk-uuid", &disk_uuid); if (r < 0) return log_error_errno(r, "Failed to acquire disk GPT uuid: %m"); @@ -1332,7 +1367,7 @@ static int context_load_partition_table( return log_error_errno(r, "Failed to parse current GPT disk label UUID: %m"); if (sd_id128_is_null(disk_uuid)) { - r = disk_acquire_uuid(context, &disk_uuid); + r = derive_uuid(context->seed, "disk-uuid", &disk_uuid); if (r < 0) return log_error_errno(r, "Failed to acquire disk GPT uuid: %m"); @@ -2212,6 +2247,66 @@ static int context_copy_blocks(Context *context) { return 0; } +static int context_mkfs(Context *context) { + Partition *p; + int fd = -1, r; + + assert(context); + + /* Make a file system */ + + LIST_FOREACH(partitions, p, context->partitions) { + _cleanup_(loop_device_unrefp) LoopDevice *d = NULL; + sd_id128_t fs_uuid; + + if (p->dropped) + continue; + + if (PARTITION_EXISTS(p)) /* Never format existing partitions */ + continue; + + if (!p->format) + continue; + + assert(p->offset != UINT64_MAX); + assert(p->new_size != UINT64_MAX); + + if (fd < 0) + assert_se((fd = fdisk_get_devfd(context->fdisk_context)) >= 0); + + /* Loopback block devices are not only useful to turn regular files into block devices, but + * also to cut out sections of block devices into new block devices. */ + + r = loop_device_make(fd, O_RDWR, p->offset, p->new_size, 0, &d); + if (r < 0) + return log_error_errno(r, "Failed to make loopback device of partition %" PRIu64 ": %m", p->partno); + + r = loop_device_flock(d, LOCK_EX); + if (r < 0) + return log_error_errno(r, "Failed to lock loopback device: %m"); + + log_info("Formatting future partition %" PRIu64 ".", p->partno); + + /* Calculate the UUID for the file system as HMAC-SHA256 of the string "file-system-uuid", + * keyed off the partition UUID. */ + r = derive_uuid(p->new_uuid, "file-system-uuid", &fs_uuid); + if (r < 0) + return r; + + r = make_filesystem(d->node, p->format, p->new_label, fs_uuid, arg_discard); + if (r < 0) + return r; + + log_info("Successfully formatted future partition %" PRIu64 ".", p->partno); + + r = loop_device_sync(d); + if (r < 0) + return log_error_errno(r, "Failed to sync loopback device: %m"); + } + + return 0; +} + static int partition_acquire_uuid(Context *context, Partition *p, sd_id128_t *ret) { struct { sd_id128_t type_uuid; @@ -2476,6 +2571,10 @@ static int context_write_partition_table( if (r < 0) return r; + r = context_mkfs(context); + if (r < 0) + return r; + LIST_FOREACH(partitions, p, context->partitions) { if (p->dropped) continue; From 0f2b2c483fa9f9ce51dbd851728b06e125ede99c Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 30 Jul 2020 22:29:48 +0200 Subject: [PATCH 08/20] mkfs-util: add support for making vfat partitions fat is a bit more limited in volume name length and UUID support. Let's add some special support for it. This is particularly useful to generate EFI system partitions. --- src/shared/mkfs-util.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/shared/mkfs-util.c b/src/shared/mkfs-util.c index a78f68e398..583ef90e65 100644 --- a/src/shared/mkfs-util.c +++ b/src/shared/mkfs-util.c @@ -4,6 +4,7 @@ #include "mkfs-util.h" #include "path-util.h" #include "process-util.h" +#include "stdio-util.h" #include "string-util.h" int mkfs_exists(const char *fstype) { @@ -94,6 +95,26 @@ int make_filesystem( else (void) execlp(mkfs, mkfs, "-L", label, "-m", j, "-m", "reflink=1", "-K", node, NULL); + } else if (streq(fstype, "vfat")) { + char mangled_label[8 + 3 + 1], vol_id[8 + 1]; + + /* Classic FAT only allows 11 character uppercase labels */ + strncpy(mangled_label, label, sizeof(mangled_label)-1); + mangled_label[sizeof(mangled_label)-1] = 0; + ascii_strupper(mangled_label); + + xsprintf(vol_id, "%08" PRIx32, + ((uint32_t) uuid.bytes[0] << 24) | + ((uint32_t) uuid.bytes[1] << 16) | + ((uint32_t) uuid.bytes[2] << 8) | + ((uint32_t) uuid.bytes[3])); /* Take first 32 byte of UUID */ + + (void) execlp(mkfs, mkfs, + "-i", vol_id, + "-n", mangled_label, + "-F", "32", /* yes, we force FAT32 here */ + node, NULL); + } else if (streq(fstype, "swap")) { (void) execlp(mkfs, mkfs, From f28d4f42f4fa64fa9a58a0d6d5d566874a66171f Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 30 Jul 2020 16:35:15 +0200 Subject: [PATCH 09/20] repart: split out code that mangles part table entries into function of its own Just some refactoring, no actual code change. --- src/partition/repart.c | 135 ++++++++++++++++++++++------------------- 1 file changed, 74 insertions(+), 61 deletions(-) diff --git a/src/partition/repart.c b/src/partition/repart.c index f2abb28f25..3e8b29384d 100644 --- a/src/partition/repart.c +++ b/src/partition/repart.c @@ -2511,70 +2511,12 @@ static int device_kernel_partitions_supported(int fd) { return FLAGS_SET(info.lo_flags, LO_FLAGS_PARTSCAN); } -static int context_write_partition_table( - Context *context, - const char *node, - bool from_scratch) { - - _cleanup_(fdisk_unref_tablep) struct fdisk_table *original_table = NULL; - int capable, r; +static int context_mangle_partitions(Context *context) { Partition *p; + int r; assert(context); - if (arg_pretty > 0 || - (arg_pretty < 0 && isatty(STDOUT_FILENO) > 0) || arg_json) { - - (void) context_dump_partitions(context, node); - - putc('\n', stdout); - - if (!arg_json) - (void) context_dump_partition_bar(context, node); - putc('\n', stdout); - fflush(stdout); - } - - if (!from_scratch && !context_changed(context)) { - log_info("No changes."); - return 0; - } - - if (arg_dry_run) { - log_notice("Refusing to repartition, please re-run with --dry-run=no."); - return 0; - } - - log_info("Applying changes."); - - if (from_scratch) { - r = context_discard_range(context, 0, context->total); - if (r == -EOPNOTSUPP) - log_info("Storage does not support discarding, not discarding entire block device data."); - else if (r < 0) - return log_error_errno(r, "Failed to discard entire block device: %m"); - else if (r > 0) - log_info("Discarded entire block device."); - } - - r = fdisk_get_partitions(context->fdisk_context, &original_table); - if (r < 0) - return log_error_errno(r, "Failed to acquire partition table: %m"); - - /* Wipe fs signatures and discard sectors where the new partitions are going to be placed and in the - * gaps between partitions, just to be sure. */ - r = context_wipe_and_discard(context, from_scratch); - if (r < 0) - return r; - - r = context_copy_blocks(context); - if (r < 0) - return r; - - r = context_mkfs(context); - if (r < 0) - return r; - LIST_FOREACH(partitions, p, context->partitions) { if (p->dropped) continue; @@ -2686,7 +2628,7 @@ static int context_write_partition_table( if (r < 0) return log_error_errno(r, "Failed to set partition label: %m"); - log_info("Creating new partition %" PRIu64 ".", p->partno); + log_info("Registering new partition %" PRIu64 " in partition table.", p->partno); r = fdisk_add_partition(context->fdisk_context, q, NULL); if (r < 0) @@ -2697,6 +2639,77 @@ static int context_write_partition_table( } } + return 0; +} + +static int context_write_partition_table( + Context *context, + const char *node, + bool from_scratch) { + + _cleanup_(fdisk_unref_tablep) struct fdisk_table *original_table = NULL; + int capable, r; + + assert(context); + + if (arg_pretty > 0 || + (arg_pretty < 0 && isatty(STDOUT_FILENO) > 0) || + arg_json) { + + (void) context_dump_partitions(context, node); + + putc('\n', stdout); + + if (!arg_json) + (void) context_dump_partition_bar(context, node); + putc('\n', stdout); + fflush(stdout); + } + + if (!from_scratch && !context_changed(context)) { + log_info("No changes."); + return 0; + } + + if (arg_dry_run) { + log_notice("Refusing to repartition, please re-run with --dry-run=no."); + return 0; + } + + log_info("Applying changes."); + + if (from_scratch) { + r = context_discard_range(context, 0, context->total); + if (r == -EOPNOTSUPP) + log_info("Storage does not support discarding, not discarding entire block device data."); + else if (r < 0) + return log_error_errno(r, "Failed to discard entire block device: %m"); + else if (r > 0) + log_info("Discarded entire block device."); + } + + r = fdisk_get_partitions(context->fdisk_context, &original_table); + if (r < 0) + return log_error_errno(r, "Failed to acquire partition table: %m"); + + /* Wipe fs signatures and discard sectors where the new partitions are going to be placed and in the + * gaps between partitions, just to be sure. */ + r = context_wipe_and_discard(context, from_scratch); + if (r < 0) + return r; + + r = context_copy_blocks(context); + if (r < 0) + return r; + + r = context_mkfs(context); + if (r < 0) + return r; + + r = context_mangle_partitions(context); + if (r < 0) + return r; + log_info("Writing new partition table."); r = fdisk_write_disklabel(context->fdisk_context); From 81873a6b6c4dcbc967c0931a1973b383259e5f97 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 30 Jul 2020 18:46:42 +0200 Subject: [PATCH 10/20] repart: let's wipe the partition table ourselves Let's issue the wiping ourselves, so that we know it's done before we write partition data onto the disk, and before the disk label is written. Before this commit the writing of the disk label would imply the wiping step, potentially overriding again what we just wrote into the disk data section. (Normally this shouldn't matter, since the partition table metadata that the wiping process deletes is at the start and end of the disk while we write our data to the middle, but you never know what kind of weird signatures might exist that depart from that.) (And effectively this ends up using the same wiping code, since that's implemented in libblkkid, and libfdisk just acts as frontend to that anyway. We now simply call it directly.) --- src/partition/repart.c | 50 +++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/src/partition/repart.c b/src/partition/repart.c index 3e8b29384d..2be34898f7 100644 --- a/src/partition/repart.c +++ b/src/partition/repart.c @@ -1339,10 +1339,6 @@ static int context_load_partition_table( } if (from_scratch) { - r = fdisk_enable_wipe(c, true); - if (r < 0) - return log_error_errno(r, "Failed to enable wiping of disk signature: %m"); - r = fdisk_create_disklabel(c, "gpt"); if (r < 0) return log_error_errno(r, "Failed to create GPT disk label: %m"); @@ -1979,32 +1975,29 @@ static bool context_changed(const Context *context) { return false; } -static int context_wipe_partition(Context *context, Partition *p) { +static int context_wipe_range(Context *context, uint64_t offset, uint64_t size) { _cleanup_(blkid_free_probep) blkid_probe probe = NULL; int r; assert(context); - assert(p); - assert(!PARTITION_EXISTS(p)); /* Safety check: never wipe existing partitions */ + assert(offset != UINT64_MAX); + assert(size != UINT64_MAX); probe = blkid_new_probe(); if (!probe) return log_oom(); - assert(p->offset != UINT64_MAX); - assert(p->new_size != UINT64_MAX); - errno = 0; - r = blkid_probe_set_device(probe, fdisk_get_devfd(context->fdisk_context), p->offset, p->new_size); + r = blkid_probe_set_device(probe, fdisk_get_devfd(context->fdisk_context), offset, size); if (r < 0) - return log_error_errno(errno ?: SYNTHETIC_ERRNO(EIO), "Failed to allocate device probe for partition %" PRIu64 ".", p->partno); + return log_error_errno(errno ?: SYNTHETIC_ERRNO(EIO), "Failed to allocate device probe for wiping."); errno = 0; if (blkid_probe_enable_superblocks(probe, true) < 0 || blkid_probe_set_superblocks_flags(probe, BLKID_SUBLKS_MAGIC|BLKID_SUBLKS_BADCSUM) < 0 || blkid_probe_enable_partitions(probe, true) < 0 || blkid_probe_set_partitions_flags(probe, BLKID_PARTS_MAGIC) < 0) - return log_error_errno(errno ?: SYNTHETIC_ERRNO(EIO), "Failed to enable superblock and partition probing for partition %" PRIu64 ".", p->partno); + return log_error_errno(errno ?: SYNTHETIC_ERRNO(EIO), "Failed to enable superblock and partition probing."); for (;;) { errno = 0; @@ -2019,11 +2012,32 @@ static int context_wipe_partition(Context *context, Partition *p) { return log_error_errno(errno ?: SYNTHETIC_ERRNO(EIO), "Failed to wipe file system signature."); } - log_info("Successfully wiped file system signatures from partition %" PRIu64 ".", p->partno); return 0; } -static int context_discard_range(Context *context, uint64_t offset, uint64_t size) { +static int context_wipe_partition(Context *context, Partition *p) { + int r; + + assert(context); + assert(p); + assert(!PARTITION_EXISTS(p)); /* Safety check: never wipe existing partitions */ + + assert(p->offset != UINT64_MAX); + assert(p->new_size != UINT64_MAX); + + r = context_wipe_range(context, p->offset, p->new_size); + if (r < 0) + return r; + + log_info("Successfully wiped file system signatures from future partition %" PRIu64 ".", p->partno); + return 0; +} + +static int context_discard_range( + Context *context, + uint64_t offset, + uint64_t size) { + struct stat st; int fd; @@ -2679,6 +2693,12 @@ static int context_write_partition_table( log_info("Applying changes."); if (from_scratch) { + r = context_wipe_range(context, 0, context->total); + if (r < 0) + return r; + + log_info("Wiped block device."); + r = context_discard_range(context, 0, context->total); if (r == -EOPNOTSUPP) log_info("Storage does not support discarding, not discarding entire block device data."); From 5b5109e2252cdf184aeecf10fac34234084eef32 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 30 Jul 2020 18:47:04 +0200 Subject: [PATCH 11/20] repart: talk about future partitions We initialize the partition contents before the partitions actually exist, hence to reduce confusion let's talk about "future partitions" up to the point where they are actually realized. --- src/partition/repart.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/partition/repart.c b/src/partition/repart.c index 2be34898f7..e8d6302296 100644 --- a/src/partition/repart.c +++ b/src/partition/repart.c @@ -2105,7 +2105,7 @@ static int context_discard_partition(Context *context, Partition *p) { r = context_discard_range(context, p->offset, p->new_size); if (r == -EOPNOTSUPP) { - log_info("Storage does not support discarding, not discarding data in new partition %" PRIu64 ".", p->partno); + log_info("Storage does not support discard, not discarding data in future partition %" PRIu64 ".", p->partno); return 0; } if (r == 0) { @@ -2113,9 +2113,9 @@ static int context_discard_partition(Context *context, Partition *p) { return 0; } if (r < 0) - return log_error_errno(r, "Failed to discard data for new partition %" PRIu64 ".", p->partno); + return log_error_errno(r, "Failed to discard data for future partition %" PRIu64 ".", p->partno); - log_info("Successfully discarded data from partition %" PRIu64 ".", p->partno); + log_info("Successfully discarded data from future partition %" PRIu64 ".", p->partno); return 1; } @@ -2156,9 +2156,9 @@ static int context_discard_gap_after(Context *context, Partition *p) { r = context_discard_range(context, gap, next - gap); if (r == -EOPNOTSUPP) { if (p) - log_info("Storage does not support discarding, not discarding gap after partition %" PRIu64 ".", p->partno); + log_info("Storage does not support discard, not discarding gap after partition %" PRIu64 ".", p->partno); else - log_info("Storage does not support discarding, not discarding gap at beginning of disk."); + log_info("Storage does not support discard, not discarding gap at beginning of disk."); return 0; } if (r == 0) /* Too short */ @@ -2249,7 +2249,7 @@ static int context_copy_blocks(Context *context) { if (lseek(fd, p->offset, SEEK_SET) == (off_t) -1) return log_error_errno(errno, "Failed to seek to partition offset: %m"); - log_info("Copying in '%s' (%s) on block level into partition %" PRIu64 ".", p->copy_blocks_path, format_bytes(buf, sizeof(buf), p->copy_blocks_size), p->partno); + log_info("Copying in '%s' (%s) on block level into future partition %" PRIu64 ".", p->copy_blocks_path, format_bytes(buf, sizeof(buf), p->copy_blocks_size), p->partno); r = copy_bytes_full(p->copy_blocks_fd, fd, p->copy_blocks_size, 0, NULL, NULL, NULL, NULL); if (r < 0) @@ -2293,7 +2293,7 @@ static int context_mkfs(Context *context) { r = loop_device_make(fd, O_RDWR, p->offset, p->new_size, 0, &d); if (r < 0) - return log_error_errno(r, "Failed to make loopback device of partition %" PRIu64 ": %m", p->partno); + return log_error_errno(r, "Failed to make loopback device of future partition %" PRIu64 ": %m", p->partno); r = loop_device_flock(d, LOCK_EX); if (r < 0) @@ -2642,7 +2642,7 @@ static int context_mangle_partitions(Context *context) { if (r < 0) return log_error_errno(r, "Failed to set partition label: %m"); - log_info("Registering new partition %" PRIu64 " in partition table.", p->partno); + log_info("Adding new partition %" PRIu64 " to partition table.", p->partno); r = fdisk_add_partition(context->fdisk_context, q, NULL); if (r < 0) @@ -2701,7 +2701,7 @@ static int context_write_partition_table( r = context_discard_range(context, 0, context->total); if (r == -EOPNOTSUPP) - log_info("Storage does not support discarding, not discarding entire block device data."); + log_info("Storage does not support discard, not discarding entire block device data."); else if (r < 0) return log_error_errno(r, "Failed to discard entire block device: %m"); else if (r > 0) From f0cb1b951caeec68d4c892c7d02e6829f6f95a48 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 30 Jul 2020 22:22:21 +0200 Subject: [PATCH 12/20] repart: wipe partition first, then discard Wiping means writing zero sectors to disk. Hence it's better to do this before we discard, so that the zeroes we use to overwrite are properly discarded. If we'd do it the other way round we'd discard the data and then reallocte it just to write zeroes. --- src/partition/repart.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/partition/repart.c b/src/partition/repart.c index e8d6302296..cbb10f7475 100644 --- a/src/partition/repart.c +++ b/src/partition/repart.c @@ -2193,17 +2193,15 @@ static int context_wipe_and_discard(Context *context, bool from_scratch) { if (!p->allocated_to_area) continue; - if (!from_scratch) { - r = context_discard_partition(context, p); - if (r < 0) - return r; - } - r = context_wipe_partition(context, p); if (r < 0) return r; if (!from_scratch) { + r = context_discard_partition(context, p); + if (r < 0) + return r; + r = context_discard_gap_after(context, p); if (r < 0) return r; From 1f0f82f1311e4c52152b8e2b6f266258709c137d Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Mon, 3 Aug 2020 12:30:42 +0200 Subject: [PATCH 13/20] dissect: create directories we want to mount on This matches how we handle things everywhere else, i.e. in .mount units, and similar: when a mount point dir is missing, we create it, let's do so too when dealing with disk images. This makes things a lot simpler, more robust, and systematic. --- src/shared/dissect-image.c | 55 +++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/src/shared/dissect-image.c b/src/shared/dissect-image.c index d573b2feec..00bb5da55f 100644 --- a/src/shared/dissect-image.c +++ b/src/shared/dissect-image.c @@ -1017,6 +1017,13 @@ static int mount_partition( } if (directory) { + if (!FLAGS_SET(flags, DISSECT_IMAGE_READ_ONLY)) { + /* Automatically create missing mount points, if necessary. */ + r = mkdir_p_root(where, directory, uid_shift, (gid_t) uid_shift, 0755); + if (r < 0) + return r; + } + r = chase_symlinks(directory, where, CHASE_PREFIX_ROOT, &chased, NULL); if (r < 0) return r; @@ -1062,7 +1069,7 @@ static int mount_partition( } int dissected_image_mount(DissectedImage *m, const char *where, uid_t uid_shift, DissectImageFlags flags) { - int r, boot_mounted; + int r, xbootldr_mounted; assert(m); assert(where); @@ -1116,29 +1123,47 @@ int dissected_image_mount(DissectedImage *m, const char *where, uid_t uid_shift, if (r < 0) return r; - boot_mounted = mount_partition(m->partitions + PARTITION_XBOOTLDR, where, "/boot", uid_shift, flags); - if (boot_mounted < 0) - return boot_mounted; + xbootldr_mounted = mount_partition(m->partitions + PARTITION_XBOOTLDR, where, "/boot", uid_shift, flags); + if (xbootldr_mounted < 0) + return xbootldr_mounted; if (m->partitions[PARTITION_ESP].found) { + int esp_done = false; + /* Mount the ESP to /efi if it exists. If it doesn't exist, use /boot instead, but only if it * exists and is empty, and we didn't already mount the XBOOTLDR partition into it. */ r = chase_symlinks("/efi", where, CHASE_PREFIX_ROOT, NULL, NULL); - if (r >= 0) { + if (r < 0) { + if (r != -ENOENT) + return r; + + /* /efi doesn't exist. Let's see if /boot is suitable then */ + + if (!xbootldr_mounted) { + _cleanup_free_ char *p = NULL; + + r = chase_symlinks("/boot", where, CHASE_PREFIX_ROOT, &p, NULL); + if (r < 0) { + if (r != -ENOENT) + return r; + } else if (dir_is_empty(p) > 0) { + /* It exists and is an empty directory. Let's mount the ESP there. */ + r = mount_partition(m->partitions + PARTITION_ESP, where, "/boot", uid_shift, flags); + if (r < 0) + return r; + + esp_done = true; + } + } + } + + if (!esp_done) { + /* OK, let's mount the ESP now to /efi (possibly creating the dir if missing) */ + r = mount_partition(m->partitions + PARTITION_ESP, where, "/efi", uid_shift, flags); if (r < 0) return r; - - } else if (boot_mounted <= 0) { - _cleanup_free_ char *p = NULL; - - r = chase_symlinks("/boot", where, CHASE_PREFIX_ROOT, &p, NULL); - if (r >= 0 && dir_is_empty(p) > 0) { - r = mount_partition(m->partitions + PARTITION_ESP, where, "/boot", uid_shift, flags); - if (r < 0) - return r; - } } } From 8a794850e76a172d9e58eb4dbccf4045d696203b Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Mon, 3 Aug 2020 12:34:40 +0200 Subject: [PATCH 14/20] repart: add new CopyFiles= setting, for copying files into freshly made file systems This makes the tool a lot more useful for streaming OS images onto disks. --- src/partition/repart.c | 268 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 240 insertions(+), 28 deletions(-) diff --git a/src/partition/repart.c b/src/partition/repart.c index cbb10f7475..35e5ea6735 100644 --- a/src/partition/repart.c +++ b/src/partition/repart.c @@ -38,11 +38,14 @@ #include "locale-util.h" #include "loop-util.h" #include "main-func.h" +#include "mkdir.h" #include "mkfs-util.h" +#include "mount-util.h" #include "parse-util.h" #include "path-util.h" #include "pretty-print.h" #include "proc-cmdline.h" +#include "process-util.h" #include "sort-util.h" #include "specifier.h" #include "stat-util.h" @@ -50,6 +53,7 @@ #include "string-util.h" #include "strv.h" #include "terminal-util.h" +#include "user-util.h" #include "utf8.h" /* If not configured otherwise use a minimal partition size of 10M */ @@ -124,6 +128,7 @@ struct Partition { uint64_t copy_blocks_size; char *format; + char **copy_files; LIST_FIELDS(Partition, partitions); }; @@ -209,6 +214,7 @@ static Partition* partition_free(Partition *p) { safe_close(p->copy_blocks_fd); free(p->format); + strv_free(p->copy_files); return mfree(p); } @@ -873,6 +879,20 @@ static int config_parse_type( return 0; } +static const Specifier specifier_table[] = { + { 'm', specifier_machine_id, NULL }, + { 'b', specifier_boot_id, NULL }, + { 'H', specifier_host_name, NULL }, + { 'l', specifier_short_host_name, NULL }, + { 'v', specifier_kernel_release, NULL }, + { 'a', specifier_architecture, NULL }, + { 'o', specifier_os_id, NULL }, + { 'w', specifier_os_version_id, NULL }, + { 'B', specifier_os_build_id, NULL }, + { 'W', specifier_os_variant_id, NULL }, + {} +}; + static int config_parse_label( const char *unit, const char *filename, @@ -885,20 +905,6 @@ static int config_parse_label( void *data, void *userdata) { - static const Specifier specifier_table[] = { - { 'm', specifier_machine_id, NULL }, - { 'b', specifier_boot_id, NULL }, - { 'H', specifier_host_name, NULL }, - { 'l', specifier_short_host_name, NULL }, - { 'v', specifier_kernel_release, NULL }, - { 'a', specifier_architecture, NULL }, - { 'o', specifier_os_id, NULL }, - { 'w', specifier_os_version_id, NULL }, - { 'B', specifier_os_build_id, NULL }, - { 'W', specifier_os_variant_id, NULL }, - {} - }; - _cleanup_free_ char16_t *recoded = NULL; _cleanup_free_ char *resolved = NULL; char **label = data; @@ -1030,22 +1036,95 @@ static int config_parse_fstype( return free_and_strdup_warn(fstype, rvalue); } +static int config_parse_copy_files( + const char *unit, + const char *filename, + unsigned line, + const char *section, + unsigned section_line, + const char *lvalue, + int ltype, + const char *rvalue, + void *data, + void *userdata) { + + _cleanup_free_ char *source = NULL, *buffer = NULL, *resolved_source = NULL, *resolved_target = NULL; + const char *p = rvalue, *target; + Partition *partition = data; + int r; + + assert(rvalue); + assert(partition); + + r = extract_first_word(&p, &source, ":", EXTRACT_CUNESCAPE|EXTRACT_DONT_COALESCE_SEPARATORS); + if (r < 0) + return log_syntax(unit, LOG_ERR, filename, line, r, "Failed to extract source path: %s", rvalue); + if (r == 0) { + log_syntax(unit, LOG_WARNING, filename, line, 0, "No argument specified: %s", rvalue); + return 0; + } + + r = extract_first_word(&p, &buffer, ":", EXTRACT_CUNESCAPE|EXTRACT_DONT_COALESCE_SEPARATORS); + if (r < 0) + return log_syntax(unit, LOG_ERR, filename, line, r, "Failed to extract target path: %s", rvalue); + if (r == 0) + target = source; /* No target, then it's the same as the source */ + else + target = buffer; + + if (!isempty(p)) + return log_syntax(unit, LOG_ERR, filename, line, SYNTHETIC_ERRNO(EINVAL), "Too many arguments: %s", rvalue); + + r = specifier_printf(source, specifier_table, NULL, &resolved_source); + if (r < 0) { + log_syntax(unit, LOG_WARNING, filename, line, r, + "Failed to expand specifiers in CopyFiles= source, ignoring: %s", rvalue); + return 0; + } + + if (!path_is_absolute(resolved_source) || !path_is_normalized(resolved_source)) { + log_syntax(unit, LOG_WARNING, filename, line, 0, + "Invalid path name in CopyFiles= source, ignoring: %s", resolved_source); + return 0; + } + + r = specifier_printf(target, specifier_table, NULL, &resolved_target); + if (r < 0) { + log_syntax(unit, LOG_WARNING, filename, line, r, + "Failed to expand specifiers in CopyFiles= target, ignoring: %s", resolved_target); + return 0; + } + + if (!path_is_absolute(resolved_target) || !path_is_normalized(resolved_target)) { + log_syntax(unit, LOG_WARNING, filename, line, 0, + "Invalid path name in CopyFiles= source, ignoring: %s", resolved_target); + return 0; + } + + r = strv_consume_pair(&partition->copy_files, TAKE_PTR(resolved_source), TAKE_PTR(resolved_target)); + if (r < 0) + return log_oom(); + + return 0; +} + static int partition_read_definition(Partition *p, const char *path) { ConfigTableItem table[] = { - { "Partition", "Type", config_parse_type, 0, &p->type_uuid }, - { "Partition", "Label", config_parse_label, 0, &p->new_label }, - { "Partition", "UUID", config_parse_id128, 0, &p->new_uuid }, - { "Partition", "Priority", config_parse_int32, 0, &p->priority }, - { "Partition", "Weight", config_parse_weight, 0, &p->weight }, - { "Partition", "PaddingWeight", config_parse_weight, 0, &p->padding_weight }, - { "Partition", "SizeMinBytes", config_parse_size4096, 1, &p->size_min }, - { "Partition", "SizeMaxBytes", config_parse_size4096, -1, &p->size_max }, - { "Partition", "PaddingMinBytes", config_parse_size4096, 1, &p->padding_min }, - { "Partition", "PaddingMaxBytes", config_parse_size4096, -1, &p->padding_max }, - { "Partition", "FactoryReset", config_parse_bool, 0, &p->factory_reset }, - { "Partition", "CopyBlocks", config_parse_path, 0, &p->copy_blocks_path }, - { "Partition", "Format", config_parse_fstype, 0, &p->format }, + { "Partition", "Type", config_parse_type, 0, &p->type_uuid }, + { "Partition", "Label", config_parse_label, 0, &p->new_label }, + { "Partition", "UUID", config_parse_id128, 0, &p->new_uuid }, + { "Partition", "Priority", config_parse_int32, 0, &p->priority }, + { "Partition", "Weight", config_parse_weight, 0, &p->weight }, + { "Partition", "PaddingWeight", config_parse_weight, 0, &p->padding_weight }, + { "Partition", "SizeMinBytes", config_parse_size4096, 1, &p->size_min }, + { "Partition", "SizeMaxBytes", config_parse_size4096, -1, &p->size_max }, + { "Partition", "PaddingMinBytes", config_parse_size4096, 1, &p->padding_min }, + { "Partition", "PaddingMaxBytes", config_parse_size4096, -1, &p->padding_max }, + { "Partition", "FactoryReset", config_parse_bool, 0, &p->factory_reset }, + { "Partition", "CopyBlocks", config_parse_path, 0, &p->copy_blocks_path }, + { "Partition", "Format", config_parse_fstype, 0, &p->format }, + { "Partition", "CopyFiles", config_parse_copy_files, 0, p }, {} }; int r; @@ -1071,10 +1150,21 @@ static int partition_read_definition(Partition *p, const char *path) { return log_syntax(NULL, LOG_ERR, path, 1, SYNTHETIC_ERRNO(EINVAL), "Type= not defined, refusing."); - if (p->copy_blocks_path && p->format) + if (p->copy_blocks_path && (p->format || !strv_isempty(p->copy_files))) return log_syntax(NULL, LOG_ERR, path, 1, SYNTHETIC_ERRNO(EINVAL), "Format= and CopyBlocks= cannot be combined, refusing."); + if (!strv_isempty(p->copy_files) && streq_ptr(p->format, "swap")) + return log_syntax(NULL, LOG_ERR, path, 1, SYNTHETIC_ERRNO(EINVAL), + "Format=swap and CopyFiles= cannot be combined, refusing."); + + if (!p->format && !strv_isempty(p->copy_files)) { + /* Pick "ext4" as file system if we are configured to copy files */ + p->format = strdup("ext4"); + if (!p->format) + return log_oom(); + } + return 0; } @@ -2259,6 +2349,124 @@ static int context_copy_blocks(Context *context) { return 0; } +static int do_copy_files(Partition *p, const char *fs) { + char **source, **target; + int r; + + assert(p); + assert(fs); + + STRV_FOREACH_PAIR(source, target, p->copy_files) { + _cleanup_close_ int sfd = -1, pfd = -1, tfd = -1; + _cleanup_free_ char *dn = NULL; + + dn = dirname_malloc(*target); + if (!dn) + return log_oom(); + + sfd = chase_symlinks_and_open(*source, arg_root, CHASE_PREFIX_ROOT|CHASE_WARN, O_CLOEXEC|O_NOCTTY, NULL); + if (sfd < 0) + return log_error_errno(sfd, "Failed to open source file '%s%s': %m", strempty(arg_root), *source); + + r = fd_verify_regular(sfd); + if (r < 0) { + if (r != -EISDIR) + return log_error_errno(r, "Failed to check type of source file '%s': %m", *source); + + /* We are looking at a directory */ + tfd = chase_symlinks_and_open(*target, fs, CHASE_PREFIX_ROOT|CHASE_WARN, O_RDONLY|O_DIRECTORY|O_CLOEXEC, NULL); + if (tfd < 0) { + if (tfd != -ENOENT) + return log_error_errno(tfd, "Failed to open target directory '%s': %m", *target); + + r = mkdir_p_root(fs, dn, UID_INVALID, GID_INVALID, 0755); + if (r < 0) + return log_error_errno(r, "Failed to create parent directory '%s': %m", dn); + + pfd = chase_symlinks_and_open(dn, fs, CHASE_PREFIX_ROOT|CHASE_WARN, O_RDONLY|O_DIRECTORY|O_CLOEXEC, NULL); + if (pfd < 0) + return log_error_errno(pfd, "Failed to open parent directory of target: %m"); + + r = copy_tree_at(sfd, ".", pfd, basename(*target), UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_MERGE|COPY_REPLACE|COPY_SIGINT); + } else + r = copy_tree_at(sfd, ".", tfd, ".", UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_MERGE|COPY_REPLACE|COPY_SIGINT); + if (r < 0) + return log_error_errno(r, "Failed to copy %s%s to %s: %m", strempty(arg_root), *source, *target); + } else { + /* We are looking at a regular file */ + + r = mkdir_p_root(fs, dn, UID_INVALID, GID_INVALID, 0755); + if (r < 0) + return log_error_errno(r, "Failed to create parent directory: %m"); + + pfd = chase_symlinks_and_open(dn, fs, CHASE_PREFIX_ROOT|CHASE_WARN, O_RDONLY|O_DIRECTORY|O_CLOEXEC, NULL); + if (pfd < 0) + return log_error_errno(tfd, "Failed to open parent directory of target: %m"); + + tfd = openat(pfd, basename(*target), O_CREAT|O_EXCL|O_WRONLY|O_CLOEXEC, 0700); + if (tfd < 0) + return log_error_errno(errno, "Failed to create target file '%s': %m", *target); + + r = copy_bytes(sfd, tfd, UINT64_MAX, COPY_REFLINK|COPY_SIGINT); + if (r < 0) + return log_error_errno(r, "Failed to copy '%s%s' to '%s': %m", strempty(arg_root), *source, *target); + + (void) copy_xattr(sfd, tfd); + (void) copy_access(sfd, tfd); + (void) copy_times(sfd, tfd, 0); + } + } + + return 0; +} + +static int partition_copy_files(Partition *p, const char *node) { + int r; + + assert(p); + assert(node); + + if (strv_isempty(p->copy_files)) + return 0; + + log_info("Populating partition %" PRIu64 " with files.", p->partno); + + /* We copy in a child process, since we have to mount the fs for that, and we don't want that fs to + * appear in the host namespace. Hence we fork a child that has its own file system namespace and + * detached mount propagation. */ + + r = safe_fork("(sd-copy)", FORK_DEATHSIG|FORK_LOG|FORK_WAIT|FORK_NEW_MOUNTNS|FORK_MOUNTNS_SLAVE, NULL); + if (r < 0) + return r; + if (r == 0) { + static const char fs[] = "/run/systemd/mount-root"; + /* This is a child process with its own mount namespace and propagation to host turned off */ + + r = mkdir_p(fs, 0700); + if (r < 0) { + log_error_errno(r, "Failed to create mount point: %m"); + _exit(EXIT_FAILURE); + } + + if (mount_verbose(LOG_ERR, node, fs, p->format, MS_NOATIME|MS_NODEV|MS_NOEXEC|MS_NOSUID, NULL) < 0) + _exit(EXIT_FAILURE); + + if (do_copy_files(p, fs) < 0) + _exit(EXIT_FAILURE); + + r = syncfs_path(AT_FDCWD, fs); + if (r < 0) { + log_error_errno(r, "Failed to synchronize written files: %m"); + _exit(EXIT_FAILURE); + } + + _exit(EXIT_SUCCESS); + } + + log_info("Successfully populated partition %" PRIu64 " with files.", p->partno); + return 0; +} + static int context_mkfs(Context *context) { Partition *p; int fd = -1, r; @@ -2311,6 +2519,10 @@ static int context_mkfs(Context *context) { log_info("Successfully formatted future partition %" PRIu64 ".", p->partno); + r = partition_copy_files(p, d->node); + if (r < 0) + return r; + r = loop_device_sync(d); if (r < 0) return log_error_errno(r, "Failed to sync loopback device: %m"); From b9df353689c34d7180ff4b271e866ca597dd516f Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Tue, 4 Aug 2020 08:57:29 +0200 Subject: [PATCH 15/20] repart: add support for optionally encrypting partitions we create --- src/partition/repart.c | 255 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 243 insertions(+), 12 deletions(-) diff --git a/src/partition/repart.c b/src/partition/repart.c index 35e5ea6735..d997d377ea 100644 --- a/src/partition/repart.c +++ b/src/partition/repart.c @@ -24,10 +24,12 @@ #include "btrfs-util.h" #include "conf-files.h" #include "conf-parser.h" +#include "crypt-util.h" #include "def.h" #include "efivars.h" #include "errno-util.h" #include "fd-util.h" +#include "fileio.h" #include "format-table.h" #include "format-util.h" #include "fs-util.h" @@ -46,6 +48,7 @@ #include "pretty-print.h" #include "proc-cmdline.h" #include "process-util.h" +#include "random-util.h" #include "sort-util.h" #include "specifier.h" #include "stat-util.h" @@ -88,9 +91,12 @@ static int arg_pretty = -1; static uint64_t arg_size = UINT64_MAX; static bool arg_json = false; static JsonFormatFlags arg_json_format_flags = 0; +static void *arg_key = NULL; +static size_t arg_key_size = 0; STATIC_DESTRUCTOR_REGISTER(arg_root, freep); STATIC_DESTRUCTOR_REGISTER(arg_definitions, freep); +STATIC_DESTRUCTOR_REGISTER(arg_key, erase_and_freep); typedef struct Partition Partition; typedef struct FreeArea FreeArea; @@ -129,6 +135,7 @@ struct Partition { char *format; char **copy_files; + bool encrypt; LIST_FIELDS(Partition, partitions); }; @@ -1125,6 +1132,7 @@ static int partition_read_definition(Partition *p, const char *path) { { "Partition", "CopyBlocks", config_parse_path, 0, &p->copy_blocks_path }, { "Partition", "Format", config_parse_fstype, 0, &p->format }, { "Partition", "CopyFiles", config_parse_copy_files, 0, p }, + { "Partition", "Encrypt", config_parse_bool, 0, &p->encrypt }, {} }; int r; @@ -1158,8 +1166,8 @@ static int partition_read_definition(Partition *p, const char *path) { return log_syntax(NULL, LOG_ERR, path, 1, SYNTHETIC_ERRNO(EINVAL), "Format=swap and CopyFiles= cannot be combined, refusing."); - if (!p->format && !strv_isempty(p->copy_files)) { - /* Pick "ext4" as file system if we are configured to copy files */ + if (!p->format && (!strv_isempty(p->copy_files) || (p->encrypt && !p->copy_blocks_path))) { + /* Pick "ext4" as file system if we are configured to copy files or encrypt the device */ p->format = strdup("ext4"); if (!p->format) return log_oom(); @@ -2307,16 +2315,139 @@ static int context_wipe_and_discard(Context *context, bool from_scratch) { return 0; } +static int partition_encrypt( + Partition *p, + const char *node, + struct crypt_device **ret_cd, + char **ret_volume, + int *ret_fd) { + + _cleanup_(crypt_freep) struct crypt_device *cd = NULL; + _cleanup_(erase_and_freep) void *volume_key = NULL; + _cleanup_free_ char *dm_name = NULL, *vol = NULL; + char suuid[ID128_UUID_STRING_MAX]; + size_t volume_key_size = 256 / 8; + sd_id128_t uuid; + int r; + + assert(p); + assert(p->encrypt); + + if (asprintf(&dm_name, "luks-repart-%08" PRIx64, random_u64()) < 0) + return log_oom(); + + if (ret_volume) { + vol = path_join("/dev/mapper/", dm_name); + if (!vol) + return log_oom(); + } + + r = derive_uuid(p->new_uuid, "luks-uuid", &uuid); + if (r < 0) + return r; + + log_info("Encrypting future partition %" PRIu64 "...", p->partno); + + volume_key = malloc(volume_key_size); + if (!volume_key) + return log_oom(); + + r = genuine_random_bytes(volume_key, volume_key_size, RANDOM_BLOCK); + if (r < 0) + return log_error_errno(r, "Failed to generate volume key: %m"); + + r = crypt_init(&cd, node); + if (r < 0) + return log_error_errno(r, "Failed to allocate libcryptsetup context: %m"); + + cryptsetup_enable_logging(cd); + + r = crypt_format(cd, + CRYPT_LUKS2, + "aes", + "xts-plain64", + id128_to_uuid_string(uuid, suuid), + volume_key, + volume_key_size, + &(struct crypt_params_luks2) { + .label = p->new_label, + .sector_size = 512U, + }); + if (r < 0) + return log_error_errno(r, "Failed to LUKS2 format future partition: %m"); + + r = crypt_keyslot_add_by_volume_key( + cd, + CRYPT_ANY_SLOT, + volume_key, + volume_key_size, + strempty(arg_key), + arg_key_size); + if (r < 0) + return log_error_errno(r, "Failed to add LUKS2 key: %m"); + + r = crypt_activate_by_volume_key( + cd, + dm_name, + volume_key, + volume_key_size, + arg_discard ? CRYPT_ACTIVATE_ALLOW_DISCARDS : 0); + if (r < 0) + return log_error_errno(r, "Failed to activate LUKS superblock: %m"); + + log_info("Successfully encrypted future partition %" PRIu64 ".", p->partno); + + if (ret_fd) { + _cleanup_close_ int dev_fd = -1; + + dev_fd = open(vol, O_RDWR|O_CLOEXEC|O_NOCTTY); + if (dev_fd < 0) + return log_error_errno(errno, "Failed to open LUKS volume '%s': %m", vol); + + *ret_fd = TAKE_FD(dev_fd); + } + + if (ret_cd) + *ret_cd = TAKE_PTR(cd); + if (ret_volume) + *ret_volume = TAKE_PTR(vol); + + return 0; +} + +static int deactivate_luks(struct crypt_device *cd, const char *node) { + int r; + + if (!cd) + return 0; + + assert(node); + + /* udev or so might access out block device in the background while we are done. Let's hence force + * detach the volume. We sync'ed before, hence this should be safe. */ + + r = crypt_deactivate_by_name(cd, basename(node), CRYPT_DEACTIVATE_FORCE); + if (r < 0) + return log_error_errno(r, "Failed to deactivate LUKS device: %m"); + + return 1; +} + static int context_copy_blocks(Context *context) { Partition *p; - int fd = -1, r; + int whole_fd = -1, r; assert(context); /* Copy in file systems on the block level */ LIST_FOREACH(partitions, p, context->partitions) { + _cleanup_(crypt_freep) struct crypt_device *cd = NULL; + _cleanup_(loop_device_unrefp) LoopDevice *d = NULL; + _cleanup_free_ char *encrypted = NULL; + _cleanup_close_ int encrypted_dev_fd = -1; char buf[FORMAT_BYTES_MAX]; + int target_fd; if (p->copy_blocks_fd < 0) continue; @@ -2331,18 +2462,57 @@ static int context_copy_blocks(Context *context) { assert(p->copy_blocks_size != UINT64_MAX); assert(p->new_size >= p->copy_blocks_size); - if (fd < 0) - assert_se((fd = fdisk_get_devfd(context->fdisk_context)) >= 0); + if (whole_fd < 0) + assert_se((whole_fd = fdisk_get_devfd(context->fdisk_context)) >= 0); - if (lseek(fd, p->offset, SEEK_SET) == (off_t) -1) - return log_error_errno(errno, "Failed to seek to partition offset: %m"); + if (p->encrypt) { + r = loop_device_make(whole_fd, O_RDWR, p->offset, p->new_size, 0, &d); + if (r < 0) + return log_error_errno(r, "Failed to make loopback device of future partition %" PRIu64 ": %m", p->partno); + + r = loop_device_flock(d, LOCK_EX); + if (r < 0) + return log_error_errno(r, "Failed to lock loopback device: %m"); + + r = partition_encrypt(p, d->node, &cd, &encrypted, &encrypted_dev_fd); + if (r < 0) + return log_error_errno(r, "Failed to encrypt device: %m"); + + if (flock(encrypted_dev_fd, LOCK_EX) < 0) + return log_error_errno(errno, "Failed to lock LUKS device: %m"); + + target_fd = encrypted_dev_fd; + } else { + if (lseek(whole_fd, p->offset, SEEK_SET) == (off_t) -1) + return log_error_errno(errno, "Failed to seek to partition offset: %m"); + + target_fd = whole_fd; + } log_info("Copying in '%s' (%s) on block level into future partition %" PRIu64 ".", p->copy_blocks_path, format_bytes(buf, sizeof(buf), p->copy_blocks_size), p->partno); - r = copy_bytes_full(p->copy_blocks_fd, fd, p->copy_blocks_size, 0, NULL, NULL, NULL, NULL); + r = copy_bytes_full(p->copy_blocks_fd, target_fd, p->copy_blocks_size, 0, NULL, NULL, NULL, NULL); if (r < 0) return log_error_errno(r, "Failed to copy in data from '%s': %m", p->copy_blocks_path); + if (fsync(target_fd) < 0) + return log_error_errno(r, "Failed to synchronize copied data blocks: %m"); + + if (p->encrypt) { + encrypted_dev_fd = safe_close(encrypted_dev_fd); + + r = deactivate_luks(cd, encrypted); + if (r < 0) + return r; + + crypt_free(cd); + cd = NULL; + + r = loop_device_sync(d); + if (r < 0) + return log_error_errno(r, "Failed to sync loopback device: %m"); + } + log_info("Copying in of '%s' on block level completed.", p->copy_blocks_path); } @@ -2476,7 +2646,11 @@ static int context_mkfs(Context *context) { /* Make a file system */ LIST_FOREACH(partitions, p, context->partitions) { + _cleanup_(crypt_freep) struct crypt_device *cd = NULL; _cleanup_(loop_device_unrefp) LoopDevice *d = NULL; + _cleanup_free_ char *encrypted = NULL; + _cleanup_close_ int encrypted_dev_fd = -1; + const char *fsdev; sd_id128_t fs_uuid; if (p->dropped) @@ -2505,6 +2679,18 @@ static int context_mkfs(Context *context) { if (r < 0) return log_error_errno(r, "Failed to lock loopback device: %m"); + if (p->encrypt) { + r = partition_encrypt(p, d->node, &cd, &encrypted, &encrypted_dev_fd); + if (r < 0) + return log_error_errno(r, "Failed to encrypt device: %m"); + + if (flock(encrypted_dev_fd, LOCK_EX) < 0) + return log_error_errno(errno, "Failed to lock LUKS device: %m"); + + fsdev = encrypted; + } else + fsdev = d->node; + log_info("Formatting future partition %" PRIu64 ".", p->partno); /* Calculate the UUID for the file system as HMAC-SHA256 of the string "file-system-uuid", @@ -2513,15 +2699,43 @@ static int context_mkfs(Context *context) { if (r < 0) return r; - r = make_filesystem(d->node, p->format, p->new_label, fs_uuid, arg_discard); - if (r < 0) + r = make_filesystem(fsdev, p->format, p->new_label, fs_uuid, arg_discard); + if (r < 0) { + encrypted_dev_fd = safe_close(encrypted_dev_fd); + (void) deactivate_luks(cd, encrypted); return r; + } log_info("Successfully formatted future partition %" PRIu64 ".", p->partno); - r = partition_copy_files(p, d->node); - if (r < 0) + /* The file system is now created, no need to delay udev further */ + if (p->encrypt) + if (flock(encrypted_dev_fd, LOCK_UN) < 0) + return log_error_errno(errno, "Failed to unlock LUKS device: %m"); + + r = partition_copy_files(p, fsdev); + if (r < 0) { + encrypted_dev_fd = safe_close(encrypted_dev_fd); + (void) deactivate_luks(cd, encrypted); return r; + } + + /* Note that we always sync explicitly here, since mkfs.fat doesn't do that on its own, and + * if we don't sync before detaching a block device the in-flight sectors possibly won't hit + * the disk. */ + + if (p->encrypt) { + if (fsync(encrypted_dev_fd) < 0) + return log_error_errno(r, "Failed to synchronize LUKS volume: %m"); + encrypted_dev_fd = safe_close(encrypted_dev_fd); + + r = deactivate_luks(cd, encrypted); + if (r < 0) + return r; + + crypt_free(cd); + cd = NULL; + } r = loop_device_sync(d); if (r < 0) @@ -3166,6 +3380,7 @@ static int help(void) { " --can-factory-reset Test whether factory reset is defined\n" " --root=PATH Operate relative to root path\n" " --definitions=DIR Find partitions in specified directory\n" + " --key-file=PATH Key to use when encrypting partitions\n" " --seed=UUID 128bit seed UUID to derive all UUIDs from\n" " --size=BYTES Grow loopback file to specified size\n" " --json=pretty|short|off\n" @@ -3194,6 +3409,7 @@ static int parse_argv(int argc, char *argv[]) { ARG_DEFINITIONS, ARG_SIZE, ARG_JSON, + ARG_KEY_FILE, }; static const struct option options[] = { @@ -3210,6 +3426,7 @@ static int parse_argv(int argc, char *argv[]) { { "definitions", required_argument, NULL, ARG_DEFINITIONS }, { "size", required_argument, NULL, ARG_SIZE }, { "json", required_argument, NULL, ARG_JSON }, + { "key-file", required_argument, NULL, ARG_KEY_FILE }, {} }; @@ -3333,6 +3550,7 @@ static int parse_argv(int argc, char *argv[]) { arg_size = rounded; break; } + case ARG_JSON: if (streq(optarg, "pretty")) { arg_json = true; @@ -3353,6 +3571,19 @@ static int parse_argv(int argc, char *argv[]) { break; + case ARG_KEY_FILE: { + _cleanup_(erase_and_freep) char *k = NULL; + size_t n = 0; + + r = read_full_file_full(AT_FDCWD, optarg, READ_FULL_FILE_SECURE|READ_FULL_FILE_CONNECT_SOCKET, &k, &n); + if (r < 0) + return log_error_errno(r, "Failed to read key file '%s': %m", optarg); + + erase_and_free(arg_key); + arg_key = TAKE_PTR(k); + arg_key_size = n; + break; + } case '?': return -EINVAL; From 170c98234530af6af487d37057b6e687569f8f91 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 5 Aug 2020 16:59:27 +0200 Subject: [PATCH 16/20] repart: if --size= is specified as "auto" determine minimal size for disk image When assembling a disk image locally, using --size=auto can be used to generate the minimal image based on the provided definitions. THis is useful to prepare images that are grown on first boot. --- src/partition/repart.c | 108 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 105 insertions(+), 3 deletions(-) diff --git a/src/partition/repart.c b/src/partition/repart.c index d997d377ea..96c4e809ae 100644 --- a/src/partition/repart.c +++ b/src/partition/repart.c @@ -49,6 +49,7 @@ #include "proc-cmdline.h" #include "process-util.h" #include "random-util.h" +#include "resize-fs.h" #include "sort-util.h" #include "specifier.h" #include "stat-util.h" @@ -65,6 +66,12 @@ /* Hard lower limit for new partition sizes */ #define HARD_MIN_SIZE 4096 +/* libfdisk takes off sightly more than 1M of the disk size when creating a GPT disk label */ +#define GPT_METADATA_SIZE (1044*1024) + +/* LUKS2 takes off 16M of the partition size with its metadata by default */ +#define LUKS2_METADATA_SIZE (16*1024*1024) + /* Note: When growing and placing new partitions we always align to 4K sector size. It's how newer hard disks * are designed, and if everything is aligned to that performance is best. And for older hard disks with 512B * sector size devices were generally assumed to have an even number of sectors, hence at the worst we'll @@ -89,6 +96,7 @@ static sd_id128_t arg_seed = SD_ID128_NULL; static bool arg_randomize = false; static int arg_pretty = -1; static uint64_t arg_size = UINT64_MAX; +static bool arg_size_auto = false; static bool arg_json = false; static JsonFormatFlags arg_json_format_flags = 0; static void *arg_key = NULL; @@ -371,8 +379,25 @@ static uint64_t partition_min_size(const Partition *p) { sz = p->current_size != UINT64_MAX ? p->current_size : HARD_MIN_SIZE; - if (p->copy_blocks_size != UINT64_MAX) - sz = MAX(p->copy_blocks_size, sz); + if (!PARTITION_EXISTS(p)) { + uint64_t d = 0; + + if (p->encrypt) + d += round_up_size(LUKS2_METADATA_SIZE, 4096); + + if (p->copy_blocks_size != UINT64_MAX) + d += round_up_size(p->copy_blocks_size, 4096); + else if (p->format || p->encrypt) { + uint64_t f; + + /* If we shall synthesize a file system, take minimal fs size into account (assumed to be 4K if not known) */ + f = p->format ? minimal_size_by_fs_name(p->format) : UINT64_MAX; + d += f == UINT64_MAX ? 4096 : f; + } + + if (d > sz) + sz = d; + } return MAX(p->size_min != UINT64_MAX ? p->size_min : DEFAULT_MIN_SIZE, sz); } @@ -1367,6 +1392,11 @@ static int context_load_partition_table( assert(context); assert(node); assert(backing_fd); + assert(!context->fdisk_context); + assert(!context->free_areas); + assert(context->start == UINT64_MAX); + assert(context->end == UINT64_MAX); + assert(context->total == UINT64_MAX); c = fdisk_new_context(); if (!c) @@ -1382,6 +1412,24 @@ static int context_load_partition_table( r = fdisk_assign_device(c, procfs_path, arg_dry_run); } + if (r == -EINVAL && arg_size_auto) { + struct stat st; + + /* libfdisk returns EINVAL if opening a file of size zero. Let's check for that, and accept + * it if automatic sizing is requested. */ + + if (*backing_fd < 0) + r = stat(node, &st); + else + r = fstat(*backing_fd, &st); + if (r < 0) + return log_error_errno(errno, "Failed to stat block device '%s': %m", node); + + if (S_ISREG(st.st_mode) && st.st_size == 0) + return /* from_scratch = */ true; + + r = -EINVAL; + } if (r < 0) return log_error_errno(r, "Failed to open device '%s': %m", node); @@ -3533,6 +3581,12 @@ static int parse_argv(int argc, char *argv[]) { case ARG_SIZE: { uint64_t parsed, rounded; + if (streq(optarg, "auto")) { + arg_size = UINT64_MAX; + arg_size_auto = true; + break; + } + r = parse_size(optarg, 1024, &parsed); if (r < 0) return log_error_errno(r, "Failed to parse --size= parameter: %s", optarg); @@ -3548,6 +3602,7 @@ static int parse_argv(int argc, char *argv[]) { parsed, rounded); arg_size = rounded; + arg_size_auto = false; break; } @@ -3607,7 +3662,7 @@ static int parse_argv(int argc, char *argv[]) { else if (dry_run >= 0) arg_dry_run = dry_run; - if (arg_empty == EMPTY_CREATE && arg_size == UINT64_MAX) + if (arg_empty == EMPTY_CREATE && (arg_size == UINT64_MAX && !arg_size_auto)) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "If --empty=create is specified, --size= must be specified, too."); @@ -3883,6 +3938,35 @@ static int resize_backing_fd(const char *node, int *fd) { return 1; } +static int determine_auto_size(Context *c) { + uint64_t sum = round_up_size(GPT_METADATA_SIZE, 4096); + char buf[FORMAT_BYTES_MAX]; + Partition *p; + + assert_se(c); + assert_se(arg_size == UINT64_MAX); + assert_se(arg_size_auto); + + LIST_FOREACH(partitions, p, c->partitions) { + uint64_t m; + + if (p->dropped) + continue; + + m = partition_min_size_with_padding(p); + if (m > UINT64_MAX - sum) + return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "Image would grow too large, refusing."); + + sum += m; + } + + assert_se(format_bytes(buf, sizeof(buf), sum)); + log_info("Automatically determined minimal disk image size as %s.", buf); + + arg_size = sum; + return 0; +} + static int run(int argc, char *argv[]) { _cleanup_(context_freep) Context* context = NULL; _cleanup_free_ char *node = NULL; @@ -3984,6 +4068,24 @@ static int run(int argc, char *argv[]) { if (r < 0) return r; + if (arg_size_auto) { + r = determine_auto_size(context); + if (r < 0) + return r; + + /* Flush out everything again, and let's grow the file first, then start fresh */ + context_unload_partition_table(context); + + assert_se(arg_size != UINT64_MAX); + r = resize_backing_fd(node, &backing_fd); + if (r < 0) + return r; + + r = context_load_partition_table(context, node, &backing_fd); + if (r < 0) + return r; + } + /* First try to fit new partitions in, dropping by priority until it fits */ for (;;) { if (context_allocate_partitions(context)) From f7011e17f1a86b4c355d644f379c668685b5e4e9 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 5 Aug 2020 18:52:12 +0200 Subject: [PATCH 17/20] test: add test for new repart features --- src/partition/test-repart.sh | 51 ++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/partition/test-repart.sh b/src/partition/test-repart.sh index 730f9cc9c3..8b1ec6c4c1 100755 --- a/src/partition/test-repart.sh +++ b/src/partition/test-repart.sh @@ -154,6 +154,57 @@ EOF cmp --bytes=41943040 --ignore-initial=0:$((512*4194264)) $D/block-copy $D/zzz +if [ `id -u` == 0 ] && type -P cryptsetup diff losetup > /dev/null ; then + echo "### Testing Format=/Encrypt=/CopyFiles=" + + # These tests require privileges unfortunately + + cat >$D/definitions/extra3.conf <$D/populated5 + + cmp $D/populated5 - < /dev/null + umount $D/mount + cryptsetup close $VOLUME + losetup -d $LOOP +else + echo "### Skipping Format=/Encrypt=/CopyFiles= test, lacking privileges or missing cryptsetup/diff/losetup" +fi + echo "### Testing json output ###" $repart $D/zzz --size=3G --dry-run=no --seed=$SEED --definitions=$D/definitions --json=help $repart $D/zzz --size=3G --dry-run=no --seed=$SEED --definitions=$D/definitions --json=pretty From dfb4d0ae0d0731c5efa0572f6433a2ee80a89dfb Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 5 Aug 2020 17:53:39 +0200 Subject: [PATCH 18/20] man: document new repart features --- man/repart.d.xml | 77 ++++++++++++++++++++++++++++++++++++++---- man/systemd-repart.xml | 39 +++++++++++++++++---- 2 files changed, 103 insertions(+), 13 deletions(-) diff --git a/man/repart.d.xml b/man/repart.d.xml index 1b104e76d0..9714a3602a 100644 --- a/man/repart.d.xml +++ b/man/repart.d.xml @@ -55,11 +55,11 @@ partition slot greater than the highest slot number currently in use. Any existing partitions that have no matching partition file are left as they are. - Note that these partition definition files do not describe the contents of the partitions, such as - the file system used. Separate mechanisms, such as - systemd-growfs8 and - systemd-makefs maybe be used to initialize or grow the file systems inside of these - partitions. + Note that these definitions may only be used to created and initialize new partitions or grow + existing ones. In the latter case it will not grow the contained files systems however; separate + mechanisms, such as + systemd-growfs8 may be + used to grow the file systems inside of these partitions. @@ -327,7 +327,72 @@ data is never overwritten. Note that the data is copied in before the partition table is updated, i.e. before the partition actually is persistently created. This provides robustness: it is guaranteed that the partition either doesn't exist or exists fully populated; it is not possible that - the partition exists but is not or only partially populated. + the partition exists but is not or only partially populated. + + This option cannot be combined with Format= or + CopyFiles=. + + + + Format= + + Takes a file system name, such as ext4, btrfs, + xfs or vfat, or the special value swap. If + specified and the partition is newly created it is formatted with the specified file system (or as + swap device). The file system UUID and label are automatically derived from the partition UUID and + label. If this option is used, the size allocation algorithm is slightly altered: the partition is + created as least as big as required for the minimal file system of the specified type (or 4KiB if the + minimal size is not known). + + This option has no effect if the partition already exists. + + Similar to the behaviour of CopyBlocks= the file system is formatted before + the partition is created, ensuring that the partition only ever exists with a fully initialized + file system. + + This option cannot be combined with CopyBlocks=. + + + + CopyFiles= + + Takes a pair of colon separated absolute file system paths. The first path refers to + a source file or directory on the host, the second path refers to a target in the file system of the + newly created partition and formatted file system. This setting may be used to copy files or + directories from the host into the file system that is created due to the Format= + option. If CopyFiles= is used without Format= specified + explicitly, Format= with a suitable default is implied (currently + ext4, but this may change in the future). This option may be used multiple times + to copy multiple files or directories from host into the newly formatted file system. The colon and + second path may be omitted in which case the source path is also used as the target path (relative to + the root of the newly created file system). If the source path refers to a directory it is copied + recursively. + + This option has no effect if the partition already exists: it cannot be used to copy additional + files into an existing partition, it may only be used to populate a file system created anew. + + The copy operation is executed before the file system is registered in the partition table, + thus ensuring that a file system populated this way only ever exists fully initialized. + + This option cannot be combined with CopyBlocks=. + + + + Encrypt= + + Takes a boolean parameter, defaulting to false. If true the partition will be + formatted with a LUKS2 superblock, before the blocks configured with CopyBlocks= + are copied in or the file system configured with Format= is created. + + The LUKS2 UUID is automatically derived from the partition UUID in a stable fashion. A single + key is added to the LUKS2 superblock, configurable with the switch to + systemd-repart. + + When used this slightly alters the size allocation logic as the implicit, minimal size limits + of Format= and CopyBlocks= are increased by the space necessary + for the LUKS2 superblock (see above). + + This option has no effect if the partition already exists. diff --git a/man/systemd-repart.xml b/man/systemd-repart.xml index 6905e99b11..fb0d4c3496 100644 --- a/man/systemd-repart.xml +++ b/man/systemd-repart.xml @@ -202,13 +202,26 @@ - Takes a size in bytes, using the usual K, M, G, T suffixes. If used the specified - device node path must refer to a regular file, which is then grown to the specified size if smaller, - before any change is made to the partition table. This is not supported if the specified node is a - block device. This switch has no effect if the file is already as large as the specified size or - larger. The specified size is implicitly rounded up to multiples of 4096. When used with - this specifies the initial size of the loopback file to - create. + Takes a size in bytes, using the usual K, M, G, T suffixes, or the special value + auto. If used the specified device node path must refer to a regular file, which + is then grown to the specified size if smaller, before any change is made to the partition table. If + specified as auto the minimal size for the disk image is automatically determined + (i.e. the minimal sizes of all partitions are summed up, taking space for additional metadata into + account). This switch is not supported if the specified node is a block device. This switch has no + effect if the file is already as large as the specified size or larger. The specified size is + implicitly rounded up to multiples of 4096. When used with this + specifies the initial size of the loopback file to create. + + The option takes the sizes of pre-existing partitions into + account. However, it does not accomodate for partition tables that are not tightly packed: the + configured partitions might still not fit into the backing device if empty space exists between + pre-existing partitions (or before the first partition) that cannot be fully filled by partitions to + grow or create. + + Also note that the automatic size determination does not take files or directories specified + with into account: operation might fail if the specified files or + directories require more disk space then the configured per-partition minimal size + limit. @@ -283,6 +296,18 @@ /run/repart.d/*.conf. + + + + Takes a file system path. Configures the encryption key to use when setting up LUKS2 + volumes configured with the Encrypt= setting in partition files. Should refer to a + regular file containing the key, or an AF_UNIX stream socket in the file + system. In the latter case a connection is made to it and the key read from it. If this switch is not + specified the empty key (i.e. zero length key) is used. This behaviour is useful for setting up encrypted + partitions during early first boot that receive their user-supplied password only in a later setup + step. + + From c2f418d5951a89a1313bfa7ac75ec4ed9bf51bca Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Tue, 4 Aug 2020 08:56:54 +0200 Subject: [PATCH 19/20] update TODO --- TODO | 7 ------- 1 file changed, 7 deletions(-) diff --git a/TODO b/TODO index 17fe6e8b2a..b63b436e29 100644 --- a/TODO +++ b/TODO @@ -234,9 +234,6 @@ Features: * systemd-repart: allow sizing partitions as factor of available RAM, so that we can reasonably size swap partitions for hibernation. -* systemd-repart: allow running mkfs before making partitions pop up + - encryption via LUKS to allow booting into an empty root with only /usr mounted in - * systemd-repart: allow managing the gpt read-only partition flag + auto-mount flag * systemd-repart: allow boolean option that ensures that if existing partition @@ -252,10 +249,6 @@ Features: * systemd-repart: add per-partition option to fail if partition already exist, i.e. is not added new. Similar, add option to fail if partition does not exist yet. -* systemd-repart: add --size=auto for generating/resizing images of minimal - size, i.e. where the image file is sized exactly as large as necessary taking - SizeMin= into account, but not a single byte larger. - * systemd-repart: allow disabling growing of specific partitions, or making them (think ESP: we don't ever want to grow it, since we cannot resize vfat) From bb2551bdcb7f1bbc91099351474ca4a415f826ea Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Mon, 24 Aug 2020 18:11:06 +0200 Subject: [PATCH 20/20] loop-util: LOOP_CONFIGURE ignores lo_sizelimit It appears LOOP_CONFIGURE in 5.8 is even more broken than initially thought: it doesn't properly propgate lo_sizelimit to the block device layer. :-( Let's hence check the block device size immediately after issuing LOOP_CONFIGURE, and if it doesn't match what we just set let's fallback to the old ioctls. This means LOOP_CONFIGURE currently works correctly only for the most simply case: no partition table logic and no size limit. Sad! (Kernel people should really be told about the concepts of tests and even CI, one day!) --- src/shared/loop-util.c | 44 ++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/src/shared/loop-util.c b/src/shared/loop-util.c index fb88f51849..fcbd5a5f79 100644 --- a/src/shared/loop-util.c +++ b/src/shared/loop-util.c @@ -47,18 +47,42 @@ static int loop_configure(int fd, const struct loop_config *c) { if (!ERRNO_IS_NOT_SUPPORTED(errno) && errno != EINVAL) return -errno; } else { - if (!FLAGS_SET(c->info.lo_flags, LO_FLAGS_PARTSCAN)) + bool good = true; + + if (c->info.lo_sizelimit != 0) { + /* Kernel 5.8 vanilla doesn't properly propagate the size limit into the block + * device. If it's used, let's immediately check if it had the desired effect + * hence. And if not use classic LOOP_SET_STATUS64. */ + uint64_t z; + + if (ioctl(fd, BLKGETSIZE64, &z) < 0) { + r = -errno; + goto fail; + } + + if (z != c->info.lo_sizelimit) { + log_debug("LOOP_CONFIGURE is broken, doesn't honour .lo_sizelimit. Falling back to LOOP_SET_STATUS64."); + good = false; + } + } + + if (FLAGS_SET(c->info.lo_flags, LO_FLAGS_PARTSCAN)) { + /* Kernel 5.8 vanilla doesn't properly propagate the partition scanning flag into the + * block device. Let's hence verify if things work correctly here before + * returning. */ + + r = blockdev_partscan_enabled(fd); + if (r < 0) + goto fail; + if (r == 0) { + log_debug("LOOP_CONFIGURE is broken, doesn't honour LO_FLAGS_PARTSCAN. Falling back to LOOP_SET_STATUS64."); + good = false; + } + } + + if (good) return 0; - /* Kernel 5.8 vanilla doesn't properly propagate the partition scanning flag into the - * block device. Let's hence verify if things work correctly here before returning. */ - - r = blockdev_partscan_enabled(fd); - if (r < 0) - goto fail; - if (r > 0) - return 0; /* All is good. */ - /* Otherwise, undo the attachment and use the old APIs */ (void) ioctl(fd, LOOP_CLR_FD); }