From 9aa3e5ebdfee7a02a68198a8f2663a45c8bd4b25 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Tue, 2 Nov 2021 23:09:31 +0100 Subject: [PATCH 1/7] user-record: add rebalanceWeight field --- docs/USER_RECORD.md | 13 ++++++++--- src/shared/user-record-show.c | 10 +++++++++ src/shared/user-record.c | 42 +++++++++++++++++++++++++++++++++++ src/shared/user-record.h | 8 +++++++ 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/docs/USER_RECORD.md b/docs/USER_RECORD.md index 9634f32b70..ea800c9928 100644 --- a/docs/USER_RECORD.md +++ b/docs/USER_RECORD.md @@ -507,6 +507,12 @@ the size configured in `diskSize` automatically at login time. If set to `shrink-and-grown` the home area is also shrunk to the minimal size possible (as dictated by used disk space and file system constraints) on logout. +`rebalanceWeight` → An unsigned integer, `null` or a boolean. Configures the +free disk space rebalancing weight for the home area. The integer must be in +the range 1…10000 to configure an explicit weight. If unset, or set to `null` +or `true` the default weight of 100 is implied. If set to 0 or `false` +rebalancing is turned off for this home area. + `service` → A string declaring the service that defines or manages this user record. It is recommended to use reverse domain name notation for this. For example, if `systemd-homed` manages a user a string of `io.systemd.Home` is @@ -729,9 +735,10 @@ that may be used in this section are identical to the equally named ones in the `fileSystemUuid`, `luksDiscard`, `luksOfflineDiscard`, `luksCipher`, `luksCipherMode`, `luksVolumeKeySize`, `luksPbkdfHashAlgorithm`, `luksPbkdfType`, `luksPbkdfTimeCostUSec`, `luksPbkdfMemoryCost`, -`luksPbkdfParallelThreads`, `rateLimitIntervalUSec`, `rateLimitBurst`, -`enforcePasswordPolicy`, `autoLogin`, `stopDelayUSec`, `killProcesses`, -`passwordChangeMinUSec`, `passwordChangeMaxUSec`, `passwordChangeWarnUSec`, +`luksPbkdfParallelThreads`, `autoResizeMode`, `rebalanceWeight`, +`rateLimitIntervalUSec`, `rateLimitBurst`, `enforcePasswordPolicy`, +`autoLogin`, `stopDelayUSec`, `killProcesses`, `passwordChangeMinUSec`, +`passwordChangeMaxUSec`, `passwordChangeWarnUSec`, `passwordChangeInactiveUSec`, `passwordChangeNow`, `pkcs11TokenUri`, `fido2HmacCredential`. diff --git a/src/shared/user-record-show.c b/src/shared/user-record-show.c index 17502845af..5335e64070 100644 --- a/src/shared/user-record-show.c +++ b/src/shared/user-record-show.c @@ -447,6 +447,16 @@ void user_record_show(UserRecord *hr, bool show_full_group_info) { if (hr->auto_resize_mode >= 0) printf(" Auto Resize: %s\n", auto_resize_mode_to_string(user_record_auto_resize_mode(hr))); + if (hr->rebalance_weight != REBALANCE_WEIGHT_UNSET) { + uint64_t rb; + + rb = user_record_rebalance_weight(hr); + if (rb == REBALANCE_WEIGHT_OFF) + printf(" Rebalance: off\n"); + else + printf(" Rebalance: weight %" PRIu64 "\n", rb); + } + if (!strv_isempty(hr->ssh_authorized_keys)) printf("SSH Pub. Key: %zu\n", strv_length(hr->ssh_authorized_keys)); diff --git a/src/shared/user-record.c b/src/shared/user-record.c index fa677adb2c..5b406d1f42 100644 --- a/src/shared/user-record.c +++ b/src/shared/user-record.c @@ -85,6 +85,7 @@ UserRecord* user_record_new(void) { .fido2_user_verification_permitted = -1, .drop_caches = -1, .auto_resize_mode = _AUTO_RESIZE_MODE_INVALID, + .rebalance_weight = REBALANCE_WEIGHT_UNSET, }; return h; @@ -984,6 +985,36 @@ static int dispatch_auto_resize_mode(const char *name, JsonVariant *variant, Jso return 0; } +static int dispatch_rebalance_weight(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata) { + uint64_t *rebalance_weight = userdata; + uintmax_t u; + + assert_se(rebalance_weight); + + if (json_variant_is_null(variant)) { + *rebalance_weight = REBALANCE_WEIGHT_UNSET; + return 0; + } + + if (json_variant_is_boolean(variant)) { + *rebalance_weight = json_variant_boolean(variant) ? REBALANCE_WEIGHT_DEFAULT : REBALANCE_WEIGHT_OFF; + return 0; + } + + if (!json_variant_is_unsigned(variant)) + return json_log(variant, flags, SYNTHETIC_ERRNO(EINVAL), "JSON field '%s' is not an unsigned integer, boolean or null.", strna(name)); + + u = json_variant_unsigned(variant); + if (u >= REBALANCE_WEIGHT_MIN && u <= REBALANCE_WEIGHT_MAX) + *rebalance_weight = (uint64_t) u; + else if (u == 0) + *rebalance_weight = REBALANCE_WEIGHT_OFF; + else + return json_log(variant, flags, SYNTHETIC_ERRNO(ERANGE), "Rebalance weight is out of valid range %" PRIu64 "…%" PRIu64 ".", REBALANCE_WEIGHT_MIN, REBALANCE_WEIGHT_MAX); + + return 0; +} + static int dispatch_privileged(const char *name, JsonVariant *variant, JsonDispatchFlags flags, void *userdata) { static const JsonDispatch privileged_dispatch_table[] = { @@ -1177,6 +1208,7 @@ static int dispatch_per_machine(const char *name, JsonVariant *variant, JsonDisp { "luksExtraMountOptions", JSON_VARIANT_STRING, json_dispatch_string, offsetof(UserRecord, luks_extra_mount_options), 0 }, { "dropCaches", JSON_VARIANT_BOOLEAN, json_dispatch_tristate, offsetof(UserRecord, drop_caches), 0 }, { "autoResizeMode", _JSON_VARIANT_TYPE_INVALID, dispatch_auto_resize_mode, offsetof(UserRecord, auto_resize_mode), 0 }, + { "rebalanceWeight", _JSON_VARIANT_TYPE_INVALID, dispatch_rebalance_weight, offsetof(UserRecord, rebalance_weight), 0 }, { "rateLimitIntervalUSec", JSON_VARIANT_UNSIGNED, json_dispatch_uint64, offsetof(UserRecord, ratelimit_interval_usec), 0 }, { "rateLimitBurst", JSON_VARIANT_UNSIGNED, json_dispatch_uint64, offsetof(UserRecord, ratelimit_burst), 0 }, { "enforcePasswordPolicy", JSON_VARIANT_BOOLEAN, json_dispatch_tristate, offsetof(UserRecord, enforce_password_policy), 0 }, @@ -1528,6 +1560,7 @@ int user_record_load(UserRecord *h, JsonVariant *v, UserRecordLoadFlags load_fla { "luksExtraMountOptions", JSON_VARIANT_STRING, json_dispatch_string, offsetof(UserRecord, luks_extra_mount_options), 0 }, { "dropCaches", JSON_VARIANT_BOOLEAN, json_dispatch_tristate, offsetof(UserRecord, drop_caches), 0 }, { "autoResizeMode", _JSON_VARIANT_TYPE_INVALID, dispatch_auto_resize_mode, offsetof(UserRecord, auto_resize_mode), 0 }, + { "rebalanceWeight", _JSON_VARIANT_TYPE_INVALID, dispatch_rebalance_weight, offsetof(UserRecord, rebalance_weight), 0 }, { "service", JSON_VARIANT_STRING, json_dispatch_string, offsetof(UserRecord, service), JSON_SAFE }, { "rateLimitIntervalUSec", JSON_VARIANT_UNSIGNED, json_dispatch_uint64, offsetof(UserRecord, ratelimit_interval_usec), 0 }, { "rateLimitBurst", JSON_VARIANT_UNSIGNED, json_dispatch_uint64, offsetof(UserRecord, ratelimit_burst), 0 }, @@ -1939,6 +1972,15 @@ AutoResizeMode user_record_auto_resize_mode(UserRecord *h) { return user_record_storage(h) == USER_LUKS ? AUTO_RESIZE_SHRINK_AND_GROW : AUTO_RESIZE_OFF; } +uint64_t user_record_rebalance_weight(UserRecord *h) { + assert(h); + + if (h->rebalance_weight == REBALANCE_WEIGHT_UNSET) + return REBALANCE_WEIGHT_DEFAULT; + + return h->rebalance_weight; +} + uint64_t user_record_ratelimit_next_try(UserRecord *h) { assert(h); diff --git a/src/shared/user-record.h b/src/shared/user-record.h index c5424eef82..276b915067 100644 --- a/src/shared/user-record.h +++ b/src/shared/user-record.h @@ -221,6 +221,12 @@ typedef enum AutoResizeMode { _AUTO_RESIZE_MODE_INVALID = -EINVAL, } AutoResizeMode; +#define REBALANCE_WEIGHT_OFF UINT64_C(0) +#define REBALANCE_WEIGHT_DEFAULT UINT64_C(100) +#define REBALANCE_WEIGHT_MIN UINT64_C(1) +#define REBALANCE_WEIGHT_MAX UINT64_C(10000) +#define REBALANCE_WEIGHT_UNSET UINT64_MAX + typedef struct UserRecord { /* The following three fields are not part of the JSON record */ unsigned n_ref; @@ -258,6 +264,7 @@ typedef struct UserRecord { char *skeleton_directory; mode_t access_mode; AutoResizeMode auto_resize_mode; + uint64_t rebalance_weight; uint64_t tasks_max; uint64_t memory_high; @@ -397,6 +404,7 @@ uint64_t user_record_ratelimit_burst(UserRecord *h); bool user_record_can_authenticate(UserRecord *h); bool user_record_drop_caches(UserRecord *h); AutoResizeMode user_record_auto_resize_mode(UserRecord *h); +uint64_t user_record_rebalance_weight(UserRecord *h); int user_record_build_image_path(UserStorage storage, const char *user_name_and_realm, char **ret); From 21505c937ce3bcb90aae85366a262462ec8fb6b2 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 4 Nov 2021 18:12:13 +0100 Subject: [PATCH 2/7] homectl: expose new rebalanceWeight JSON use record field --- man/homectl.xml | 22 +++++++++++++++++-- src/home/homectl.c | 54 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/man/homectl.xml b/man/homectl.xml index 5ba93bbe13..ca9225a1e0 100644 --- a/man/homectl.xml +++ b/man/homectl.xml @@ -376,8 +376,8 @@ Note that FIDO2 devices suitable for this option must implement the hmac-secret extension. Most current devices (such as the YubiKey 5 series) do. If the extension is not implemented the device cannot be used for unlocking home directories. - - The FIDO2 device may be subsequently removed by setting the device path to an empty string + + The FIDO2 device may be subsequently removed by setting the device path to an empty string (e.g. homectl update $USER --fido2-device=""). Note that many hardware security tokens implement both FIDO2 and PKCS#11/PIV (and thus may be @@ -706,6 +706,24 @@ again. + + + + Configures the weight parameter for the free disk space rebalancing logic. Only + applies to the LUKS2 backend (since for the LUKS2 backend disk space is allocated from a per-user + loopback file system instead of immediately from a common pool like the other backends do it). In + regular intervals free disk space in the active home areas and their backing storage is redistributed + among them, taking the weight value configured here into account. Expects an integer in the range + 1…10000, or the special string off. If not specified defaults to 100. The weight + is used to scale free space made available to the home areas: a home area with a weight of 200 will + get twice the free space as one with a weight of 100; a home area with a weight of 50 will get half + of that. The backing file system will be assigned space for a weight of 20. If set to + off no automatic free space distribution is done for this home area. Note that + resizing the home area explicitly (with homectl resize see below) will implicitly + turn off the automatic rebalancing. To reenable the automatic rebalancing use + with an empty parameter. + + BOOL BOOL diff --git a/src/home/homectl.c b/src/home/homectl.c index 706ce75dfb..0c69299c84 100644 --- a/src/home/homectl.c +++ b/src/home/homectl.c @@ -2270,6 +2270,7 @@ static int help(int argc, char *argv[], void *userdata) { " --luks-extra-mount-options=OPTIONS\n" " LUKS extra mount options\n" " --auto-resize-mode=MODE Automatically grow/shrink home on login/logout\n" + " --rebalance-weight=WEIGHT Weight while rebalancing\n" "\n%4$sMounting User Record Properties:%5$s\n" " --nosuid=BOOL Control the 'nosuid' flag of the home mount\n" " --nodev=BOOL Control the 'nodev' flag of the home mount\n" @@ -2370,6 +2371,7 @@ static int parse_argv(int argc, char *argv[]) { ARG_DROP_CACHES, ARG_LUKS_EXTRA_MOUNT_OPTIONS, ARG_AUTO_RESIZE_MODE, + ARG_REBALANCE_WEIGHT, }; static const struct option options[] = { @@ -2456,6 +2458,7 @@ static int parse_argv(int argc, char *argv[]) { { "drop-caches", required_argument, NULL, ARG_DROP_CACHES }, { "luks-extra-mount-options", required_argument, NULL, ARG_LUKS_EXTRA_MOUNT_OPTIONS }, { "auto-resize-mode", required_argument, NULL, ARG_AUTO_RESIZE_MODE }, + { "rebalance-weight", required_argument, NULL, ARG_REBALANCE_WEIGHT }, {} }; @@ -2931,13 +2934,13 @@ static int parse_argv(int argc, char *argv[]) { case ARG_DISK_SIZE: if (isempty(optarg)) { - r = drop_from_identity("diskSize"); - if (r < 0) - return r; + const char *prop; - r = drop_from_identity("diskSizeRelative"); - if (r < 0) - return r; + FOREACH_STRING(prop, "diskSize", "diskSizeRelative", "rebalanceWeight") { + r = drop_from_identity(prop); + if (r < 0) + return r; + } arg_disk_size = arg_disk_size_relative = UINT64_MAX; break; @@ -2973,6 +2976,11 @@ static int parse_argv(int argc, char *argv[]) { arg_disk_size = UINT64_MAX; } + /* Automatically turn off the rebalance logic if user configured a size explicitly */ + r = json_variant_set_field_unsigned(&arg_identity_extra_this_machine, "rebalanceWeight", REBALANCE_WEIGHT_OFF); + if (r < 0) + return log_error_errno(r, "Failed to set rebalanceWeight field: %m"); + break; case ARG_ACCESS_MODE: { @@ -3571,6 +3579,40 @@ static int parse_argv(int argc, char *argv[]) { break; + case ARG_REBALANCE_WEIGHT: { + uint64_t u; + + if (isempty(optarg)) { + r = drop_from_identity("rebalanceWeight"); + if (r < 0) + return r; + } + + if (streq(optarg, "off")) + u = REBALANCE_WEIGHT_OFF; + else { + r = safe_atou64(optarg, &u); + if (r < 0) + return log_error_errno(r, "Failed to parse --rebalance-weight= argument: %s", optarg); + + if (u < REBALANCE_WEIGHT_MIN || u > REBALANCE_WEIGHT_MAX) + return log_error_errno(SYNTHETIC_ERRNO(ERANGE), "Rebalancing weight out of valid range %" PRIu64 "…%" PRIu64 ": %s", + REBALANCE_WEIGHT_MIN, REBALANCE_WEIGHT_MAX, optarg); + } + + /* Drop from per machine stuff and everywhere */ + r = drop_from_identity("rebalanceWeight"); + if (r < 0) + return r; + + /* Add to main identity */ + r = json_variant_set_field_unsigned(&arg_identity_extra, "rebalanceWeight", u); + if (r < 0) + return log_error_errno(r, "Failed to set rebalanceWeight field: %m"); + + break; + } + case 'j': arg_json_format_flags = JSON_FORMAT_PRETTY_AUTO|JSON_FORMAT_COLOR_AUTO; break; From d357b80d331ebed709b7a9d71e014c319ba5bd79 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Tue, 2 Nov 2021 23:11:59 +0100 Subject: [PATCH 3/7] homed: add automatic grow/shrink ("rebalancing") --- meson.build | 3 +- src/home/home-util.h | 9 +- src/home/homed-home-bus.c | 2 +- src/home/homed-home.c | 67 +++++- src/home/homed-home.h | 14 +- src/home/homed-manager.c | 410 ++++++++++++++++++++++++++++++++++++ src/home/homed-manager.h | 20 ++ src/home/homework.c | 27 ++- src/home/user-record-util.c | 126 +++++++++++ src/home/user-record-util.h | 3 + src/shared/user-record.h | 1 + 11 files changed, 668 insertions(+), 14 deletions(-) diff --git a/meson.build b/meson.build index 6263e7c0fc..97732b6243 100644 --- a/meson.build +++ b/meson.build @@ -2387,7 +2387,8 @@ if conf.get('ENABLE_HOMED') == 1 link_with : [libshared], dependencies : [threads, libcrypt, - libopenssl], + libopenssl, + libm], install_rpath : rootlibexecdir, install : true, install_dir : rootlibexecdir) diff --git a/src/home/home-util.h b/src/home/home-util.h index ca4c068f37..69a88000f8 100644 --- a/src/home/home-util.h +++ b/src/home/home-util.h @@ -16,8 +16,13 @@ #define USER_DISK_SIZE_MIN (UINT64_C(5)*1024*1024) #define USER_DISK_SIZE_MAX (UINT64_C(5)*1024*1024*1024*1024) -/* The default disk size to use when nothing else is specified, relative to free disk space */ -#define USER_DISK_SIZE_DEFAULT_PERCENT 85 +/* The default disk size to use when nothing else is specified, relative to free disk space. We calculate + * this from the default rebalancing weights, so that what we create initially doesn't immediately require + * rebalancing. */ +#define USER_DISK_SIZE_DEFAULT_PERCENT ((unsigned) ((100 * REBALANCE_WEIGHT_DEFAULT) / (REBALANCE_WEIGHT_DEFAULT + REBALANCE_WEIGHT_BACKING))) + +/* This should be 83% right now, i.e. 100 of (100 + 20). Let's protect us against accidental changes. */ +assert_cc(USER_DISK_SIZE_DEFAULT_PERCENT == 83U); bool suitable_user_name(const char *name); int suitable_realm(const char *realm); diff --git a/src/home/homed-home-bus.c b/src/home/homed-home-bus.c index c71256d15e..9e9f537d6c 100644 --- a/src/home/homed-home-bus.c +++ b/src/home/homed-home-bus.c @@ -485,7 +485,7 @@ int bus_home_method_resize( if (r == 0) return 1; /* Will call us back */ - r = home_resize(h, sz, secret, error); + r = home_resize(h, sz, secret, /* automatic= */ false, error); if (r < 0) return r; diff --git a/src/home/homed-home.c b/src/home/homed-home.c index 2cc1f8b384..470c7f07f6 100644 --- a/src/home/homed-home.c +++ b/src/home/homed-home.c @@ -161,6 +161,7 @@ int home_new(Manager *m, UserRecord *hr, const char *sysfs, Home **ret) { (void) bus_manager_emit_auto_login_changed(m); (void) bus_home_emit_change(home); + (void) manager_schedule_rebalance(m, /* immediately= */ false); if (ret) *ret = TAKE_PTR(home); @@ -193,6 +194,8 @@ Home *home_free(Home *h) { if (h->manager->gc_focus == h) h->manager->gc_focus = NULL; + + (void) manager_schedule_rebalance(h->manager, /* immediately= */ false); } user_record_unref(h->record); @@ -489,6 +492,7 @@ static void home_set_state(Home *h, HomeState state) { * enqueue it for GC too. */ home_schedule_operation(h, NULL, NULL); + manager_reschedule_rebalance(h->manager); manager_enqueue_gc(h->manager, h); } } @@ -727,6 +731,7 @@ static void home_fixate_finish(Home *h, int ret, UserRecord *hr) { /* Reset the state to "invalid", which makes home_get_state() test if the image exists and returns * HOME_ABSENT vs. HOME_INACTIVE as necessary. */ home_set_state(h, _HOME_STATE_INVALID); + (void) manager_schedule_rebalance(h->manager, /* immediately= */ false); return; fail: @@ -781,6 +786,9 @@ static void home_activate_finish(Home *h, int ret, UserRecord *hr) { finish: h->current_operation = operation_result_unref(h->current_operation, r, &error); home_set_state(h, _HOME_STATE_INVALID); + + if (r >= 0) + (void) manager_schedule_rebalance(h->manager, /* immediately= */ true); } static void home_deactivate_finish(Home *h, int ret, UserRecord *hr) { @@ -803,6 +811,9 @@ static void home_deactivate_finish(Home *h, int ret, UserRecord *hr) { finish: h->current_operation = operation_result_unref(h->current_operation, r, &error); home_set_state(h, _HOME_STATE_INVALID); + + if (r >= 0) + (void) manager_schedule_rebalance(h->manager, /* immediately= */ true); } static void home_remove_finish(Home *h, int ret, UserRecord *hr) { @@ -841,6 +852,8 @@ static void home_remove_finish(Home *h, int ret, UserRecord *hr) { /* Unload this record from memory too now. */ h = home_free(h); + + (void) manager_schedule_rebalance(m, /* immediately= */ true); return; fail: @@ -885,6 +898,8 @@ static void home_create_finish(Home *h, int ret, UserRecord *hr) { h->current_operation = operation_result_unref(h->current_operation, 0, NULL); home_set_state(h, _HOME_STATE_INVALID); + + (void) manager_schedule_rebalance(h->manager, /* immediately= */ true); } static void home_change_finish(Home *h, int ret, UserRecord *hr) { @@ -918,6 +933,7 @@ static void home_change_finish(Home *h, int ret, UserRecord *hr) { } log_debug("Change operation of %s completed.", h->user_name); + (void) manager_schedule_rebalance(h->manager, /* immediately= */ false); r = 0; finish: @@ -1683,7 +1699,12 @@ int home_update(Home *h, UserRecord *hr, sd_bus_error *error) { return 0; } -int home_resize(Home *h, uint64_t disk_size, UserRecord *secret, sd_bus_error *error) { +int home_resize(Home *h, + uint64_t disk_size, + UserRecord *secret, + bool automatic, + sd_bus_error *error) { + _cleanup_(user_record_unrefp) UserRecord *c = NULL; HomeState state; int r; @@ -1711,6 +1732,12 @@ int home_resize(Home *h, uint64_t disk_size, UserRecord *secret, sd_bus_error *e if (r < 0) return r; + /* If the user didn't specify any size explicitly and rebalancing is on, then the disk size is + * determined by automatic rebalancing and hence not user configured but determined by us and thus + * applied anyway. */ + if (disk_size == UINT64_MAX && h->record->rebalance_weight != REBALANCE_WEIGHT_OFF) + return sd_bus_error_set(error, SD_BUS_ERROR_INVALID_ARGS, "Disk size is being determined by automatic disk space rebalancing."); + if (disk_size == UINT64_MAX || disk_size == h->record->disk_size) { if (h->record->disk_size == UINT64_MAX) return sd_bus_error_set(error, SD_BUS_ERROR_INVALID_ARGS, "No disk size to resize to specified."); @@ -1732,6 +1759,11 @@ int home_resize(Home *h, uint64_t disk_size, UserRecord *secret, sd_bus_error *e if (r < 0) return r; + /* If user picked an explicit size, then turn off rebalancing, so that we don't undo what user chose */ + r = user_record_set_rebalance_weight(c, REBALANCE_WEIGHT_OFF); + if (r < 0) + return r; + r = user_record_update_last_changed(c, false); if (r == -ECHRNG) return sd_bus_error_setf(error, BUS_ERROR_HOME_RECORD_MISMATCH, "Record last change time of %s is newer than current time, cannot update.", h->user_name); @@ -1746,7 +1778,7 @@ int home_resize(Home *h, uint64_t disk_size, UserRecord *secret, sd_bus_error *e c = TAKE_PTR(signed_c); } - r = home_update_internal(h, "resize", c, secret, error); + r = home_update_internal(h, automatic ? "resize-auto" : "resize", c, secret, error); if (r < 0) return r; @@ -2965,6 +2997,8 @@ static int on_pending(sd_event_source *s, void *userdata) { if (r < 0) return log_error_errno(r, "Failed to disable event source: %m"); + /* No operations pending anymore, maybe this is a good time to trigger a rebalancing */ + manager_reschedule_rebalance(h->manager); return 0; } @@ -3121,6 +3155,35 @@ int home_wait_for_worker(Home *h) { return 1; } +bool home_shall_rebalance(Home *h) { + HomeState state; + + assert(h); + + /* Determines if the home directory is a candidate for rebalancing */ + + if (!user_record_shall_rebalance(h->record)) + return false; + + state = home_get_state(h); + if (!HOME_STATE_SHALL_REBALANCE(state)) + return false; + + return true; +} + +bool home_is_busy(Home *h) { + assert(h); + + if (h->current_operation) + return true; + + if (!ordered_set_isempty(h->pending_operations)) + return true; + + return HOME_STATE_IS_EXECUTING_OPERATION(home_get_state(h)); +} + static const char* const home_state_table[_HOME_STATE_MAX] = { [HOME_UNFIXATED] = "unfixated", [HOME_ABSENT] = "absent", diff --git a/src/home/homed-home.h b/src/home/homed-home.h index 37c54b0557..0f314aad93 100644 --- a/src/home/homed-home.h +++ b/src/home/homed-home.h @@ -88,6 +88,8 @@ static inline bool HOME_STATE_SHALL_PIN(HomeState state) { HOME_AUTHENTICATING_FOR_ACQUIRE); } +#define HOME_STATE_SHALL_REBALANCE(state) HOME_STATE_SHALL_PIN(state) + static inline bool HOME_STATE_MAY_RETRY_DEACTIVATE(HomeState state) { /* Indicates when to leave the deactivate retry timer active */ return IN_SET(state, @@ -165,6 +167,12 @@ struct Home { /* An fd that locks the backing file of LUKS home dirs with a BSD lock. */ int luks_lock_fd; + + /* Space metrics during rebalancing */ + uint64_t rebalance_size, rebalance_usage, rebalance_free, rebalance_min, rebalance_weight, rebalance_goal; + + /* Whether a rebalance operation is pending */ + bool rebalance_pending; }; int home_new(Manager *m, UserRecord *hr, const char *sysfs, Home **ret); @@ -183,7 +191,7 @@ int home_deactivate(Home *h, bool force, sd_bus_error *error); int home_create(Home *h, UserRecord *secret, sd_bus_error *error); int home_remove(Home *h, sd_bus_error *error); int home_update(Home *h, UserRecord *new_record, sd_bus_error *error); -int home_resize(Home *h, uint64_t disk_size, UserRecord *secret, sd_bus_error *error); +int home_resize(Home *h, uint64_t disk_size, UserRecord *secret, bool automatic, sd_bus_error *error); int home_passwd(Home *h, UserRecord *new_secret, UserRecord *old_secret, sd_bus_error *error); int home_unregister(Home *h, sd_bus_error *error); int home_lock(Home *h, sd_bus_error *error); @@ -208,5 +216,9 @@ int home_set_current_message(Home *h, sd_bus_message *m); int home_wait_for_worker(Home *h); +bool home_shall_rebalance(Home *h); + +bool home_is_busy(Home *h); + const char *home_state_to_string(HomeState state); HomeState home_state_from_string(const char *s); diff --git a/src/home/homed-manager.c b/src/home/homed-manager.c index a97a0dee76..3851234a37 100644 --- a/src/home/homed-manager.c +++ b/src/home/homed-manager.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -35,7 +36,9 @@ #include "process-util.h" #include "quota-util.h" #include "random-util.h" +#include "resize-fs.h" #include "socket-util.h" +#include "sort-util.h" #include "stat-util.h" #include "strv.h" #include "sync-util.h" @@ -201,6 +204,7 @@ int manager_new(Manager **ret) { *m = (Manager) { .default_storage = _USER_STORAGE_INVALID, + .rebalance_interval_usec = 2 * USEC_PER_MINUTE, /* initially, rebalance every 2min */ }; r = manager_parse_config_file(m); @@ -259,6 +263,7 @@ Manager* manager_free(Manager *m) { m->deferred_rescan_event_source = sd_event_source_unref(m->deferred_rescan_event_source); m->deferred_gc_event_source = sd_event_source_unref(m->deferred_gc_event_source); m->deferred_auto_login_event_source = sd_event_source_unref(m->deferred_auto_login_event_source); + m->rebalance_event_source = sd_event_source_unref(m->rebalance_event_source); sd_event_unref(m->event); @@ -1770,3 +1775,408 @@ int manager_enqueue_gc(Manager *m, Home *focus) { (void) sd_event_source_set_description(m->deferred_gc_event_source, "deferred-gc"); return 1; } + +static bool manager_shall_rebalance(Manager *m) { + Home *h; + + assert(m); + + if (IN_SET(m->rebalance_state, REBALANCE_PENDING, REBALANCE_SHRINKING, REBALANCE_GROWING)) + return true; + + HASHMAP_FOREACH(h, m->homes_by_name) + if (home_shall_rebalance(h)) + return true; + + return false; +} + +static int home_cmp(Home *const*a, Home *const*b) { + int r; + + assert(a); + assert(*a); + assert(b); + assert(*b); + + /* Order user records by their weight (and by their name, to make things stable). We put the records + * with the heighest weight last, since we distribute space from the beginning and round down, hence + * later entries tend to get slightly more than earlier entries. */ + + r = CMP(user_record_rebalance_weight((*a)->record), user_record_rebalance_weight((*b)->record)); + if (r != 0) + return r; + + return strcmp((*a)->user_name, (*b)->user_name); +} + +static int manager_rebalance_calculate(Manager *m) { + uint64_t weight_sum, free_sum, usage_sum = 0, min_free = UINT64_MAX; + _cleanup_free_ Home **array = NULL; + bool relevant = false; + struct statfs sfs; + int c = 0, r; + Home *h; + + assert(m); + + if (statfs(get_home_root(), &sfs) < 0) + return log_error_errno(errno, "Failed to statfs() /home: %m"); + + free_sum = (uint64_t) sfs.f_bsize * sfs.f_bavail; /* This much free space is available on the + * underlying pool directory */ + + weight_sum = REBALANCE_WEIGHT_BACKING; /* Grant the underlying pool directory a fixed weight of 20 + * (home dirs get 100 by default, i.e. 5x more). This weight + * is not configurable, the per-home weights are. */ + + HASHMAP_FOREACH(h, m->homes_by_name) { + statfs_f_type_t fstype; + h->rebalance_pending = false; /* First, reset the flag, we only want it to be true for the + * homes that qualify for rebalancing */ + + if (!home_shall_rebalance(h)) /* Only look at actual candidates */ + continue; + + if (home_is_busy(h)) + return -EBUSY; /* Let's not rebalance if there's a busy home directory. */ + + r = home_get_disk_status( + h, + &h->rebalance_size, + &h->rebalance_usage, + &h->rebalance_free, + NULL, + NULL, + &fstype, + NULL); + if (r < 0) { + log_warning_errno(r, "Failed to get free space of home '%s', ignoring.", h->user_name); + continue; + } + + if (h->rebalance_free > UINT64_MAX - free_sum) + return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "Rebalance free overflow"); + free_sum += h->rebalance_free; + + if (h->rebalance_usage > UINT64_MAX - usage_sum) + return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "Rebalance usage overflow"); + usage_sum += h->rebalance_usage; + + h->rebalance_weight = user_record_rebalance_weight(h->record); + if (h->rebalance_weight > UINT64_MAX - weight_sum) + return log_error_errno(SYNTHETIC_ERRNO(EOVERFLOW), "Rebalance weight overflow"); + weight_sum += h->rebalance_weight; + + h->rebalance_min = minimal_size_by_fs_magic(fstype); + + if (!GREEDY_REALLOC(array, c+1)) + return log_oom(); + + array[c++] = h; + } + + if (c == 0) { + log_debug("No homes to rebalance."); + return 0; + } + + assert(weight_sum > 0); + + log_debug("Disk space usage by all home directories to rebalance: %s — available disk space: %s", + FORMAT_BYTES(usage_sum), FORMAT_BYTES(free_sum)); + + /* Bring the home directories in a well-defined order, so that we distribute space in a reproducible + * way for the same parameters. */ + typesafe_qsort(array, c, home_cmp); + + for (int i = 0; i < c; i++) { + uint64_t new_free; + double d; + + h = array[i]; + + assert(h->rebalance_free <= free_sum); + assert(h->rebalance_usage <= usage_sum); + assert(h->rebalance_weight <= weight_sum); + + d = ((double) (free_sum / 4096) * (double) h->rebalance_weight) / (double) weight_sum; /* Calculate new space for this home in units of 4K */ + + /* Convert from units of 4K back to bytes */ + if (d >= (double) (UINT64_MAX/4096)) + new_free = UINT64_MAX; + else + new_free = (uint64_t) d * 4096; + + /* Subtract the weight and assigned space from the sums now, to distribute the rounding noise + * to the remaining home dirs */ + free_sum = LESS_BY(free_sum, new_free); + weight_sum = LESS_BY(weight_sum, h->rebalance_weight); + + /* Keep track of home directory with the least amount of space left: we want to schedule the + * next rebalance more quickly if this is low */ + if (new_free < min_free) + min_free = h->rebalance_size; + + if (new_free > UINT64_MAX - h->rebalance_usage) + h->rebalance_goal = UINT64_MAX-1; /* maximum size */ + else { + h->rebalance_goal = h->rebalance_usage + new_free; + + if (h->rebalance_min != UINT64_MAX && h->rebalance_goal < h->rebalance_min) + h->rebalance_goal = h->rebalance_min; + } + + /* Skip over this home if the state doesn't match the operation */ + if ((m->rebalance_state == REBALANCE_SHRINKING && h->rebalance_goal > h->rebalance_size) || + (m->rebalance_state == REBALANCE_GROWING && h->rebalance_goal < h->rebalance_size)) + h->rebalance_pending = false; + else { + log_debug("Rebalancing home directory '%s' %s → %s.", h->user_name, + FORMAT_BYTES(h->rebalance_size), FORMAT_BYTES(h->rebalance_goal)); + h->rebalance_pending = true; + } + + if ((fabs((double) h->rebalance_size - (double) h->rebalance_goal) * 100 / (double) h->rebalance_size) >= 5.0) + relevant = true; + } + + /* Scale next rebalancing interval based on the least amount of space of any of the home + * directories. We pick a time in the range 1min … 15min, scaled by log2(min_free), so that: + * 10M → ~0.7min, 100M → ~2.7min, 1G → ~4.6min, 10G → ~6.5min, 100G ~8.4 */ + m->rebalance_interval_usec = (usec_t) CLAMP((LESS_BY(log2(min_free), 22)*15*USEC_PER_MINUTE)/26, + 1 * USEC_PER_MINUTE, + 15 * USEC_PER_MINUTE); + + + log_debug("Rebalancing interval set to %s.", FORMAT_TIMESPAN(m->rebalance_interval_usec, USEC_PER_MSEC)); + + /* Let's suppress small resizes, growing/shrinking file systems isn't free after all */ + if (!relevant) { + log_debug("Skipping rebalancing, since all calculated size changes are below ±5%%."); + return 0; + } + + return c; +} + +static int manager_rebalance_apply(Manager *m) { + int c = 0, r; + Home *h; + + assert(m); + + HASHMAP_FOREACH(h, m->homes_by_name) { + _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL; + + if (!h->rebalance_pending) + continue; + + h->rebalance_pending = false; + + r = home_resize(h, h->rebalance_goal, /* secret= */ NULL, /* automatic= */ true, &error); + if (r < 0) + log_warning_errno(r, "Failed to resize home '%s' for rebalancing, ignoring: %s", + h->user_name, bus_error_message(&error, r)); + else + c++; + } + + return c; +} + +static int manager_rebalance_now(Manager *m) { + RebalanceState busy_state; /* the state to revert to when operation fails if busy */ + int r; + + assert(m); + + log_debug("Rebalancing now..."); + + /* We maintain a simple state engine here to keep track of what we are doing. We'll first shrink all + * homes that shall be shrinked and then grow all homes that shall be grown, so that they can take up + * the space now freed. */ + + for (;;) { + switch (m->rebalance_state) { + + case REBALANCE_IDLE: + case REBALANCE_PENDING: + case REBALANCE_WAITING: + /* First shrink large home dirs */ + m->rebalance_state = REBALANCE_SHRINKING; + busy_state = REBALANCE_PENDING; + log_debug("Shrinking phase.."); + break; + + case REBALANCE_SHRINKING: + /* Then grow small home dirs */ + m->rebalance_state = REBALANCE_GROWING; + busy_state = REBALANCE_SHRINKING; + log_debug("Growing phase.."); + break; + + case REBALANCE_GROWING: + /* Finally, we are done */ + log_info("Rebalancing complete."); + m->rebalance_state = REBALANCE_IDLE; + r = 0; + goto finish; + + case REBALANCE_OFF: + default: + assert_not_reached(); + } + + r = manager_rebalance_calculate(m); + if (r == -EBUSY) { + /* Calculations failed because one home directory is currently busy. Revert to a state that + * tells us what to do next. */ + log_debug("Can't enter phase, busy."); + m->rebalance_state = busy_state; + return r; + } + if (r < 0) + goto finish; + if (r == 0) + continue; /* got to next step immediately, if there's nothing to do */ + + r = manager_rebalance_apply(m); + if (r < 0) + goto finish; + if (r > 0) + break; /* At least one resize operation is now pending, we are done for now */ + + /* If there was nothing to apply, go for next state right-away */ + } + + return 0; + +finish: + /* Reset state and schedule next rebalance */ + m->rebalance_state = REBALANCE_IDLE; + (void) manager_schedule_rebalance(m, /* immediately= */ false); + return r; +} + +static int on_rebalance_timer(sd_event_source *s, usec_t t, void *userdata) { + Manager *m = userdata; + + assert(s); + assert(m); + assert(IN_SET(m->rebalance_state, REBALANCE_WAITING, REBALANCE_PENDING, REBALANCE_SHRINKING, REBALANCE_GROWING)); + + (void) manager_rebalance_now(m); + return 0; +} + +int manager_schedule_rebalance(Manager *m, bool immediately) { + int r; + + assert(m); + + /* Check if there are any records where rebalancing is requested */ + if (!manager_shall_rebalance(m)) { + log_debug("Not scheduling rebalancing, not needed."); + goto turn_off; + } + + if (immediately) { + /* If we are told to rebalance immediately, then mark a rebalance as pending (even if we area + * already running one) */ + + if (m->rebalance_event_source) { + r = sd_event_source_set_time(m->rebalance_event_source, 0); + if (r < 0) { + log_error_errno(r, "Failed to schedule immediate rebalancing: %m"); + goto turn_off; + } + + r = sd_event_source_set_enabled(m->rebalance_event_source, SD_EVENT_ONESHOT); + if (r < 0) { + log_error_errno(r, "Failed to enable rebalancing event source: %m"); + goto turn_off; + } + } else { + r = sd_event_add_time(m->event, &m->rebalance_event_source, CLOCK_MONOTONIC, 0, USEC_PER_SEC, on_rebalance_timer, m); + if (r < 0) { + log_error_errno(r, "Failed to allocate rebalance event source: %m"); + goto turn_off; + } + + r = sd_event_source_set_priority(m->rebalance_event_source, SD_EVENT_PRIORITY_IDLE + 10); + if (r < 0) { + log_error_errno(r, "Failed to set rebalance event source priority: %m"); + goto turn_off; + } + + (void) sd_event_source_set_description(m->rebalance_event_source, "rebalance"); + + } + + if (!IN_SET(m->rebalance_state, REBALANCE_PENDING, REBALANCE_SHRINKING, REBALANCE_GROWING)) + m->rebalance_state = REBALANCE_PENDING; + + log_debug("Scheduled immediate rebalancing..."); + return 0; + } + + /* If we are told to schedule a rebalancing eventually, then do so only if we are not executing + * anything yet. Also if we have something scheduled already, leave it in place */ + if (!IN_SET(m->rebalance_state, REBALANCE_OFF, REBALANCE_IDLE)) + return 0; + + if (m->rebalance_event_source) { + r = sd_event_source_set_time_relative(m->rebalance_event_source, m->rebalance_interval_usec); + if (r < 0) { + log_error_errno(r, "Failed to schedule immediate rebalancing: %m"); + goto turn_off; + } + + r = sd_event_source_set_enabled(m->rebalance_event_source, SD_EVENT_ONESHOT); + if (r < 0) { + log_error_errno(r, "Failed to enable rebalancing event source: %m"); + goto turn_off; + } + } else { + r = sd_event_add_time_relative(m->event, &m->rebalance_event_source, CLOCK_MONOTONIC, m->rebalance_interval_usec, USEC_PER_SEC, on_rebalance_timer, m); + if (r < 0) { + log_error_errno(r, "Failed to allocate rebalance event source: %m"); + goto turn_off; + } + + r = sd_event_source_set_priority(m->rebalance_event_source, SD_EVENT_PRIORITY_IDLE + 10); + if (r < 0) { + log_error_errno(r, "Failed to set rebalance event source priority: %m"); + goto turn_off; + } + + (void) sd_event_source_set_description(m->rebalance_event_source, "rebalance"); + } + + m->rebalance_state = REBALANCE_WAITING; /* We managed to enqueue a timer event, we now wait until it fires */ + log_debug("Scheduled rebalancing in %s...", FORMAT_TIMESPAN(m->rebalance_interval_usec, 0)); + return 0; + +turn_off: + m->rebalance_event_source = sd_event_source_disable_unref(m->rebalance_event_source); + m->rebalance_state = REBALANCE_OFF; + return r; +} + +int manager_reschedule_rebalance(Manager *m) { + int r; + + assert(m); + + /* If a rebalance is pending reschedules it so it gets executed immediately */ + + if (!IN_SET(m->rebalance_state, REBALANCE_PENDING, REBALANCE_SHRINKING, REBALANCE_GROWING)) + return 0; + + r = manager_schedule_rebalance(m, /* immediately= */ true); + if (r < 0) + return r; + + return 1; +} diff --git a/src/home/homed-manager.h b/src/home/homed-manager.h index f5659636c3..cf6d58b258 100644 --- a/src/home/homed-manager.h +++ b/src/home/homed-manager.h @@ -13,6 +13,18 @@ typedef struct Manager Manager; #include "homed-home.h" #include "varlink.h" +/* The LUKS free disk space rebalancing logic goes through this state machine */ +typedef enum RebalanceState { + REBALANCE_OFF, /* No rebalancing enabled */ + REBALANCE_IDLE, /* Rebalancing enabled, but currently nothing scheduled */ + REBALANCE_WAITING, /* Rebalancing has been requested for a later point in time */ + REBALANCE_PENDING, /* Rebalancing has been requested and will be executed ASAP */ + REBALANCE_SHRINKING, /* Rebalancing ongoing, and we are running all shrinking operations */ + REBALANCE_GROWING, /* Rebalancing ongoign, and we are running all growing operations */ + _REBALANCE_STATE_MAX, + _REBALANCE_STATE_INVALID = -1, +} RebalanceState; + struct Manager { sd_event *event; sd_bus *bus; @@ -39,6 +51,8 @@ struct Manager { sd_event_source *deferred_gc_event_source; sd_event_source *deferred_auto_login_event_source; + sd_event_source *rebalance_event_source; + Home *gc_focus; VarlinkServer *varlink_server; @@ -46,6 +60,9 @@ struct Manager { EVP_PKEY *private_key; /* actually a pair of private and public key */ Hashmap *public_keys; /* key name [char*] → publick key [EVP_PKEY*] */ + + RebalanceState rebalance_state; + usec_t rebalance_interval_usec; }; int manager_new(Manager **ret); @@ -59,6 +76,9 @@ int manager_augment_record_with_uid(Manager *m, UserRecord *hr); int manager_enqueue_rescan(Manager *m); int manager_enqueue_gc(Manager *m, Home *focus); +int manager_schedule_rebalance(Manager *m, bool immediately); +int manager_reschedule_rebalance(Manager *m); + int manager_verify_user_record(Manager *m, UserRecord *hr); int manager_acquire_key_pair(Manager *m); diff --git a/src/home/homework.c b/src/home/homework.c index 520b6b6f06..db6c15242f 100644 --- a/src/home/homework.c +++ b/src/home/homework.c @@ -1639,7 +1639,7 @@ static int home_update(UserRecord *h, UserRecord **ret) { return 0; } -static int home_resize(UserRecord *h, UserRecord **ret) { +static int home_resize(UserRecord *h, bool automatic, UserRecord **ret) { _cleanup_(home_setup_done) HomeSetup setup = HOME_SETUP_INIT; _cleanup_(password_cache_free) PasswordCache cache = {}; HomeSetupFlags flags = 0; @@ -1651,15 +1651,26 @@ static int home_resize(UserRecord *h, UserRecord **ret) { if (h->disk_size == UINT64_MAX) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "No target size specified, refusing."); - r = user_record_authenticate(h, h, &cache, /* strict_verify= */ true); - if (r < 0) - return r; - assert(r > 0); /* Insist that a password was verified */ + if (automatic) + /* In automatic mode don't want to ask the user for the password, hence load it from the kernel keyring */ + password_cache_load_keyring(h, &cache); + else { + /* In manual mode let's ensure the user is fully authenticated */ + r = user_record_authenticate(h, h, &cache, /* strict_verify= */ true); + if (r < 0) + return r; + assert(r > 0); /* Insist that a password was verified */ + } r = home_validate_update(h, &setup, &flags); if (r < 0) return r; + /* In automatic mode let's skip syncing identities, because we can't validate them, since we can't + * ask the user for reauthentication */ + if (automatic) + flags |= HOME_SETUP_RESIZE_DONT_SYNC_IDENTITIES; + switch (user_record_storage(h)) { case USER_LUKS: @@ -1931,8 +1942,10 @@ static int run(int argc, char *argv[]) { r = home_remove(home); else if (streq(argv[1], "update")) r = home_update(home, &new_home); - else if (streq(argv[1], "resize")) - r = home_resize(home, &new_home); + else if (streq(argv[1], "resize")) /* Resize on user request */ + r = home_resize(home, false, &new_home); + else if (streq(argv[1], "resize-auto")) /* Automatic resize */ + r = home_resize(home, true, &new_home); else if (streq(argv[1], "passwd")) r = home_passwd(home, &new_home); else if (streq(argv[1], "inspect")) diff --git a/src/home/user-record-util.c b/src/home/user-record-util.c index c4746cedc6..f4578783f3 100644 --- a/src/home/user-record-util.c +++ b/src/home/user-record-util.c @@ -1387,3 +1387,129 @@ int user_record_is_supported(UserRecord *hr, sd_bus_error *error) { return 0; } + +bool user_record_shall_rebalance(UserRecord *h) { + assert(h); + + if (user_record_rebalance_weight(h) == REBALANCE_WEIGHT_OFF) + return false; + + if (user_record_storage(h) != USER_LUKS) + return false; + + if (!path_startswith(user_record_image_path(h), get_home_root())) /* This is the only pool we rebalance in */ + return false; + + return true; +} + +int user_record_set_rebalance_weight(UserRecord *h, uint64_t weight) { + _cleanup_(json_variant_unrefp) JsonVariant *new_per_machine_array = NULL, *machine_id_variant = NULL, + *machine_id_array = NULL, *per_machine_entry = NULL; + _cleanup_free_ JsonVariant **array = NULL; + size_t idx = SIZE_MAX, n; + JsonVariant *per_machine; + sd_id128_t mid; + int r; + + assert(h); + + if (!h->json) + return -EUNATCH; + + r = sd_id128_get_machine(&mid); + if (r < 0) + return r; + + r = json_variant_new_id128(&machine_id_variant, mid); + if (r < 0) + return r; + + r = json_variant_new_array(&machine_id_array, (JsonVariant*[]) { machine_id_variant }, 1); + if (r < 0) + return r; + + per_machine = json_variant_by_key(h->json, "perMachine"); + if (per_machine) { + if (!json_variant_is_array(per_machine)) + return -EINVAL; + + n = json_variant_elements(per_machine); + + array = new(JsonVariant*, n + 1); + if (!array) + return -ENOMEM; + + for (size_t i = 0; i < n; i++) { + JsonVariant *m; + + array[i] = json_variant_by_index(per_machine, i); + + if (!json_variant_is_object(array[i])) + return -EINVAL; + + m = json_variant_by_key(array[i], "matchMachineId"); + if (!m) { + /* No machineId field? Let's ignore this, but invalidate what we found so far */ + idx = SIZE_MAX; + continue; + } + + if (json_variant_equal(m, machine_id_variant) || + json_variant_equal(m, machine_id_array)) { + /* Matches exactly what we are looking for. Let's use this */ + idx = i; + continue; + } + + r = per_machine_id_match(m, JSON_PERMISSIVE); + if (r < 0) + return r; + if (r > 0) + /* Also matches what we are looking for, but with a broader match. In this + * case let's ignore this entry, and add a new specific one to the end. */ + idx = SIZE_MAX; + } + + if (idx == SIZE_MAX) + idx = n++; /* Nothing suitable found, place new entry at end */ + else + per_machine_entry = json_variant_ref(array[idx]); + + } else { + array = new(JsonVariant*, 1); + if (!array) + return -ENOMEM; + + idx = 0; + n = 1; + } + + if (!per_machine_entry) { + r = json_variant_set_field(&per_machine_entry, "matchMachineId", machine_id_array); + if (r < 0) + return r; + } + + if (weight == REBALANCE_WEIGHT_UNSET) + r = json_variant_set_field(&per_machine_entry, "rebalanceWeight", NULL); /* set explicitly to NULL (so that the perMachine setting we are setting here can override the global setting) */ + else + r = json_variant_set_field_unsigned(&per_machine_entry, "rebalanceWeight", weight); + if (r < 0) + return r; + + assert(idx < n); + array[idx] = per_machine_entry; + + r = json_variant_new_array(&new_per_machine_array, array, n); + if (r < 0) + return r; + + r = json_variant_set_field(&h->json, "perMachine", new_per_machine_array); + if (r < 0) + return r; + + h->rebalance_weight = weight; + h->mask |= USER_RECORD_PER_MACHINE; + return 0; +} diff --git a/src/home/user-record-util.h b/src/home/user-record-util.h index 74f4a0eaab..508e2bdb8d 100644 --- a/src/home/user-record-util.h +++ b/src/home/user-record-util.h @@ -60,3 +60,6 @@ int user_record_bad_authentication(UserRecord *h); int user_record_ratelimit(UserRecord *h); int user_record_is_supported(UserRecord *hr, sd_bus_error *error); + +bool user_record_shall_rebalance(UserRecord *h); +int user_record_set_rebalance_weight(UserRecord *h, uint64_t weight); diff --git a/src/shared/user-record.h b/src/shared/user-record.h index 276b915067..9bd77cffeb 100644 --- a/src/shared/user-record.h +++ b/src/shared/user-record.h @@ -223,6 +223,7 @@ typedef enum AutoResizeMode { #define REBALANCE_WEIGHT_OFF UINT64_C(0) #define REBALANCE_WEIGHT_DEFAULT UINT64_C(100) +#define REBALANCE_WEIGHT_BACKING UINT64_C(20) #define REBALANCE_WEIGHT_MIN UINT64_C(1) #define REBALANCE_WEIGHT_MAX UINT64_C(10000) #define REBALANCE_WEIGHT_UNSET UINT64_MAX From 4950591627de2e859fb5fd1ae6d67e9c2e1b395e Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 4 Nov 2021 16:32:05 +0100 Subject: [PATCH 4/7] homed: add explicit API for requesting rebalancing too --- man/org.freedesktop.home1.xml | 7 +++++ src/home/homed-manager-bus.c | 22 +++++++++++++++ src/home/homed-manager.c | 34 +++++++++++++++++++++-- src/home/homed-manager.h | 6 ++++ src/home/org.freedesktop.home1.conf | 4 +++ src/libsystemd/sd-bus/bus-common-errors.c | 1 + src/libsystemd/sd-bus/bus-common-errors.h | 1 + 7 files changed, 72 insertions(+), 3 deletions(-) diff --git a/man/org.freedesktop.home1.xml b/man/org.freedesktop.home1.xml index b977e1b46f..537c373089 100644 --- a/man/org.freedesktop.home1.xml +++ b/man/org.freedesktop.home1.xml @@ -96,6 +96,7 @@ node /org/freedesktop/home1 { ReleaseHome(in s user_name); LockAllHomes(); DeactivateAllHomes(); + Rebalance(); properties: readonly a(sso) AutoLogin = [...]; }; @@ -159,6 +160,8 @@ node /org/freedesktop/home1 { + + @@ -346,6 +349,10 @@ node /org/freedesktop/home1 { DeactivateAllHomes() deactivates all home areas that are currently active. This is usually invoked automatically shortly before system shutdown. + + Rebalance() synchronously rebalances free disk space between home + areas. This only executes an operation if at least one home area using the LUKS2 backend is active and + has rebalancing enabled, and is otherwise a NOP. diff --git a/src/home/homed-manager-bus.c b/src/home/homed-manager-bus.c index 7ac5b8d0fc..31f82dc1dc 100644 --- a/src/home/homed-manager-bus.c +++ b/src/home/homed-manager-bus.c @@ -635,6 +635,27 @@ static int method_deactivate_all_homes(sd_bus_message *message, void *userdata, return sd_bus_reply_method_return(message, NULL); } +static int method_rebalance(sd_bus_message *message, void *userdata, sd_bus_error *error) { + Manager *m = userdata; + int r; + + assert(m); + + r = manager_schedule_rebalance(m, /* immediately= */ true); + if (r == 0) + return sd_bus_reply_method_errorf(message, BUS_ERROR_REBALANCE_NOT_NEEDED, "No home directories need rebalancing."); + if (r < 0) + return r; + + /* Keep a reference to this message, so that we can reply to it once we are done */ + r = set_ensure_put(&m->rebalance_queued_method_calls, &bus_message_hash_ops, message); + if (r < 0) + return log_error_errno(r, "Failed to track rebalance bus message: %m"); + + sd_bus_message_ref(message); + return 1; +} + static const sd_bus_vtable manager_vtable[] = { SD_BUS_VTABLE_START(0), @@ -843,6 +864,7 @@ static const sd_bus_vtable manager_vtable[] = { /* An operation that acts on all homes that allow it */ SD_BUS_METHOD("LockAllHomes", NULL, NULL, method_lock_all_homes, 0), SD_BUS_METHOD("DeactivateAllHomes", NULL, NULL, method_deactivate_all_homes, 0), + SD_BUS_METHOD("Rebalance", NULL, NULL, method_rebalance, 0), SD_BUS_VTABLE_END }; diff --git a/src/home/homed-manager.c b/src/home/homed-manager.c index 3851234a37..3e5bd9c29d 100644 --- a/src/home/homed-manager.c +++ b/src/home/homed-manager.c @@ -1985,6 +1985,24 @@ static int manager_rebalance_apply(Manager *m) { return c; } +static void manager_rebalance_reply_messages(Manager *m) { + int r; + + assert(m); + + for (;;) { + _cleanup_(sd_bus_message_unrefp) sd_bus_message *msg = + set_steal_first(m->rebalance_pending_method_calls); + + if (!msg) + break; + + r = sd_bus_reply_method_return(msg, NULL); + if (r < 0) + log_debug_errno(r, "Failed to reply to rebalance method call, ignoring: %m"); + } +} + static int manager_rebalance_now(Manager *m) { RebalanceState busy_state; /* the state to revert to when operation fails if busy */ int r; @@ -2006,6 +2024,13 @@ static int manager_rebalance_now(Manager *m) { /* First shrink large home dirs */ m->rebalance_state = REBALANCE_SHRINKING; busy_state = REBALANCE_PENDING; + + /* We are initiating the next rebalancing cycle now, let's make the queued methods + * calls the pending ones, and flush out any pending ones (which shouldn't exist at + * this time anyway) */ + set_clear(m->rebalance_pending_method_calls); + SWAP_TWO(m->rebalance_pending_method_calls, m->rebalance_queued_method_calls); + log_debug("Shrinking phase.."); break; @@ -2055,6 +2080,7 @@ static int manager_rebalance_now(Manager *m) { finish: /* Reset state and schedule next rebalance */ m->rebalance_state = REBALANCE_IDLE; + manager_rebalance_reply_messages(m); (void) manager_schedule_rebalance(m, /* immediately= */ false); return r; } @@ -2078,6 +2104,7 @@ int manager_schedule_rebalance(Manager *m, bool immediately) { /* Check if there are any records where rebalancing is requested */ if (!manager_shall_rebalance(m)) { log_debug("Not scheduling rebalancing, not needed."); + r = 0; /* report that we didn't schedule anything because nothing needed it */ goto turn_off; } @@ -2118,13 +2145,13 @@ int manager_schedule_rebalance(Manager *m, bool immediately) { m->rebalance_state = REBALANCE_PENDING; log_debug("Scheduled immediate rebalancing..."); - return 0; + return 1; /* report that we scheduled something */ } /* If we are told to schedule a rebalancing eventually, then do so only if we are not executing * anything yet. Also if we have something scheduled already, leave it in place */ if (!IN_SET(m->rebalance_state, REBALANCE_OFF, REBALANCE_IDLE)) - return 0; + return 1; /* report that there's already something scheduled */ if (m->rebalance_event_source) { r = sd_event_source_set_time_relative(m->rebalance_event_source, m->rebalance_interval_usec); @@ -2156,11 +2183,12 @@ int manager_schedule_rebalance(Manager *m, bool immediately) { m->rebalance_state = REBALANCE_WAITING; /* We managed to enqueue a timer event, we now wait until it fires */ log_debug("Scheduled rebalancing in %s...", FORMAT_TIMESPAN(m->rebalance_interval_usec, 0)); - return 0; + return 1; /* report that we scheduled something */ turn_off: m->rebalance_event_source = sd_event_source_disable_unref(m->rebalance_event_source); m->rebalance_state = REBALANCE_OFF; + manager_rebalance_reply_messages(m); return r; } diff --git a/src/home/homed-manager.h b/src/home/homed-manager.h index cf6d58b258..18e7542e13 100644 --- a/src/home/homed-manager.h +++ b/src/home/homed-manager.h @@ -63,6 +63,12 @@ struct Manager { RebalanceState rebalance_state; usec_t rebalance_interval_usec; + + /* In order to allow synchronous rebalance requests via bus calls we maintain two pools of bus + * messages: 'rebalance_pending_methods' are the method calls we are currently operating on and + * running a rebalancing operation for. 'rebalance_queued_method_calls' are the method calls that + * have been queued since then and that we'll operate on once we complete the current run. */ + Set *rebalance_pending_method_calls, *rebalance_queued_method_calls; }; int manager_new(Manager **ret); diff --git a/src/home/org.freedesktop.home1.conf b/src/home/org.freedesktop.home1.conf index 1975d5f1a2..bac6587b8e 100644 --- a/src/home/org.freedesktop.home1.conf +++ b/src/home/org.freedesktop.home1.conf @@ -125,6 +125,10 @@ send_interface="org.freedesktop.home1.Manager" send_member="LockAllHomes"/> + + Date: Thu, 4 Nov 2021 17:38:13 +0100 Subject: [PATCH 5/7] homectl: add new "homectl rebalance" command Let's add an explicit, synchronous command to request immediate rebalancing and wait for it. --- man/homectl.xml | 12 ++++++++++++ src/home/homectl.c | 28 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/man/homectl.xml b/man/homectl.xml index ca9225a1e0..5a06346d7b 100644 --- a/man/homectl.xml +++ b/man/homectl.xml @@ -946,6 +946,18 @@ scripts and such, but requires authentication with the user's credentials in order to be able to unlock the user's home directory. + + + rebalance + + Rebalance free disk space between active home areas and the backing storage. See + above. This executes no operation unless there's at least one + active LUKS2 home area that has disk space rebalancing enabled. This operation is synchronous: it + will only complete once disk space is rebalanced according to the rebalancing weights. Note that + rebalancing also takes place automatically in the background in regular intervals. Use this command + to synchronously ensure disk space is properly redistributed before initiating an operation requiring + large amounts of disk space. + diff --git a/src/home/homectl.c b/src/home/homectl.c index 0c69299c84..6057c04bae 100644 --- a/src/home/homectl.c +++ b/src/home/homectl.c @@ -2103,6 +2103,32 @@ static int deactivate_all_homes(int argc, char *argv[], void *userdata) { return 0; } +static int rebalance(int argc, char *argv[], void *userdata) { + _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL; + _cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL; + _cleanup_(sd_bus_flush_close_unrefp) sd_bus *bus = NULL; + int r; + + r = acquire_bus(&bus); + if (r < 0) + return r; + + r = bus_message_new_method_call(bus, &m, bus_mgr, "Rebalance"); + if (r < 0) + return bus_log_create_error(r); + + r = sd_bus_call(bus, m, HOME_SLOW_BUS_CALL_TIMEOUT_USEC, &error, NULL); + if (r < 0) { + if (sd_bus_error_has_name(&error, BUS_ERROR_REBALANCE_NOT_NEEDED)) + log_info("No homes needed rebalancing."); + else + return log_error_errno(r, "Failed to rebalance: %s", bus_error_message(&error, r)); + } else + log_info("Completed rebalancing."); + + return 0; +} + static int drop_from_identity(const char *field) { int r; @@ -2157,6 +2183,7 @@ static int help(int argc, char *argv[], void *userdata) { " unlock USER… Unlock a temporarily locked home area\n" " lock-all Lock all suitable home areas\n" " deactivate-all Deactivate all active home areas\n" + " rebalance Rebalance free space between home areas\n" " with USER [COMMAND…] Run shell or command with access to a home area\n" "\n%4$sOptions:%5$s\n" " -h --help Show this help\n" @@ -3746,6 +3773,7 @@ static int run(int argc, char *argv[]) { { "with", 2, VERB_ANY, 0, with_home }, { "lock-all", VERB_ANY, 1, 0, lock_all_homes }, { "deactivate-all", VERB_ANY, 1, 0, deactivate_all_homes }, + { "rebalance", VERB_ANY, 1, 0, rebalance }, {} }; From c313d04ab0ed39ed786cedc77cdf96355f8dbd06 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 25 Nov 2021 10:48:52 +0100 Subject: [PATCH 6/7] test: add test case for homed rebalancing logic --- test/units/testsuite-46.sh | 40 ++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/test/units/testsuite-46.sh b/test/units/testsuite-46.sh index 1ca5f03302..be70a39485 100755 --- a/test/units/testsuite-46.sh +++ b/test/units/testsuite-46.sh @@ -22,6 +22,8 @@ inspect() { # diff uses the grep BREs for pattern matching diff -I '^\s*Disk \(Size\|Free\|Floor\|Ceiling\):' /tmp/{a,b} rm /tmp/{a,b} + + homectl inspect --json=pretty "$USERNAME" } systemd-analyze log-level debug @@ -29,8 +31,8 @@ systemd-analyze log-target console systemctl service-log-level systemd-homed debug # Create a tmpfs to use as backing store for the home dir. That way we can enforce a size limit nicely. -mkdir -p /home-pool -mount -t tmpfs tmpfs /home-pool -o size=290M +mkdir -p /home +mount -t tmpfs tmpfs /home -o size=290M # we enable --luks-discard= since we run our tests in a tight VM, hence don't # needlessly pressure for storage. We also set the cheapest KDF, since we don't @@ -38,7 +40,7 @@ mount -t tmpfs tmpfs /home-pool -o size=290M NEWPASSWORD=xEhErW0ndafV4s homectl create test-user \ --disk-size=min \ --luks-discard=yes \ - --image-path=/home-pool/test-user.home \ + --image-path=/home/test-user.home \ --luks-pbkdf-type=pbkdf2 \ --luks-pbkdf-time-cost=1ms inspect test-user @@ -110,8 +112,38 @@ if ! systemd-detect-virt -cq ; then PASSWORD=xEhErW0ndafV4s homectl resize test-user 256M inspect test-user - homectl deactivate test-user + # minimize again + PASSWORD=xEhErW0ndafV4s homectl resize test-user min inspect test-user + + # Increase space, so that we can reasonably rebalance free space between to home dirs + mount /home -o remount,size=800M + + # create second user + NEWPASSWORD=uuXoo8ei homectl create test-user2 \ + --disk-size=min \ + --luks-discard=yes \ + --image-path=/home/test-user2.home \ + --luks-pbkdf-type=pbkdf2 \ + --luks-pbkdf-time-cost=1ms + inspect test-user2 + + # activate second user + PASSWORD=uuXoo8ei homectl activate test-user2 + inspect test-user2 + + # set second user's rebalance weight to 100 + PASSWORD=uuXoo8ei homectl update test-user2 --rebalance-weight=100 + inspect test-user2 + + # set first user's rebalance weight to quarter of that of the second + PASSWORD=xEhErW0ndafV4s homectl update test-user --rebalance-weight=25 + inspect test-user + + # synchronously rebalance + homectl rebalance + inspect test-user + inspect test-user2 fi PASSWORD=xEhErW0ndafV4s homectl with test-user -- test ! -f /home/test-user/xyz From 6d975fe7b3471fcc5d6c9dee7655d60925ba9d2c Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Fri, 29 Oct 2021 09:56:22 +0200 Subject: [PATCH 7/7] update TODO --- TODO | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/TODO b/TODO index 3c6d76266c..14431d3a3b 100644 --- a/TODO +++ b/TODO @@ -1274,8 +1274,6 @@ Features: - when user tries to log into record signed by unrecognized key, automatically add key to our chain after polkit auth - rollback when resize fails mid-operation - GNOME's side for forget key on suspend (requires rework so that lock screen runs outside of uid) - - resize on login? - - shrink fs on logout? - update LUKS password on login if we find there's a password that unlocks the JSON record but not the LUKS device. - create on activate? - properties: icon url?, preferred session type?, administrator bool (which translates to 'wheel' membership)?, address?, telephone?, vcard?, samba stuff?, parental controls? @@ -1297,16 +1295,10 @@ Features: - make slice for users configurable (requires logind rework) - logind: populate auto-login list bus property from PKCS#11 token - when determining state of a LUKS home directory, check DM suspended sysfs file - - introduce API for "making room", that grows/shrinks home directory - according to elastic parameters, discards blocks, and removes additional snapshots. Call it - either from UI when disk space gets low - when homed is in use, maybe start the user session manager in a mount namespace with MS_SLAVE, so that mounts propagate down but not up - eg, user A setting up a backup volume doesn't mean user B sees it - use credentials logic/TPM2 logic to store homed signing key - - during login resize fs automatically towards size goal. Specifically, - resize to diskSize if possible, but leave a certain amount (configured by a - new value diskLeaveFreeSize) of space free on the backing fs. - permit multiple user record signing keys to be used locally, and pick the right one for signing records automatically depending on a pre-existing signature @@ -1327,6 +1319,10 @@ Features: fallback logic to get a regular user created on uninitialized systems. - store PKCS#11 + FIDO2 token info in LUKS2 header, compatible with systemd-cryptsetup, so that it can unlock homed volumes + - maybe make all *.home files owned by `systemd-home` user or so, so that we + can easily set overall quota for all users + - on login, if we can't fallocate initially, but rebalance is on, then allow + login in discard mode, then immediately rebalance, then turn off discard * add a new switch --auto-definitions=yes/no or so to systemd-repart. If specified, synthesize a definition automatically if we can: enlarge last