importd: add support for downloading OCI images (#39621)

This adds the ability to download OCI images via importd. 

Not a fan of the OCI format tbh, in particular its security properties
are a bit sad. But I guess it exists and is very popular, hence we might
as well add support for it, even if it comes at much weaker security
properties than DDIs.

Fixes #36447

src/core/dbus-execute.c

@@ -1258,6 +1258,7 @@ const sd_bus_vtable bus_exec_vtable[] = {
         SD_BUS_PROPERTY("RootHashSignaturePath", "s", NULL, offsetof(ExecContext, root_hash_sig_path), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_PROPERTY("RootVerity", "s", NULL, offsetof(ExecContext, root_verity), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_PROPERTY("RootEphemeral", "b", bus_property_get_bool, offsetof(ExecContext, root_ephemeral), SD_BUS_VTABLE_PROPERTY_CONST),
+        SD_BUS_PROPERTY("RootMStack", "s", NULL, offsetof(ExecContext, root_mstack), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_PROPERTY("ExtensionDirectories", "as", NULL, offsetof(ExecContext, extension_directories), SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_PROPERTY("ExtensionImages", "a(sba(ss))", property_get_extension_images, 0, SD_BUS_VTABLE_PROPERTY_CONST),
         SD_BUS_PROPERTY("MountImages", "a(ssba(ss))", property_get_mount_images, 0, SD_BUS_VTABLE_PROPERTY_CONST),
@@ -1882,6 +1883,9 @@ int bus_exec_context_set_transient_property(
         if (streq(name, "RootImage"))
                 return bus_set_transient_path(u, name, &c->root_image, message, flags, reterr_error);
 
+        if (streq(name, "RootMStack"))
+                return bus_set_transient_path(u, name, &c->root_mstack, message, flags, reterr_error);
+
         if (streq(name, "RootImageOptions")) {
                 _cleanup_(mount_options_free_allp) MountOptions *options = NULL;
                 _cleanup_free_ char *format_str = NULL;

src/core/exec-invoke.c

@@ -14,6 +14,7 @@
 #include <unistd.h>
 
 #include "sd-messages.h"
+#include "sd-varlink.h"
 
 #include "apparmor-util.h"      /* IWYU pragma: keep */
 #include "argv-util.h"
@@ -55,8 +56,10 @@
 #include "mkdir-label.h"
 #include "mount-util.h"
 #include "mountpoint-util.h"
+#include "mstack.h"
 #include "namespace-util.h"
 #include "nsflags.h"
+#include "nsresource.h"
 #include "open-file.h"
 #include "osc-context.h"
 #include "pam-util.h"
@@ -81,6 +84,7 @@
 #include "strv.h"
 #include "strxcpyx.h"
 #include "terminal-util.h"
+#include "uid-range.h"
 #include "user-util.h"
 #include "utmp-wtmp.h"
 #include "vpick.h"
@@ -2395,11 +2399,14 @@ static int setup_private_users_child(int unshare_ready_fd, const char *uid_map,
 }
 
 static int setup_private_users(
+                sd_varlink *nsresource_link,
                 PrivateUsers private_users,
-                uid_t ouid,
-                gid_t ogid,
-                uid_t uid,
-                gid_t gid,
+                uid_t saved_uid,    /* service manager uid */
+                gid_t saved_gid,    /* service manager gid */
+                uid_t *uid,         /* unit uid (seen from inside) [input+output] */
+                gid_t *gid,         /* unit gid (ditto)            [input+output] */
+                uid_t *outside_uid, /* uid seen from the outside (which is the same as *uid, except of userns is used) */
+                gid_t *outside_gid, /* gid seen from the outside (similar) */
                 bool allow_setgroups) {
 
         _cleanup_free_ char *uid_map = NULL, *gid_map = NULL;
@@ -2410,6 +2417,11 @@ static int setup_private_users(
         ssize_t n;
         int r;
 
+        assert(uid);
+        assert(gid);
+        assert(outside_uid);
+        assert(outside_gid);
+
         /* Set up a user namespace and map the original UID/GID (IDs from before any user or group changes, i.e.
          * the IDs from the user or system manager(s)) to itself, the selected UID/GID to itself, and everything else to
          * nobody. In order to be able to write this mapping we need CAP_SETUID in the original user namespace, which
@@ -2425,6 +2437,45 @@ static int setup_private_users(
         case PRIVATE_USERS_NO:
                 return 0; /* Early exit */
 
+        case PRIVATE_USERS_MANAGED: {
+                if (uid_is_valid(*uid) || uid_is_valid(*gid))
+                        return log_debug_errno(SYNTHETIC_ERRNO(EPERM), "When allocating dynamic user namespace range, target UID/GID must be root, refusing.");
+
+                _cleanup_close_ int userns_fd = nsresource_allocate_userns(
+                                nsresource_link,
+                                /* name= */ NULL,
+                                NSRESOURCE_UIDS_64K);
+                if (userns_fd < 0)
+                        return userns_fd;
+
+                _cleanup_(uid_range_freep) UIDRange *uid_range = NULL;
+                r = uid_range_load_userns_by_fd(userns_fd, UID_RANGE_USERNS_OUTSIDE, &uid_range);
+                if (r < 0)
+                        return log_debug_errno(r, "Failed to read outside userns UID range: %m");
+
+                _cleanup_(uid_range_freep) UIDRange *gid_range = NULL;
+                r = uid_range_load_userns_by_fd(userns_fd, GID_RANGE_USERNS_OUTSIDE, &gid_range);
+                if (r < 0)
+                        return log_debug_errno(r, "Failed to read outside userns GID range: %m");
+
+                if (setns(userns_fd, CLONE_NEWUSER) < 0)
+                        return log_debug_errno(errno, "Failed to join freshly allocated user namespace: %m");
+
+                /* In "managed" mode the originating UID is not mapped hence we need to explicitly become root in the new userns now. */
+                r = reset_uid_gid();
+                if (r < 0)
+                        return log_debug_errno(r, "Failed to reset UID/GID to root: %m");
+
+                /* Now report are new UIDs/GIDS, both inside and outside */
+                *uid = 0;
+                *gid = 0;
+
+                *outside_uid = uid_range_base(uid_range);
+                *outside_gid = uid_range_base(gid_range);
+
+                return 1; /* Early exit */
+        }
+
         case PRIVATE_USERS_IDENTITY:
                 uid_map = strdup("0 0 65536\n");
                 if (!uid_map)
@@ -2445,15 +2496,15 @@ static int setup_private_users(
 
         case PRIVATE_USERS_SELF:
                 /* Can only set up multiple mappings with CAP_SETUID. */
-                if (uid_is_valid(uid) && uid != ouid && have_effective_cap(CAP_SETUID) > 0)
+                if (uid_is_valid(*uid) && *uid != saved_uid && have_effective_cap(CAP_SETUID) > 0)
                         r = asprintf(&uid_map,
                                      UID_FMT " " UID_FMT " 1\n"     /* Map $OUID → $OUID */
                                      UID_FMT " " UID_FMT " 1\n",    /* Map $UID → $UID */
-                                     ouid, ouid, uid, uid);
+                                     saved_uid, saved_uid, *uid, *uid);
                 else
                         r = asprintf(&uid_map,
                                      UID_FMT " " UID_FMT " 1\n",    /* Map $OUID → $OUID */
-                                     ouid, ouid);
+                                     saved_uid, saved_uid);
                 if (r < 0)
                         return -ENOMEM;
 
@@ -2479,15 +2530,15 @@ static int setup_private_users(
 
         case PRIVATE_USERS_SELF:
                 /* Can only set up multiple mappings with CAP_SETGID. */
-                if (gid_is_valid(gid) && gid != ogid && have_effective_cap(CAP_SETGID) > 0)
+                if (gid_is_valid(*gid) && *gid != saved_gid && have_effective_cap(CAP_SETGID) > 0)
                         r = asprintf(&gid_map,
                                      GID_FMT " " GID_FMT " 1\n"     /* Map $OGID → $OGID */
                                      GID_FMT " " GID_FMT " 1\n",    /* Map $GID → $GID */
-                                     ogid, ogid, gid, gid);
+                                     saved_gid, saved_gid, *gid, *gid);
                 else
                         r = asprintf(&gid_map,
                                      GID_FMT " " GID_FMT " 1\n",    /* Map $OGID -> $OGID */
-                                     ogid, ogid);
+                                     saved_gid, saved_gid);
                 if (r < 0)
                         return -ENOMEM;
                 break;
@@ -3487,8 +3538,7 @@ static int compile_symlinks(
 
 static bool insist_on_sandboxing(
                 const ExecContext *context,
-                const char *root_dir,
-                const char *root_image,
+                const PinnedResource *rootfs,
                 const BindMount *bind_mounts,
                 size_t n_bind_mounts) {
 
@@ -3502,7 +3552,7 @@ static bool insist_on_sandboxing(
         if (context->n_temporary_filesystems > 0)
                 return true;
 
-        if (root_dir || root_image || context->root_directory_as_fd)
+        if (pinned_resource_is_set(rootfs))
                 return true;
 
         if (context->n_mount_images > 0)
@@ -3529,8 +3579,7 @@ static bool insist_on_sandboxing(
 static int setup_ephemeral(
                 const ExecContext *context,
                 ExecRuntime *runtime,
-                char **root_image,            /* both input and output! modified if ephemeral logic enabled */
-                char **root_directory,        /* ditto */
+                PinnedResource *rootfs,  /* both input and output! modified if ephemeral logic enabled */
                 char **reterr_path) {
 
         _cleanup_close_ int fd = -EBADF;
@@ -3538,12 +3587,10 @@ static int setup_ephemeral(
         int r;
 
         assert(context);
-        assert(!context->root_directory_as_fd);
         assert(runtime);
-        assert(root_image);
-        assert(root_directory);
+        assert(rootfs);
 
-        if (!*root_image && !*root_directory)
+        if (!rootfs->image && !rootfs->directory)
                 return 0;
 
         if (!runtime->ephemeral_copy)
@@ -3569,32 +3616,32 @@ static int setup_ephemeral(
         if (fd != -EAGAIN)
                 return log_debug_errno(fd, "Failed to receive file descriptor queued on ephemeral storage socket: %m");
 
-        if (*root_image) {
-                log_debug("Making ephemeral copy of %s to %s", *root_image, new_root);
+        if (rootfs->image) {
+                log_debug("Making ephemeral copy of %s to %s", rootfs->image, new_root);
 
-                fd = copy_file(*root_image, new_root, O_EXCL, 0600,
+                fd = copy_file(rootfs->image, new_root, O_EXCL, 0600,
                                COPY_LOCK_BSD|COPY_REFLINK|COPY_CRTIME|COPY_NOCOW_AFTER);
                 if (fd < 0) {
-                        *reterr_path = strdup(*root_image);
+                        *reterr_path = strdup(rootfs->image);
                         return log_debug_errno(fd, "Failed to copy image %s to %s: %m",
-                                               *root_image, new_root);
+                                               rootfs->image, new_root);
                 }
         } else {
-                assert(*root_directory);
+                assert(rootfs->directory);
 
-                log_debug("Making ephemeral snapshot of %s to %s", *root_directory, new_root);
+                log_debug("Making ephemeral snapshot of %s to %s", rootfs->directory, new_root);
 
                 fd = btrfs_subvol_snapshot_at(
-                                AT_FDCWD, *root_directory,
+                                AT_FDCWD, rootfs->directory,
                                 AT_FDCWD, new_root,
                                 BTRFS_SNAPSHOT_FALLBACK_COPY |
                                 BTRFS_SNAPSHOT_FALLBACK_DIRECTORY |
                                 BTRFS_SNAPSHOT_RECURSIVE |
                                 BTRFS_SNAPSHOT_LOCK_BSD);
                 if (fd < 0) {
-                        *reterr_path = strdup(*root_directory);
+                        *reterr_path = strdup(rootfs->directory);
                         return log_debug_errno(fd, "Failed to snapshot directory %s to %s: %m",
-                                               *root_directory, new_root);
+                                               rootfs->directory, new_root);
                 }
         }
 
@@ -3602,11 +3649,14 @@ static int setup_ephemeral(
         if (r < 0)
                 return log_debug_errno(r, "Failed to queue file descriptor on ephemeral storage socket: %m");
 
-        if (*root_image)
-                free_and_replace(*root_image, new_root);
-        else {
-                assert(*root_directory);
-                free_and_replace(*root_directory, new_root);
+        if (rootfs->image) {
+                free_and_replace(rootfs->image, new_root);
+                close_and_replace(rootfs->image_fd, fd);
+        } else {
+                assert(rootfs->directory);
+
+                free_and_replace(rootfs->directory, new_root);
+                close_and_replace(rootfs->directory_fd, fd);
         }
 
         return 1;
@@ -3660,20 +3710,35 @@ static int verity_settings_prepare(
         return 0;
 }
 
-static int pick_versions(
+static int pin_rootfs(
                 const ExecContext *context,
                 const ExecParameters *params,
-                char **ret_root_image,
-                char **ret_root_directory,
+                PinnedResource *ret,
                 char **reterr_path) {
 
         int r;
 
         assert(context);
-        assert(!context->root_directory_as_fd);
         assert(params);
-        assert(ret_root_image);
-        assert(ret_root_directory);
+        assert(ret);
+
+        if (!FLAGS_SET(params->flags, EXEC_APPLY_CHROOT)) {
+                *ret = PINNED_RESOURCE_NULL;
+                return 0;
+        }
+
+        if (context->root_directory_as_fd) {
+                _cleanup_close_ int fd = fcntl(params->root_directory_fd, F_DUPFD_CLOEXEC, 3);
+                if (fd < 0)
+                        return log_debug_errno(errno, "Failed to duplicate root directory fd: %m");
+
+                *ret = (PinnedResource) {
+                        .directory_fd = TAKE_FD(fd),
+                        .image_fd = -EBADF,
+                };
+
+                return 1;
+        }
 
         if (context->root_image) {
                 _cleanup_(pick_result_done) PickResult result = PICK_RESULT_NULL;
@@ -3695,8 +3760,24 @@ static int pick_versions(
                         return log_debug_errno(SYNTHETIC_ERRNO(ENOENT), "No matching entry in .v/ directory %s found.", context->root_image);
                 }
 
-                *ret_root_image = TAKE_PTR(result.path);
-                *ret_root_directory = NULL;
+                /* path_pick() returns us an O_PATH fd, let's turn this into a fully opened file, because
+                 * mountfsd will want this later, and it wants a fully opened fd, so that security checks
+                 * have been passed */
+                _cleanup_close_ int reopened_fd = -EBADF;
+                reopened_fd = fd_reopen(result.fd, O_CLOEXEC|O_NONBLOCK|O_NOCTTY|O_RDWR);
+                if (ERRNO_IS_NEG_FS_WRITE_REFUSED(reopened_fd))
+                        reopened_fd = fd_reopen(result.fd, O_CLOEXEC|O_NONBLOCK|O_NOCTTY|O_RDONLY);
+                if (reopened_fd < 0) {
+                        *reterr_path = strdup(context->root_image);
+                        return log_debug_errno(reopened_fd, "Failed to open image '%s': %m", context->root_image);
+                }
+
+                *ret = (PinnedResource) {
+                        .image = TAKE_PTR(result.path),
+                        .image_fd = TAKE_FD(reopened_fd),
+                        .directory_fd = -EBADF,
+                };
+
                 return r;
         }
 
@@ -3720,12 +3801,53 @@ static int pick_versions(
                         return log_debug_errno(SYNTHETIC_ERRNO(ENOENT), "No matching entry in .v/ directory %s found.", context->root_directory);
                 }
 
-                *ret_root_image = NULL;
-                *ret_root_directory = TAKE_PTR(result.path);
+                *ret = (PinnedResource) {
+                        .directory = TAKE_PTR(result.path),
+                        .directory_fd = TAKE_FD(result.fd),
+                        .image_fd = -EBADF,
+                };
+
                 return r;
         }
 
-        *ret_root_image = *ret_root_directory = NULL;
+        if (context->root_mstack) {
+                _cleanup_(pick_result_done) PickResult result = PICK_RESULT_NULL;
+
+                r = path_pick(/* toplevel_path= */ NULL,
+                              /* toplevel_fd= */ AT_FDCWD,
+                              context->root_mstack,
+                              pick_filter_image_mstack,
+                              /* n_filters= */ 1,
+                              PICK_ARCHITECTURE|PICK_TRIES|PICK_RESOLVE,
+                              &result);
+                if (r < 0) {
+                        *reterr_path = strdup(context->root_mstack);
+                        return r;
+                }
+
+                if (!result.path) {
+                        *reterr_path = strdup(context->root_mstack);
+                        return log_debug_errno(SYNTHETIC_ERRNO(ENOENT), "No matching entry in .v/ directory %s found.", context->root_mstack);
+                }
+
+                _cleanup_(mstack_freep) MStack *mstack = NULL;
+                r = mstack_load(result.path, result.fd, &mstack);
+                if (r < 0) {
+                        *reterr_path = TAKE_PTR(result.path);
+                        return r;
+                }
+
+                *ret = (PinnedResource) {
+                        .mstack = TAKE_PTR(result.path),
+                        .mstack_loaded = TAKE_PTR(mstack),
+                        .image_fd = -EBADF,
+                        .directory_fd = -EBADF,
+                };
+
+                return r;
+        }
+
+        *ret = PINNED_RESOURCE_NULL;
         return 0;
 }
 
@@ -3733,7 +3855,8 @@ static int apply_mount_namespace(
                 ExecCommandFlags command_flags,
                 const ExecContext *context,
                 const ExecParameters *params,
-                ExecRuntime *runtime,
+                const ExecRuntime *runtime,
+                const PinnedResource *rootfs,
                 const char *memory_pressure_path,
                 bool needs_sandboxing,
                 uid_t exec_directory_uid,
@@ -3741,13 +3864,14 @@ static int apply_mount_namespace(
                 PidRef *bpffs_pidref,
                 int bpffs_socket_fd,
                 int bpffs_errno_pipe,
+                sd_varlink *mountfsd_link,
                 char **reterr_path) {
 
         _cleanup_(verity_settings_done) VeritySettings verity = VERITY_SETTINGS_DEFAULT;
         _cleanup_strv_free_ char **empty_directories = NULL, **symlinks = NULL,
                         **read_write_paths_cleanup = NULL;
         _cleanup_free_ char *creds_path = NULL, *incoming_dir = NULL, *propagate_dir = NULL,
-                *private_namespace_dir = NULL, *host_os_release_stage = NULL, *root_image = NULL, *root_dir = NULL;
+                *private_namespace_dir = NULL, *host_os_release_stage = NULL;
         const char *tmp_dir = NULL, *var_tmp_dir = NULL;
         char **read_write_paths;
         bool setup_os_release_symlink;
@@ -3761,26 +3885,6 @@ static int apply_mount_namespace(
 
         CLEANUP_ARRAY(bind_mounts, n_bind_mounts, bind_mount_free_many);
 
-        if (params->flags & EXEC_APPLY_CHROOT && !context->root_directory_as_fd) {
-                r = pick_versions(
-                                context,
-                                params,
-                                &root_image,
-                                &root_dir,
-                                reterr_path);
-                if (r < 0)
-                        return r;
-
-                r = setup_ephemeral(
-                                context,
-                                runtime,
-                                &root_image,
-                                &root_dir,
-                                reterr_path);
-                if (r < 0)
-                        return r;
-        }
-
         r = compile_bind_mounts(context, params, exec_directory_uid, exec_directory_gid, &bind_mounts, &n_bind_mounts, &empty_directories);
         if (r < 0)
                 return r;
@@ -3819,7 +3923,7 @@ static int apply_mount_namespace(
         }
 
         /* Symlinks (exec dirs, os-release) are set up after other mounts, before they are made read-only. */
-        setup_os_release_symlink = needs_sandboxing && exec_context_get_effective_mount_apivfs(context) && (root_dir || root_image);
+        setup_os_release_symlink = needs_sandboxing && exec_context_get_effective_mount_apivfs(context) && pinned_resource_is_set(rootfs);
         r = compile_symlinks(context, params, setup_os_release_symlink, &symlinks);
         if (r < 0)
                 return r;
@@ -3866,10 +3970,10 @@ static int apply_mount_namespace(
                         return -ENOMEM;
         }
 
-        if (root_image) {
+        if (rootfs->image) {
                 r = verity_settings_prepare(
                         &verity,
-                        root_image,
+                        rootfs->image,
                         &context->root_hash, context->root_hash_path,
                         &context->root_hash_sig, context->root_hash_sig_path,
                         context->root_verity);
@@ -3880,9 +3984,7 @@ static int apply_mount_namespace(
         NamespaceParameters parameters = {
                 .runtime_scope = params->runtime_scope,
 
-                .root_directory = root_dir,
-                .root_image = root_image,
-                .root_directory_fd = params->flags & EXEC_APPLY_CHROOT ? params->root_directory_fd : -EBADF,
+                .rootfs = rootfs,
                 .root_image_options = context->root_image_options,
                 .root_image_policy = context->root_image_policy ?: &image_policy_service,
 
@@ -3930,7 +4032,7 @@ static int apply_mount_namespace(
                 /* If DynamicUser=no and RootDirectory= is set then lets pass a relaxed sandbox info,
                  * otherwise enforce it, don't ignore protected paths and fail if we are enable to apply the
                  * sandbox inside the mount namespace. */
-                .ignore_protect_paths = !needs_sandboxing && !context->dynamic_user && root_dir,
+                .ignore_protect_paths = !needs_sandboxing && !context->dynamic_user && pinned_resource_is_set(rootfs),
 
                 .protect_control_groups = needs_sandboxing ? exec_get_protect_control_groups(context) : PROTECT_CONTROL_GROUPS_NO,
                 .protect_kernel_tunables = needs_sandboxing && context->protect_kernel_tunables,
@@ -3956,10 +4058,13 @@ static int apply_mount_namespace(
                 .protect_proc = needs_sandboxing ? context->protect_proc : PROTECT_PROC_DEFAULT,
                 .proc_subset = needs_sandboxing ? context->proc_subset : PROC_SUBSET_ALL,
                 .private_bpf = needs_sandboxing ? context->private_bpf : PRIVATE_BPF_NO,
+                .private_users = needs_sandboxing ? context->private_users : PRIVATE_USERS_NO,
 
                 .bpffs_pidref = bpffs_pidref,
                 .bpffs_socket_fd = bpffs_socket_fd,
                 .bpffs_errno_pipe = bpffs_errno_pipe,
+
+                .mountfsd_link = mountfsd_link,
         };
 
         r = setup_namespace(&parameters, reterr_path);
@@ -3970,17 +4075,18 @@ static int apply_mount_namespace(
         if (r == -ENOANO) {
                 if (insist_on_sandboxing(
                                     context,
-                                    root_dir, root_image,
+                                    rootfs,
                                     bind_mounts,
                                     n_bind_mounts))
                         return log_debug_errno(SYNTHETIC_ERRNO(EOPNOTSUPP),
                                                "Failed to set up namespace, and refusing to continue since "
                                                "the selected namespacing options alter mount environment non-trivially.\n"
-                                               "Bind mounts: %zu, temporary filesystems: %zu, root directory: %s, root image: %s, dynamic user: %s",
+                                               "Bind mounts: %zu, temporary filesystems: %zu, root directory: %s, root image: %s, root mstack: %s, dynamic user: %s",
                                                n_bind_mounts,
                                                context->n_temporary_filesystems,
-                                               yes_no(root_dir),
-                                               yes_no(root_image),
+                                               yes_no(rootfs->directory_fd >= 0),
+                                               yes_no(rootfs->image_fd >= 0),
+                                               yes_no(!!rootfs->mstack_loaded),
                                                yes_no(context->dynamic_user));
 
                 log_debug("Failed to set up namespace, assuming containerized execution and ignoring.");
@@ -4530,10 +4636,8 @@ static bool exec_needs_cap_sys_admin(const ExecContext *context, const ExecParam
                context->bind_log_sockets > 0 ||
                context->n_bind_mounts > 0 ||
                context->n_temporary_filesystems > 0 ||
-               context->root_directory ||
-               context->root_directory_as_fd ||
+               exec_context_with_rootfs_strict(context) ||
                !strv_isempty(context->extension_directories) ||
-               context->root_image ||
                context->n_mount_images > 0 ||
                context->n_extension_images > 0 ||
                context->protect_system != PROTECT_SYSTEM_NO ||
@@ -4605,7 +4709,8 @@ static bool exec_namespace_is_delegated(
 static int setup_delegated_namespaces(
                 const ExecContext *context,
                 ExecParameters *params,
-                ExecRuntime *runtime,
+                const ExecRuntime *runtime,
+                const PinnedResource *rootfs,
                 bool delegate,
                 const char *memory_pressure_path,
                 uid_t uid,
@@ -4616,6 +4721,7 @@ static int setup_delegated_namespaces(
                 PidRef *bpffs_pidref,
                 int bpffs_socket_fd,
                 int bpffs_errno_pipe,
+                sd_varlink *mountfsd_link,
                 int *reterr_exit_status) {
 
         int r;
@@ -4630,6 +4736,7 @@ static int setup_delegated_namespaces(
         assert(context);
         assert(params);
         assert(runtime);
+        assert(rootfs);
         assert(reterr_exit_status);
 
         if (exec_needs_network_namespace(context) &&
@@ -4730,18 +4837,21 @@ static int setup_delegated_namespaces(
             exec_namespace_is_delegated(context, params, have_cap_sys_admin, CLONE_NEWNS) == delegate) {
                 _cleanup_free_ char *error_path = NULL;
 
-                r = apply_mount_namespace(command->flags,
-                                          context,
-                                          params,
-                                          runtime,
-                                          memory_pressure_path,
-                                          needs_sandboxing,
-                                          uid,
-                                          gid,
-                                          bpffs_pidref,
-                                          bpffs_socket_fd,
-                                          bpffs_errno_pipe,
-                                          &error_path);
+                r = apply_mount_namespace(
+                                command->flags,
+                                context,
+                                params,
+                                runtime,
+                                rootfs,
+                                memory_pressure_path,
+                                needs_sandboxing,
+                                uid,
+                                gid,
+                                bpffs_pidref,
+                                bpffs_socket_fd,
+                                bpffs_errno_pipe,
+                                mountfsd_link,
+                                &error_path);
                 if (r < 0) {
                         *reterr_exit_status = EXIT_NAMESPACE;
                         return log_error_errno(r, "Failed to set up mount namespacing%s%s: %m",
@@ -5093,10 +5203,10 @@ int exec_invoke(
 #if HAVE_SECCOMP
         uint64_t saved_bset = 0;
 #endif
-        uid_t saved_uid = getuid();
-        gid_t saved_gid = getgid();
-        uid_t uid = UID_INVALID;
-        gid_t gid = GID_INVALID;
+        /* saved_uid → where we started from; uid → where we ended up in; outside_uid → where we ended up in,
+         * but seen from the outside (covers userns mappings) */
+        uid_t uid = UID_INVALID, saved_uid = getuid(), outside_uid = UID_INVALID;
+        gid_t gid = GID_INVALID, saved_gid = getgid(), outside_gid = GID_INVALID;
         int secure_bits;
         _cleanup_free_ gid_t *gids = NULL, *gids_after_pam = NULL;
         int ngids = 0, ngids_after_pam = 0;
@@ -5266,7 +5376,9 @@ int exec_invoke(
                 return log_error_errno(errno, "Failed to update environment: %m");
         }
 
-        if (context->dynamic_user && runtime->dynamic_creds) {
+        if (exec_context_get_effective_private_users(context, params) == PRIVATE_USERS_MANAGED)
+                log_debug("Running with a managed user namespace, not initializing UIDs/GIDs.");
+        else if (context->dynamic_user && runtime->dynamic_creds) {
                 _cleanup_strv_free_ char **suggested_paths = NULL;
 
                 /* On top of that, make sure we bypass our own NSS module nss-systemd comprehensively for any NSS
@@ -5348,20 +5460,22 @@ int exec_invoke(
                 }
         }
 
-        /* Initialize user supplementary groups and get SupplementaryGroups= ones */
-        ngids = get_supplementary_groups(context, username, gid, &gids);
-        if (ngids < 0) {
-                *exit_status = EXIT_GROUP;
-                return log_error_errno(ngids, "Failed to determine supplementary groups: %m");
-        }
+        if (exec_context_get_effective_private_users(context, params) != PRIVATE_USERS_MANAGED) {
+                /* Initialize user supplementary groups and get SupplementaryGroups= ones */
+                ngids = get_supplementary_groups(context, username, gid, &gids);
+                if (ngids < 0) {
+                        *exit_status = EXIT_GROUP;
+                        return log_error_errno(ngids, "Failed to determine supplementary groups: %m");
+                }
 
-        r = send_user_lookup(params->unit_id, params->user_lookup_fd, uid, gid);
-        if (r < 0) {
-                *exit_status = EXIT_USER;
-                return log_error_errno(r, "Failed to send user credentials to PID1: %m");
-        }
+                r = send_user_lookup(params->unit_id, params->user_lookup_fd, uid, gid);
+                if (r < 0) {
+                        *exit_status = EXIT_USER;
+                        return log_error_errno(r, "Failed to send user credentials to PID1: %m");
+                }
 
-        params->user_lookup_fd = safe_close(params->user_lookup_fd);
+                params->user_lookup_fd = safe_close(params->user_lookup_fd);
+        }
 
         r = acquire_home(context, &pwent_home, &home_buffer);
         if (r < 0) {
@@ -5697,6 +5811,24 @@ int exec_invoke(
                 }
         }
 
+        _cleanup_(sd_varlink_unrefp) sd_varlink *mountfsd_link = NULL, *nsresource_link = NULL;
+        if (needs_sandboxing &&
+            exec_context_get_effective_private_users(context, params) == PRIVATE_USERS_MANAGED) {
+
+                /* In managed mode we need to allocate a userns via nsresource, and then assign mounts to
+                 * it. We must do so with our original privileges (since after creating the userns, we might
+                 * simply not have the necessary privs for the IPC calls anymore), hence do this here, ahead
+                 * of time. */
+
+                r = mountfsd_connect(&mountfsd_link);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to connect to mountfsd: %m");
+
+                r = nsresource_connect(&nsresource_link);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to connect to nsresourced: %m");
+        }
+
         needs_mount_namespace = exec_needs_mount_namespace(context, params, runtime);
 
         for (ExecDirectoryType dt = 0; dt < _EXEC_DIRECTORY_TYPE_MAX; dt++) {
@@ -5867,6 +5999,20 @@ int exec_invoke(
                 }
         }
 
+        _cleanup_(pinned_resource_done) PinnedResource rootfs = PINNED_RESOURCE_NULL;
+        _cleanup_free_ char *error_path = NULL;
+        r = pin_rootfs(context, params, &rootfs, &error_path);
+        if (r < 0) {
+                *exit_status = EXIT_NAMESPACE;
+                return log_error_errno(r, "Failed to open service's root fs%s%s: %m", error_path ? ": " : "", strempty(error_path));
+        }
+
+        r = setup_ephemeral(context, runtime, &rootfs, &error_path);
+        if (r < 0) {
+                *exit_status = EXIT_NAMESPACE;
+                return log_error_errno(r, "Failed to make ephemeral copy of service's root fs%s%s: %m", error_path ? ": " : "", strempty(error_path));
+        }
+
         /* Load a bunch of libraries we'll possibly need later, before we turn off dlopen() */
         (void) dlopen_bpf();
         (void) dlopen_cryptsetup();
@@ -5891,16 +6037,26 @@ int exec_invoke(
 
                 /* The kernel requires /proc/pid/setgroups be set to "deny" prior to writing /proc/pid/gid_map in
                  * unprivileged user namespaces. */
-                r = setup_private_users(pu, saved_uid, saved_gid, uid, gid, /* allow_setgroups= */ false);
+                r = setup_private_users(
+                                nsresource_link,
+                                pu,
+                                saved_uid,
+                                saved_gid,
+                                &uid,
+                                &gid,
+                                &outside_uid,
+                                &outside_gid,
+                                /* allow_setgroups= */ false);
                 /* If it was requested explicitly and we can't set it up, fail early. Otherwise, continue and let
                  * the actual requested operations fail (or silently continue). */
-                if (r < 0 && context->private_users != PRIVATE_USERS_NO) {
-                        *exit_status = EXIT_USER;
-                        return log_error_errno(r, "Failed to set up user namespacing for unprivileged user: %m");
-                }
-                if (r < 0)
-                        log_info_errno(r, "Failed to set up user namespacing for unprivileged user, ignoring: %m");
-                else {
+                if (r < 0) {
+                        if (context->private_users != PRIVATE_USERS_NO) {
+                                *exit_status = EXIT_USER;
+                                return log_error_errno(r, "Failed to set up user namespacing for unprivileged user: %m");
+                        }
+
+                        log_notice_errno(r, "Failed to set up user namespacing for unprivileged user, ignoring: %m");
+                } else {
                         assert(r > 0);
                         userns_set_up = true;
                         log_debug("Set up unprivileged user namespace");
@@ -5912,6 +6068,7 @@ int exec_invoke(
                         context,
                         params,
                         runtime,
+                        &rootfs,
                         /* delegate= */ false,
                         memory_pressure_path,
                         uid,
@@ -5922,6 +6079,7 @@ int exec_invoke(
                         &bpffs_pidref,
                         bpffs_socket_fd,
                         bpffs_errno_pipe,
+                        mountfsd_link,
                         exit_status);
         if (r < 0)
                 return r;
@@ -5971,8 +6129,16 @@ int exec_invoke(
         } else if (needs_sandboxing && !userns_set_up) {
                 PrivateUsers pu = exec_context_get_effective_private_users(context, params);
 
-                r = setup_private_users(pu, saved_uid, saved_gid, uid, gid,
-                                        /* allow_setgroups= */ pu == PRIVATE_USERS_FULL);
+                r = setup_private_users(
+                                nsresource_link,
+                                pu,
+                                saved_uid,
+                                saved_gid,
+                                &uid,
+                                &gid,
+                                &outside_uid,
+                                &outside_gid,
+                                /* allow_setgroups= */ pu == PRIVATE_USERS_FULL);
                 if (r < 0) {
                         *exit_status = EXIT_USER;
                         return log_error_errno(r, "Failed to set up user namespacing: %m");
@@ -5981,11 +6147,25 @@ int exec_invoke(
                         log_debug("Set up privileged user namespace");
         }
 
+        if (params->user_lookup_fd >= 0) {
+                /* If we haven't sent the UIDs/GIDs we settled on upstream yet, let's do so now, as we
+                 * finally know our final UID/GID range */
+
+                r = send_user_lookup(params->unit_id, params->user_lookup_fd, outside_uid, outside_gid);
+                if (r < 0) {
+                        *exit_status = EXIT_USER;
+                        return log_error_errno(r, "Failed to send user credentials to PID1: %m");
+                }
+
+                params->user_lookup_fd = safe_close(params->user_lookup_fd);
+        }
+
         /* Call setup_delegated_namespaces() the second time to unshare all delegated namespaces. */
         r = setup_delegated_namespaces(
                         context,
                         params,
                         runtime,
+                        &rootfs,
                         /* delegate= */ true,
                         memory_pressure_path,
                         uid,
@@ -5996,10 +6176,19 @@ int exec_invoke(
                         &bpffs_pidref,
                         bpffs_socket_fd,
                         bpffs_errno_pipe,
+                        mountfsd_link,
                         exit_status);
         if (r < 0)
                 return r;
 
+        /* We are done now with the nsresourced/mountfsd shenanigans, let's close the connections */
+        nsresource_link = sd_varlink_unref(nsresource_link);
+        mountfsd_link = sd_varlink_unref(mountfsd_link);
+
+        /* We don't need the pinned rootfs anymore at this point. Close the fds now, so that they are
+         * definitely gone before we do our fd rearrangements below. */
+        pinned_resource_done(&rootfs);
+
         /* Kill unnecessary process, for the case that e.g. when the bpffs mount point is hidden. */
         pidref_done_sigkill_wait(&bpffs_pidref);
 
@@ -6433,7 +6622,6 @@ int exec_invoke(
                         }
                 }
 #endif
-
         }
 
         if (!strv_isempty(context->unset_environment)) {

src/core/execute-serialize.c

@@ -1633,6 +1633,10 @@ static int exec_context_serialize(const ExecContext *c, FILE *f) {
         if (r < 0)
                 return r;
 
+        r = serialize_item_escaped(f, "exec-context-root-mstack", c->root_mstack);
+        if (r < 0)
+                return r;
+
         r = serialize_item_format(f, "exec-context-umask", "%04o", c->umask);
         if (r < 0)
                 return r;
@@ -2568,6 +2572,14 @@ static int exec_context_deserialize(ExecContext *c, FILE *f) {
                         if (r < 0)
                                 return r;
                         c->root_ephemeral = r;
+                } else if ((val = startswith(l, "exec-context-root-mstack="))) {
+                        ssize_t k;
+                        char *p;
+
+                        k = cunescape(val, 0, &p);
+                        if (k < 0)
+                                return k;
+                        free_and_replace(c->root_mstack, p);
                 } else if ((val = startswith(l, "exec-context-umask="))) {
                         r = parse_mode(val, &c->umask);
                         if (r < 0)

src/core/execute.c

@@ -263,7 +263,8 @@ bool exec_needs_mount_namespace(
 
         assert(context);
 
-        if (context->root_image)
+        if (context->root_image ||
+            context->root_mstack)
                 return true;
 
         if (context->root_directory_as_fd)
@@ -356,7 +357,7 @@ const char* exec_get_private_notify_socket_path(const ExecContext *context, cons
         if (!needs_sandboxing)
                 return NULL;
 
-        if (!context->root_directory && !context->root_image && !context->root_directory_as_fd)
+        if (!exec_context_with_rootfs(context))
                 return NULL;
 
         if (!exec_context_get_effective_mount_apivfs(context))
@@ -684,6 +685,7 @@ void exec_context_done(ExecContext *c) {
         iovec_done(&c->root_hash_sig);
         c->root_hash_sig_path = mfree(c->root_hash_sig_path);
         c->root_verity = mfree(c->root_verity);
+        c->root_mstack = mfree(c->root_mstack);
         c->tty_path = mfree(c->tty_path);
         c->syslog_identifier = mfree(c->syslog_identifier);
         c->user = mfree(c->user);
@@ -1203,6 +1205,9 @@ void exec_context_dump(const ExecContext *c, FILE* f, const char *prefix) {
         if (c->root_verity)
                 fprintf(f, "%sRootVerity: %s\n", prefix, c->root_verity);
 
+        if (c->root_mstack)
+                fprintf(f, "%sRootMStack: %s\n", prefix, c->root_mstack);
+
         STRV_FOREACH(e, c->environment)
                 fprintf(f, "%sEnvironment: %s\n", prefix, *e);
 
@@ -2058,9 +2063,19 @@ bool exec_context_restrict_filesystems_set(const ExecContext *c) {
 bool exec_context_with_rootfs(const ExecContext *c) {
         assert(c);
 
-        /* Checks if RootDirectory=, RootImage= or RootDirectoryFileDescriptor= are used */
+        /* Checks if RootDirectory=, RootImage=, RootMStack= or RootDirectoryFileDescriptor= are used */
 
-        return !empty_or_root(c->root_directory) || c->root_image || c->root_directory_as_fd;
+        return !empty_or_root(c->root_directory) || c->root_image || c->root_directory_as_fd || c->root_mstack;
+}
+
+bool exec_context_with_rootfs_strict(const ExecContext *c) {
+        assert(c);
+
+        /* just like exec_context_with_rootfs(), but doesn't suppress a root directory of "/", i.e. returns
+         * true in more cases: when a root directory is explicitly configured, even if it's our usual
+         * root. */
+
+        return c->root_directory || c->root_image || c->root_directory_as_fd || c->root_mstack;
 }
 
 int exec_context_has_vpicked_extensions(const ExecContext *context) {

src/core/execute.h

@@ -199,9 +199,12 @@ typedef struct ExecContext {
         char **unset_environment;
 
         struct rlimit *rlimit[_RLIMIT_MAX];
-        char *working_directory, *root_directory, *root_image, *root_verity, *root_hash_path, *root_hash_sig_path;
+        char *working_directory;
+        char *root_directory;
+        char *root_image, *root_verity, *root_hash_path, *root_hash_sig_path;
         struct iovec root_hash, root_hash_sig;
         MountOptions *root_image_options;
+        char *root_mstack;
         bool root_ephemeral;
         bool working_directory_missing_ok:1;
         bool working_directory_home:1;
@@ -577,6 +580,7 @@ char** exec_context_get_restrict_filesystems(const ExecContext *c);
 bool exec_context_restrict_namespaces_set(const ExecContext *c);
 bool exec_context_restrict_filesystems_set(const ExecContext *c);
 bool exec_context_with_rootfs(const ExecContext *c);
+bool exec_context_with_rootfs_strict(const ExecContext *c);
 
 int exec_context_has_vpicked_extensions(const ExecContext *context);
 

src/core/load-fragment-gperf.gperf.in

@@ -11,6 +11,7 @@
 {{type}}.RootHashSignature,                   config_parse_exec_root_hash_sig,                    0,                                  offsetof({{type}}, exec_context)
 {{type}}.RootVerity,                          config_parse_unit_path_printf,                      true,                               offsetof({{type}}, exec_context.root_verity)
 {{type}}.RootEphemeral,                       config_parse_bool,                                  0,                                  offsetof({{type}}, exec_context.root_ephemeral)
+{{type}}.RootMStack,                          config_parse_unit_path_printf,                      true,                               offsetof({{type}}, exec_context.root_mstack)
 {{type}}.ExtensionDirectories,                config_parse_namespace_path_strv,                   0,                                  offsetof({{type}}, exec_context.extension_directories)
 {{type}}.ExtensionImages,                     config_parse_extension_images,                      0,                                  offsetof({{type}}, exec_context)
 {{type}}.ExtensionImagePolicy,                config_parse_image_policy,                          0,                                  offsetof({{type}}, exec_context.extension_image_policy)

src/core/namespace.c

@@ -34,6 +34,7 @@
 #include "mkdir-label.h"
 #include "mount-util.h"
 #include "mountpoint-util.h"
+#include "mstack.h"
 #include "namespace.h"
 #include "namespace-util.h"
 #include "nsflags.h"
@@ -1360,7 +1361,7 @@ static int mount_private_dev(const MountEntry *m, const NamespaceParameters *p)
 
         /* We assume /run/systemd/journal/ is available if not changing root, which isn't entirely accurate
          * but shouldn't matter, as either way the user would get ENOENT when accessing /dev/log */
-        if ((!p->root_image && !p->root_directory && p->root_directory_fd < 0) || p->bind_log_sockets) {
+        if (!pinned_resource_is_set(p->rootfs) || p->bind_log_sockets) {
                 const char *devlog = strjoina(temporary_mount, "/dev/log");
                 if (symlink("/run/systemd/journal/dev-log", devlog) < 0)
                         log_debug_errno(errno,
@@ -1454,16 +1455,45 @@ static int mount_private_apivfs(
         assert(entry_path);
         assert(bind_source);
 
+        bool noprivs = false;
+
+        /* First, check if we have enough privileges to mount a new instance. */
+        _cleanup_close_ int mount_fd = make_fsmount(
+                        LOG_DEBUG,
+                        /* what= */ fstype,
+                        fstype,
+                        MS_NOSUID|MS_NOEXEC|MS_NODEV,
+                        opts,
+                        /* userns_fd= */ -EBADF);
+        if (ERRNO_IS_NEG_PRIVILEGE(mount_fd))
+                noprivs = true;
+        else if (ERRNO_IS_NEG_NOT_SUPPORTED(mount_fd)) {
+                /* Fallback for kernels lacking mount_setattr() */
+
+                // FIXME: This compatibility code path shall be removed once kernel 5.12
+                //        becomes the new minimal baseline
+
+                r = create_temporary_mount_point(scope, &temporary_mount);
+                if (r < 0)
+                        return r;
+
+                r = mount_nofollow_verbose(
+                                LOG_DEBUG,
+                                fstype,
+                                temporary_mount,
+                                fstype,
+                                MS_NOSUID|MS_NOEXEC|MS_NODEV,
+                                opts);
+                if (ERRNO_IS_NEG_PRIVILEGE(r))
+                        noprivs = true;
+                else if (r < 0)
+                        return r;
+        } else if (mount_fd < 0)
+                return log_debug_errno(mount_fd, "Failed to make file system mount: %m");
+
         (void) mkdir_p_label(entry_path, 0755);
 
-        /* First, check if we have enough privileges to mount a new instance. Note, a new sysfs instance
-         * cannot be mounted on an already existing mount. Let's use a temporary place. */
-        r = create_temporary_mount_point(scope, &temporary_mount);
-        if (r < 0)
-                return r;
-
-        r = mount_nofollow_verbose(LOG_DEBUG, fstype, temporary_mount, fstype, MS_NOSUID|MS_NOEXEC|MS_NODEV, opts);
-        if (ERRNO_IS_NEG_PRIVILEGE(r)) {
+        if (noprivs) {
                 /* When we do not have enough privileges to mount a new instance, fall back to use an
                  * existing mount. */
 
@@ -1482,8 +1512,6 @@ static int mount_private_apivfs(
 
                 return 1;
         }
-        if (r < 0)
-                return r;
 
         /* OK. We have a new mount instance. Let's clear an existing mount and its submounts. */
         r = umount_recursive(entry_path, /* flags= */ 0);
@@ -1491,9 +1519,16 @@ static int mount_private_apivfs(
                 log_debug_errno(r, "Failed to unmount directories below '%s', ignoring: %m", entry_path);
 
         /* Then, move the new mount instance. */
-        r = mount_nofollow_verbose(LOG_DEBUG, temporary_mount, entry_path, /* fstype= */ NULL, MS_MOVE, /* options= */ NULL);
-        if (r < 0)
-                return r;
+        if (mount_fd >= 0) {
+                r = RET_NERRNO(move_mount(mount_fd, "", -EBADF, entry_path, MOVE_MOUNT_F_EMPTY_PATH));
+                if (r < 0)
+                        return log_debug_errno(r, "Failed to attach '%s' to '%s': %m", fstype, entry_path);
+        } else if (temporary_mount) {
+                r = mount_nofollow_verbose(LOG_DEBUG, temporary_mount, entry_path, /* fstype= */ NULL, MS_MOVE, /* options= */ NULL);
+                if (r < 0)
+                        return r;
+        } else
+                assert_not_reached();
 
         /* We mounted a new instance now. Let's bind mount the children over now. This matters for nspawn
          * where a bunch of files are overmounted, in particular the boot id. */
@@ -1974,12 +2009,20 @@ static int apply_one_mount(
                 }
 
                 r = chase(mount_entry_source(m), NULL, CHASE_TRAIL_SLASH|CHASE_TRIGGER_AUTOFS, &chased, NULL);
-                if (r == -ENOENT && m->ignore) {
-                        log_debug_errno(r, "Path %s does not exist, ignoring.", mount_entry_source(m));
-                        return 0;
-                }
-                if (r < 0)
+                if (r < 0) {
+                        if (m->ignore) {
+                                if (r == -ENOENT) {
+                                        log_debug_errno(r, "Path '%s' does not exist, ignoring.", mount_entry_source(m));
+                                        return 0;
+                                }
+                                if (ERRNO_IS_NEG_PRIVILEGE(r)) {
+                                        log_debug_errno(r, "Path '%s' is not accessible, ignoring: %m", mount_entry_source(m));
+                                        return 0;
+                                }
+                        }
+
                         return log_debug_errno(r, "Failed to follow symlinks on %s: %m", mount_entry_source(m));
+                }
 
                 log_debug("Followed source symlinks %s %s %s.",
                           mount_entry_source(m), glyph(GLYPH_ARROW_RIGHT), chased);
@@ -2497,6 +2540,17 @@ static bool home_read_only(
         return false;
 }
 
+static bool namespace_read_only(const NamespaceParameters *p) {
+        assert(p);
+
+        return root_read_only(p->read_only_paths,
+                              p->protect_system) &&
+                home_read_only(p->read_only_paths, p->inaccessible_paths, p->empty_directories,
+                               p->bind_mounts, p->n_bind_mounts, p->temporary_filesystems, p->n_temporary_filesystems,
+                               p->protect_home) &&
+                strv_isempty(p->read_write_paths);
+}
+
 int setup_namespace(const NamespaceParameters *p, char **reterr_path) {
 
         _cleanup_(loop_device_unrefp) LoopDevice *loop_device = NULL;
@@ -2518,6 +2572,7 @@ int setup_namespace(const NamespaceParameters *p, char **reterr_path) {
                 DISSECT_IMAGE_PIN_PARTITION_DEVICES |
                 DISSECT_IMAGE_ALLOW_USERSPACE_VERITY |
                 DISSECT_IMAGE_VERITY_SHARE;
+        MStackFlags mstack_flags = 0;
         int r;
 
         assert(p);
@@ -2529,87 +2584,136 @@ int setup_namespace(const NamespaceParameters *p, char **reterr_path) {
         bool setup_propagate = !isempty(p->propagate_dir) && !isempty(p->incoming_dir);
         unsigned long mount_propagation_flag = p->mount_propagation_flag != 0 ? p->mount_propagation_flag : MS_SHARED;
 
-        if (p->root_image) {
-                /* Make the whole image read-only if we can determine that we only access it in a read-only fashion. */
-                if (root_read_only(p->read_only_paths,
-                                   p->protect_system) &&
-                    home_read_only(p->read_only_paths, p->inaccessible_paths, p->empty_directories,
-                                   p->bind_mounts, p->n_bind_mounts, p->temporary_filesystems, p->n_temporary_filesystems,
-                                   p->protect_home) &&
-                    strv_isempty(p->read_write_paths))
-                        dissect_image_flags |= DISSECT_IMAGE_READ_ONLY;
+        /* Make the whole image read-only if we can determine that we only access it in a read-only fashion. */
+        bool ro = namespace_read_only(p);
+        if (ro) {
+                dissect_image_flags |= DISSECT_IMAGE_READ_ONLY;
+                mstack_flags |= MSTACK_RDONLY;
+        }
 
-                SET_FLAG(dissect_image_flags, DISSECT_IMAGE_NO_PARTITION_TABLE, p->verity && p->verity->data_path);
+        _cleanup_close_ int _root_mount_fd = -EBADF;
+        int root_mount_fd = -EBADF;
+        if (pinned_resource_is_set(p->rootfs)) {
+                if (p->rootfs->directory_fd >= 0) {
+
+                        /* In "managed" mode we need to map from foreign UID/GID space, hence go via mountfsd */
+                        if (p->private_users == PRIVATE_USERS_MANAGED) {
+                                userns_fd = namespace_open_by_type(NAMESPACE_USER);
+                                if (userns_fd < 0)
+                                        return log_debug_errno(userns_fd, "Failed to open our own user namespace: %m");
+
+                                r = mountfsd_mount_directory_fd(
+                                                p->mountfsd_link,
+                                                p->rootfs->directory_fd,
+                                                userns_fd,
+                                                dissect_image_flags,
+                                                &_root_mount_fd);
+                                if (r < 0)
+                                        return r;
+
+                                root_mount_fd = _root_mount_fd;
+                        }
+
+                        /* Try to to clone the directory mount if we have privs to, so that we can apply the
+                         * MS_SLAVE propagation settings right-away. */
+                        if (root_mount_fd < 0) {
+                                _root_mount_fd = open_tree_attr_with_fallback(
+                                                p->rootfs->directory_fd,
+                                                "",
+                                                OPEN_TREE_CLONE|OPEN_TREE_CLOEXEC|AT_SYMLINK_NOFOLLOW|AT_EMPTY_PATH|AT_RECURSIVE,
+                                                &(struct mount_attr) {
+                                                        /* We just remounted / as slave, but that didn't affect the detached
+                                                         * mount that we just mounted, so remount that one as slave recursive
+                                                         * as well now. */
+                                                        .propagation = MS_SLAVE,
+                                                });
+                                if (_root_mount_fd < 0 && !ERRNO_IS_NEG_PRIVILEGE(_root_mount_fd) && _root_mount_fd != -EINVAL)
+                                        return log_debug_errno(_root_mount_fd, "Failed to clone specified directory: %m");
+
+                                root_mount_fd = _root_mount_fd;
+                        }
+                        /* If we have only a root fd (and we couldn't make it ours), and we have no path,
+                         * then try to go on with the literal fd */
+                        if (root_mount_fd < 0 && !p->rootfs->directory)
+                                root_mount_fd = p->rootfs->directory_fd;
+                }
+
+                if (p->rootfs->image_fd >= 0) {
+                        SET_FLAG(dissect_image_flags, DISSECT_IMAGE_NO_PARTITION_TABLE, p->verity && p->verity->data_path);
 
-                /* First check if we have a verity device already open and with a fstype pinned by policy. If it
-                * cannot be found, then fallback to the slow path (full dissect). */
-                r = dissected_image_new_from_existing_verity(
-                                p->root_image,
-                                p->verity,
-                                p->root_image_options,
-                                p->root_image_policy,
-                                /* image_filter= */ NULL,
-                                p->runtime_scope,
-                                dissect_image_flags,
-                                &dissected_image);
-                if (r < 0 && !ERRNO_IS_NEG_DEVICE_ABSENT(r) && r != -ENOPKG)
-                        return r;
-                if (r >= 0)
-                        log_debug("Reusing pre-existing verity-protected root image %s", p->root_image);
-                else {
                         if (p->runtime_scope == RUNTIME_SCOPE_SYSTEM) {
                                 /* In system mode we mount directly */
 
-                                r = loop_device_make_by_path(
-                                                p->root_image,
-                                                FLAGS_SET(dissect_image_flags, DISSECT_IMAGE_DEVICE_READ_ONLY) ? O_RDONLY : -1 /* < 0 means writable if possible, read-only as fallback */,
-                                                /* sector_size= */ UINT32_MAX,
-                                                FLAGS_SET(dissect_image_flags, DISSECT_IMAGE_NO_PARTITION_TABLE) ? 0 : LO_FLAGS_PARTSCAN,
-                                                LOCK_SH,
-                                                &loop_device);
-                                if (r < 0)
-                                        return log_debug_errno(r, "Failed to create loop device for root image: %m");
-
-                                r = dissect_loop_device(
-                                                loop_device,
+                                /* First check if we have a verity device already open and with a fstype pinned by policy. If it
+                                 * cannot be found, then fallback to the slow path (full dissect). */
+                                r = dissected_image_new_from_existing_verity(
+                                                p->rootfs->image,
                                                 p->verity,
                                                 p->root_image_options,
                                                 p->root_image_policy,
                                                 /* image_filter= */ NULL,
+                                                p->runtime_scope,
                                                 dissect_image_flags,
                                                 &dissected_image);
-                                if (r < 0)
-                                        return log_debug_errno(r, "Failed to dissect image: %m");
-
-                                r = dissected_image_load_verity_sig_partition(
-                                                dissected_image,
-                                                loop_device->fd,
-                                                p->verity);
-                                if (r < 0)
+                                if (r < 0 && !ERRNO_IS_NEG_DEVICE_ABSENT(r) && r != -ENOPKG)
                                         return r;
+                                if (r >= 0)
+                                        log_debug("Reusing pre-existing verity-protected root image %s", p->rootfs->image);
+                                else {
+                                        r = loop_device_make(
+                                                        p->rootfs->image_fd,
+                                                        FLAGS_SET(dissect_image_flags, DISSECT_IMAGE_DEVICE_READ_ONLY) ? O_RDONLY : -1 /* < 0 means take access mode from fd */,
+                                                        /* offset= */ 0,
+                                                        /* size= */ UINT64_MAX,
+                                                        /* sector_size= */ UINT32_MAX,
+                                                        FLAGS_SET(dissect_image_flags, DISSECT_IMAGE_NO_PARTITION_TABLE) ? 0 : LO_FLAGS_PARTSCAN,
+                                                        LOCK_SH,
+                                                        &loop_device);
+                                        if (r < 0)
+                                                return log_debug_errno(r, "Failed to create loop device for root image: %m");
 
-                                r = dissected_image_guess_verity_roothash(
-                                                dissected_image,
-                                                p->verity);
-                                if (r < 0)
-                                        return r;
+                                        r = dissect_loop_device(
+                                                        loop_device,
+                                                        p->verity,
+                                                        p->root_image_options,
+                                                        p->root_image_policy,
+                                                        /* image_filter= */ NULL,
+                                                        dissect_image_flags,
+                                                        &dissected_image);
+                                        if (r < 0)
+                                                return log_debug_errno(r, "Failed to dissect image: %m");
 
-                                r = dissected_image_decrypt(
-                                                dissected_image,
-                                                /* root= */ NULL,
-                                                /* passphrase= */ NULL,
-                                                p->verity,
-                                                p->root_image_policy,
-                                                dissect_image_flags);
-                                if (r < 0)
-                                        return log_debug_errno(r, "Failed to decrypt dissected image: %m");
+                                        r = dissected_image_load_verity_sig_partition(
+                                                        dissected_image,
+                                                        loop_device->fd,
+                                                        p->verity);
+                                        if (r < 0)
+                                                return r;
+
+                                        r = dissected_image_guess_verity_roothash(
+                                                        dissected_image,
+                                                        p->verity);
+                                        if (r < 0)
+                                                return r;
+
+                                        r = dissected_image_decrypt(
+                                                        dissected_image,
+                                                        /* root= */ NULL,
+                                                        /* passphrase= */ NULL,
+                                                        p->verity,
+                                                        p->root_image_policy,
+                                                        dissect_image_flags);
+                                        if (r < 0)
+                                                return log_debug_errno(r, "Failed to decrypt dissected image: %m");
+                                }
                         } else {
                                 userns_fd = namespace_open_by_type(NAMESPACE_USER);
                                 if (userns_fd < 0)
                                         return log_debug_errno(userns_fd, "Failed to open our own user namespace: %m");
 
-                                r = mountfsd_mount_image(
-                                                p->root_image,
+                                r = mountfsd_mount_image_fd(
+                                                p->mountfsd_link,
+                                                p->rootfs->image_fd,
                                                 userns_fd,
                                                 p->root_image_options,
                                                 p->root_image_policy,
@@ -2620,10 +2724,28 @@ int setup_namespace(const NamespaceParameters *p, char **reterr_path) {
                                         return r;
                         }
                 }
+
+                if (p->rootfs->mstack_loaded) {
+                        if (p->runtime_scope != RUNTIME_SCOPE_SYSTEM) {
+                                userns_fd = namespace_open_by_type(NAMESPACE_USER);
+                                if (userns_fd < 0)
+                                        return log_debug_errno(userns_fd, "Failed to open our own user namespace: %m");
+                        }
+
+                        r = mstack_open_images(
+                                        p->rootfs->mstack_loaded,
+                                        p->mountfsd_link,
+                                        userns_fd,
+                                        p->root_image_policy,
+                                        /* image_filter= */ NULL,
+                                        mstack_flags);
+                        if (r < 0)
+                                return r;
+                }
         }
 
-        if (p->root_directory)
-                root = p->root_directory;
+        if (p->rootfs && p->rootfs->directory)
+                root = p->rootfs->directory;
         else {
                 /* /run/systemd should have been created by PID 1 early on already, but in some cases, like
                  * when running tests (test-execute), it might not have been created yet so let's make sure
@@ -2964,21 +3086,36 @@ int setup_namespace(const NamespaceParameters *p, char **reterr_path) {
 
         /* Remount / as SLAVE so that nothing now mounted in the namespace
          * shows up in the parent */
-        if (mount(NULL, "/", NULL, MS_SLAVE|MS_REC, NULL) < 0)
-                return log_debug_errno(errno, "Failed to remount '/' as SLAVE: %m");
+        r = mount_nofollow_verbose(LOG_DEBUG, /* what= */ NULL, "/", /* fstype= */ NULL, MS_SLAVE|MS_REC, /* options= */ NULL);
+        if (r < 0)
+                return r;
 
-        if (p->root_directory_fd >= 0) {
+        if (root_mount_fd >= 0) {
+                /* If we have root_mount_fd we have a ready-to-use detached mount. Attach it. */
 
-                if (move_mount(p->root_directory_fd, "", AT_FDCWD, root, MOVE_MOUNT_F_EMPTY_PATH) < 0)
+                if (move_mount(root_mount_fd, "", AT_FDCWD, root, MOVE_MOUNT_F_EMPTY_PATH) < 0)
                         return log_debug_errno(errno, "Failed to move detached mount to '%s': %m", root);
 
-                /* We just remounted / as slave, but that didn't affect the detached mount that we just
-                 * mounted, so remount that one as slave recursive as well now. */
+                r = mount_nofollow_verbose(LOG_DEBUG, /* what= */ NULL, root, /* fstype= */ NULL, MS_SLAVE|MS_REC, /* options= */ NULL);
+                if (r < 0)
+                        return r;
 
-                if (mount(NULL, root, NULL, MS_SLAVE|MS_REC, NULL) < 0)
-                        return log_debug_errno(errno, "Failed to remount '%s' as SLAVE: %m", root);
+        } else if (p->rootfs && p->rootfs->directory) {
+
+                /* If we do not have root_mount_fd, but a directory was specified, then we can use it directly. */
+
+                /* A root directory is specified. Turn its directory into bind mount, if it isn't one yet. */
+                r = path_is_mount_point_full(root, /* root = */ NULL, AT_SYMLINK_FOLLOW);
+                if (r < 0)
+                        return log_debug_errno(r, "Failed to detect that %s is a mount point or not: %m", root);
+                if (r == 0) {
+                        r = mount_nofollow_verbose(LOG_DEBUG, root, root, /* fstype= */ NULL, MS_BIND|MS_REC, /* options= */ NULL);
+                        if (r < 0)
+                                return r;
+                }
+
+        } else if (dissected_image) {
 
-        } else if (p->root_image) {
                 /* A root image is specified, mount it to the right place */
                 r = dissected_image_mount(
                                 dissected_image,
@@ -3002,17 +3139,15 @@ int setup_namespace(const NamespaceParameters *p, char **reterr_path) {
                 if (r < 0)
                         return log_debug_errno(r, "Failed to relinquish dissected image: %m");
 
-        } else if (p->root_directory) {
+        } else if (p->rootfs && p->rootfs->mstack_loaded) {
 
-                /* A root directory is specified. Turn its directory into bind mount, if it isn't one yet. */
-                r = path_is_mount_point_full(root, /* root= */ NULL, AT_SYMLINK_FOLLOW);
+                r = mstack_make_mounts(p->rootfs->mstack_loaded, root, mstack_flags);
                 if (r < 0)
-                        return log_debug_errno(r, "Failed to detect that %s is a mount point or not: %m", root);
-                if (r == 0) {
-                        r = mount_nofollow_verbose(LOG_DEBUG, root, root, NULL, MS_BIND|MS_REC, NULL);
-                        if (r < 0)
-                                return r;
-                }
+                        return r;
+
+                r = mstack_bind_mounts(p->rootfs->mstack_loaded, root, /* where_fd= */ -EBADF, mstack_flags, /* ret_root_fd= */ NULL);
+                if (r < 0)
+                        return r;
 
         } else {
                 /* Let's mount the main root directory to the root directory to use */
@@ -3022,7 +3157,7 @@ int setup_namespace(const NamespaceParameters *p, char **reterr_path) {
         }
 
         /* Try to set up the new root directory before mounting anything else there. */
-        if (p->root_image || p->root_directory || p->root_directory_fd >= 0)
+        if (pinned_resource_is_set(p->rootfs))
                 (void) base_filesystem_create(root, UID_INVALID, GID_INVALID);
 
         /* Now make the magic happen */
@@ -3031,8 +3166,8 @@ int setup_namespace(const NamespaceParameters *p, char **reterr_path) {
                 return r;
 
         /* MS_MOVE does not work on MS_SHARED so the remount MS_SHARED will be done later */
-        r = mount_switch_root(root, /* mount_propagation_flag= */ 0);
-        if (r == -EINVAL && p->root_directory) {
+        r = mount_switch_root(root, /* mount_propagation_flag = */ 0);
+        if (r == -EINVAL && p->rootfs && p->rootfs->directory) {
                 /* If we are using root_directory and we don't have privileges (ie: user manager in a user
                  * namespace) and the root_directory is already a mount point in the parent namespace,
                  * MS_MOVE will fail as we don't have permission to change it (with EINVAL rather than
@@ -3041,6 +3176,7 @@ int setup_namespace(const NamespaceParameters *p, char **reterr_path) {
                 r = mount_nofollow_verbose(LOG_DEBUG, root, root, NULL, MS_BIND|MS_REC, NULL);
                 if (r < 0)
                         return r;
+
                 r = mount_switch_root(root, /* mount_propagation_flag= */ 0);
         }
         if (r < 0)
@@ -4094,6 +4230,7 @@ static const char* const private_users_table[_PRIVATE_USERS_MAX] = {
         [PRIVATE_USERS_SELF]     = "self",
         [PRIVATE_USERS_IDENTITY] = "identity",
         [PRIVATE_USERS_FULL]     = "full",
+        [PRIVATE_USERS_MANAGED]  = "managed",
 };
 
 DEFINE_STRING_TABLE_LOOKUP_WITH_BOOLEAN(private_users, PrivateUsers, PRIVATE_USERS_SELF);
@@ -4104,3 +4241,26 @@ static const char* const private_pids_table[_PRIVATE_PIDS_MAX] = {
 };
 
 DEFINE_STRING_TABLE_LOOKUP_WITH_BOOLEAN(private_pids, PrivatePIDs, PRIVATE_PIDS_YES);
+
+void pinned_resource_done(PinnedResource *p) {
+        assert(p);
+
+        p->directory_fd = safe_close(p->directory_fd);
+        p->directory = mfree(p->directory);
+        p->image_fd = safe_close(p->image_fd);
+        p->image = mfree(p->image);
+        p->mstack_loaded = mstack_free(p->mstack_loaded);
+        p->mstack = mfree(p->mstack);
+}
+
+bool pinned_resource_is_set(const PinnedResource *p) {
+        if (!p)
+                return false;
+
+        return p->directory_fd >= 0 ||
+                p->directory ||
+                p->image_fd >= 0 ||
+                p->image ||
+                p->mstack_loaded ||
+                p->mstack;
+}

src/core/namespace.h

@@ -70,6 +70,7 @@ typedef enum PrivateUsers {
         PRIVATE_USERS_SELF,
         PRIVATE_USERS_IDENTITY,
         PRIVATE_USERS_FULL,
+        PRIVATE_USERS_MANAGED,
         _PRIVATE_USERS_MAX,
         _PRIVATE_USERS_INVALID = -EINVAL,
 } PrivateUsers;
@@ -90,6 +91,24 @@ typedef enum PrivatePIDs {
         _PRIVATE_PIDS_INVALID = -EINVAL,
 } PrivatePIDs;
 
+typedef struct PinnedResource {
+        /* Pins a disk image, directory or mstack by file descriptors. The paths are stored too, but they are
+         * intended to be decoration only, to enhance log messages and should not be load-bearing
+         * otherwise. */
+        int directory_fd;
+        char *directory;
+        int image_fd;
+        char *image;
+        MStack *mstack_loaded;
+        char *mstack;
+} PinnedResource;
+
+#define PINNED_RESOURCE_NULL                    \
+        (PinnedResource) {                      \
+                .directory_fd = -EBADF,         \
+                .image_fd = -EBADF,             \
+        }
+
 typedef struct BindMount {
         char *source;
         char *destination;
@@ -127,9 +146,7 @@ typedef struct MountImage {
 typedef struct NamespaceParameters {
         RuntimeScope runtime_scope;
 
-        int root_directory_fd;
-        const char *root_directory;
-        const char *root_image;
+        const PinnedResource *rootfs;
         const MountOptions *root_image_options;
         const ImagePolicy *root_image_policy;
 
@@ -199,10 +216,13 @@ typedef struct NamespaceParameters {
         PrivateTmp private_tmp;
         PrivateTmp private_var_tmp;
         PrivatePIDs private_pids;
+        PrivateUsers private_users;
 
         PidRef *bpffs_pidref;
         int bpffs_socket_fd;
         int bpffs_errno_pipe;
+
+        sd_varlink *mountfsd_link;
 } NamespaceParameters;
 
 int setup_namespace(const NamespaceParameters *p, char **reterr_path);
@@ -300,3 +320,6 @@ int refresh_extensions_in_namespace(
                 const PidRef *target,
                 const char *hierarchy_env,
                 const NamespaceParameters *p);
+
+void pinned_resource_done(PinnedResource *p);
+bool pinned_resource_is_set(const PinnedResource *p);

src/core/service.c

@@ -3019,7 +3019,6 @@ static void service_enter_refresh_extensions(Service *s) {
                         .n_extension_images = s->exec_context.n_extension_images,
                         .extension_directories = s->exec_context.extension_directories,
                         .extension_image_policy = s->exec_context.extension_image_policy,
-                        .root_directory_fd = -EBADF,
                 };
 
                 /* Only reload confext, and not sysext as they also typically contain the executable(s) used
@@ -5887,8 +5886,9 @@ int service_determine_exec_selinux_label(Service *s, char **ret) {
 
         if (s->exec_context.root_image ||
             s->exec_context.n_extension_images > 0 ||
-            !strv_isempty(s->exec_context.extension_directories)) /* We cannot chase paths through images */
-                return log_unit_debug_errno(UNIT(s), SYNTHETIC_ERRNO(ENODATA), "Service with RootImage=, ExtensionImages= or ExtensionDirectories= set, cannot determine socket SELinux label before activation, ignoring.");
+            !strv_isempty(s->exec_context.extension_directories) ||
+            s->exec_context.root_mstack) /* We cannot chase paths through images */
+                return log_unit_debug_errno(UNIT(s), SYNTHETIC_ERRNO(ENODATA), "Service with RootImage=, ExtensionImages=, ExtensionDirectories= or RootMStack= set cannot determine socket SELinux label before activation, ignoring.");
 
         ExecCommand *c = s->exec_command[SERVICE_EXEC_START];
         if (!c)

src/core/unit.c

@@ -1266,6 +1266,12 @@ int unit_add_exec_dependencies(Unit *u, ExecContext *c) {
                         return r;
         }
 
+        if (c->root_mstack) {
+                r = unit_add_mounts_for(u, c->root_mstack, UNIT_DEPENDENCY_FILE, UNIT_MOUNT_WANTS);
+                if (r < 0)
+                        return r;
+        }
+
         for (ExecDirectoryType dt = 0; dt < _EXEC_DIRECTORY_TYPE_MAX; dt++) {
                 if (!u->manager->prefix[dt])
                         continue;
@@ -1322,9 +1328,9 @@ int unit_add_exec_dependencies(Unit *u, ExecContext *c) {
                         return r;
         }
 
-        if (c->root_image) {
+        if (c->root_image || c->root_mstack) {
                 /* We need to wait for /dev/loopX to appear when doing RootImage=, hence let's add an
-                 * implicit dependency on udev */
+                 * implicit dependency on udev. (And for RootMStack= we might need it) */
 
                 r = unit_add_dependency_by_name(u, UNIT_AFTER, SPECIAL_UDEVD_SERVICE, true, UNIT_DEPENDENCY_FILE);
                 if (r < 0)
@@ -4322,6 +4328,9 @@ static int unit_verify_contexts(const Unit *u) {
         if (ec->pam_name && kc && !IN_SET(kc->kill_mode, KILL_CONTROL_GROUP, KILL_MIXED))
                 return log_unit_error_errno(u, SYNTHETIC_ERRNO(ENOEXEC), "Unit has PAM enabled. Kill mode must be set to 'control-group' or 'mixed'. Refusing.");
 
+        if ((ec->user || ec->dynamic_user || ec->group || ec->pam_name) && ec->private_users == PRIVATE_USERS_MANAGED)
+                return log_unit_error_errno(u, SYNTHETIC_ERRNO(ENOEXEC), "PrivateUsers=managed may not be used in combination with User=/DynamicUser=/Group=/PAMName=, refusing.");
+
         return 0;
 }
 
@@ -4461,9 +4470,10 @@ int unit_patch_contexts(Unit *u) {
 
                 /* Only add these if needed, as they imply that everything else is blocked. */
                 if (cgroup_context_has_device_policy(cc)) {
-                        if (ec->root_image || ec->mount_images) {
+                        if (ec->root_image || ec->mount_images || ec->root_mstack) {
 
-                                /* When RootImage= or MountImages= is specified, the following devices are touched. */
+                                /* When RootImage= or MountImages= is specified, the following devices are
+                                 * touched. For RootMStack= there's the possibility the are touched. */
                                 FOREACH_STRING(p, "/dev/loop-control", "/dev/mapper/control") {
                                         r = cgroup_context_add_device_allow(cc, p, CGROUP_DEVICE_READ|CGROUP_DEVICE_WRITE);
                                         if (r < 0)

src/core/varlink-execute.c

@@ -794,6 +794,7 @@ int unit_exec_context_build_json(sd_json_variant **ret, const char *name, void *
                         JSON_BUILD_PAIR_CALLBACK_NON_NULL("WorkingDirectory", working_directory_build_json, c),
                         JSON_BUILD_PAIR_STRING_NON_EMPTY("RootDirectory", c->root_directory),
                         JSON_BUILD_PAIR_STRING_NON_EMPTY("RootImage", c->root_image),
+                        JSON_BUILD_PAIR_STRING_NON_EMPTY("RootMStack", c->root_mstack),
                         JSON_BUILD_PAIR_CALLBACK_NON_NULL("RootImageOptions", root_image_options_build_json, c->root_image_options),
                         SD_JSON_BUILD_PAIR_BOOLEAN("RootEphemeral", c->root_ephemeral),
                         JSON_BUILD_PAIR_BASE64_NON_EMPTY("RootHash", c->root_hash.iov_base, c->root_hash.iov_len),