Merge pull request #34067 from LukeShu/lukeshu/nspawn-fuse

nspawn: enable FUSE in containers
This commit is contained in:
Yu Watanabe
2024-09-09 19:32:16 +09:00
committed by GitHub
6 changed files with 363 additions and 49 deletions
+34 -21
View File
@@ -15,6 +15,7 @@
static int append_machine_properties(
sd_bus_message *m,
bool enable_fuse,
CustomMount *mounts,
unsigned n_mounts,
int kill_signal,
@@ -29,24 +30,23 @@ static int append_machine_properties(
if (r < 0)
return bus_log_create_error(r);
/* If you make changes here, also make sure to update systemd-nspawn@.service, to keep the device policies in
* sync regardless if we are run with or without the --keep-unit switch. */
/* If you make changes here, also make sure to update systemd-nspawn@.service, to keep the device
* policies in sync regardless if we are run with or without the --keep-unit switch. */
r = sd_bus_message_append(m, "(sv)", "DeviceAllow", "a(ss)", 2,
/* Allow the container to
* access and create the API
* device nodes, so that
* PrivateDevices= in the
* container can work
* fine */
/* Allow the container to access and create the API device nodes, so that
* PrivateDevices= in the container can work fine */
"/dev/net/tun", "rwm",
/* Allow the container
* access to ptys. However,
* do not permit the
* container to ever create
* these device nodes. */
/* Allow the container access to ptys. However, do not permit the container
* to ever create these device nodes. */
"char-pts", "rw");
if (r < 0)
return bus_log_create_error(r);
if (enable_fuse) {
r = sd_bus_message_append(m, "(sv)", "DeviceAllow", "a(ss)", 1,
"/dev/fuse", "rw");
if (r < 0)
return bus_log_create_error(r);
}
for (j = 0; j < n_mounts; j++) {
CustomMount *cm = mounts + j;
@@ -147,16 +147,16 @@ int register_machine(
int kill_signal,
char **properties,
sd_bus_message *properties_message,
bool keep_unit,
const char *service,
StartMode start_mode) {
StartMode start_mode,
RegisterMachineFlags flags) {
_cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL;
int r;
assert(bus);
if (keep_unit) {
if (FLAGS_SET(flags, REGISTER_MACHINE_KEEP_UNIT)) {
r = bus_call_method(
bus,
bus_machine_mgr,
@@ -207,6 +207,7 @@ int register_machine(
r = append_machine_properties(
m,
FLAGS_SET(flags, REGISTER_MACHINE_ENABLE_FUSE),
mounts,
n_mounts,
kill_signal,
@@ -262,8 +263,8 @@ int allocate_scope(
int kill_signal,
char **properties,
sd_bus_message *properties_message,
bool allow_pidfd,
StartMode start_mode) {
StartMode start_mode,
AllocateScopeFlags flags) {
_cleanup_(sd_bus_message_unrefp) sd_bus_message *m = NULL, *reply = NULL;
_cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL;
@@ -302,7 +303,7 @@ int allocate_scope(
if (r < 0)
return log_error_errno(r, "Failed to allocate PID reference: %m");
r = bus_append_scope_pidref(m, &pidref, allow_pidfd);
r = bus_append_scope_pidref(m, &pidref, FLAGS_SET(flags, ALLOCATE_SCOPE_ALLOW_PIDFD));
if (r < 0)
return bus_log_create_error(r);
@@ -327,6 +328,7 @@ int allocate_scope(
r = append_machine_properties(
m,
FLAGS_SET(flags, ALLOCATE_SCOPE_ENABLE_FUSE),
mounts,
n_mounts,
kill_signal,
@@ -354,9 +356,20 @@ int allocate_scope(
if (r < 0) {
/* If this failed with a property we couldn't write, this is quite likely because the server
* doesn't support PIDFDs yet, let's try without. */
if (allow_pidfd &&
if (FLAGS_SET(flags, ALLOCATE_SCOPE_ALLOW_PIDFD) &&
sd_bus_error_has_names(&error, SD_BUS_ERROR_UNKNOWN_PROPERTY, SD_BUS_ERROR_PROPERTY_READ_ONLY))
return allocate_scope(bus, machine_name, pid, slice, mounts, n_mounts, kill_signal, properties, properties_message, /* allow_pidfd= */ false, start_mode);
return allocate_scope(
bus,
machine_name,
pid,
slice,
mounts,
n_mounts,
kill_signal,
properties,
properties_message,
start_mode,
flags & ~ALLOCATE_SCOPE_ALLOW_PIDFD);
return log_error_errno(r, "Failed to allocate scope: %s", bus_error_message(&error, r));
}
+36 -2
View File
@@ -8,8 +8,42 @@
#include "nspawn-mount.h"
#include "nspawn-settings.h"
int register_machine(sd_bus *bus, const char *machine_name, pid_t pid, const char *directory, sd_id128_t uuid, int local_ifindex, const char *slice, CustomMount *mounts, unsigned n_mounts, int kill_signal, char **properties, sd_bus_message *properties_message, bool keep_unit, const char *service, StartMode start_mode);
typedef enum RegisterMachineFlags {
REGISTER_MACHINE_KEEP_UNIT = 1 << 0,
REGISTER_MACHINE_ENABLE_FUSE = 1 << 1,
} RegisterMachineFlags;
int register_machine(
sd_bus *bus,
const char *machine_name,
pid_t pid,
const char *directory,
sd_id128_t uuid,
int local_ifindex,
const char *slice,
CustomMount *mounts, unsigned n_mounts,
int kill_signal,
char **properties,
sd_bus_message *properties_message,
const char *service,
StartMode start_mode,
RegisterMachineFlags flags);
int unregister_machine(sd_bus *bus, const char *machine_name);
int allocate_scope(sd_bus *bus, const char *machine_name, pid_t pid, const char *slice, CustomMount *mounts, unsigned n_mounts, int kill_signal, char **properties, sd_bus_message *properties_message, bool allow_pidfds, StartMode start_mode);
typedef enum AllocateScopeFlags {
ALLOCATE_SCOPE_ALLOW_PIDFD = 1 << 0,
ALLOCATE_SCOPE_ENABLE_FUSE = 1 << 1,
} AllocateScopeFlags;
int allocate_scope(
sd_bus *bus,
const char *machine_name,
pid_t pid,
const char *slice,
CustomMount *mounts, unsigned n_mounts,
int kill_signal,
char **properties,
sd_bus_message *properties_message,
StartMode start_mode,
AllocateScopeFlags flags);
int terminate_scope(sd_bus *bus, const char *machine_name);
+130 -23
View File
@@ -2,6 +2,7 @@
#include <errno.h>
#include <getopt.h>
#include <linux/fuse.h>
#include <linux/loop.h>
#if HAVE_SELINUX
#include <selinux/selinux.h>
@@ -2147,31 +2148,112 @@ static int setup_boot_id(void) {
return mount_nofollow_verbose(LOG_ERR, NULL, to, NULL, MS_BIND|MS_REMOUNT|MS_RDONLY|MS_NOSUID|MS_NOEXEC|MS_NODEV, NULL);
}
static int copy_devnodes(const char *dest) {
static const char devnodes[] =
"null\0"
"zero\0"
"full\0"
"random\0"
"urandom\0"
"tty\0"
"net/tun\0";
static int get_fuse_version(uint32_t *ret_major, uint32_t *ret_minor) {
/* Must be called with mount privileges, either via arg_privileged or by being uid=0 in new
* CLONE_NEWUSER/CLONE_NEWNS namespaces. This is true when called from outer_child(). */
ssize_t n;
_cleanup_close_ int fuse_fd = -EBADF, mnt_fd = -EBADF;
_cleanup_free_ char *opts = NULL;
union {
char unstructured[FUSE_MIN_READ_BUFFER];
struct {
struct fuse_in_header header;
/* Don't use <linux/fuse.h>:`struct fuse_init_in` because a newer fuse.h might give
* us a bigger struct than what an older kernel actually gives us, and that would
* break our .header.len check. */
struct {
uint32_t major;
uint32_t minor;
} body;
} structured;
} request;
assert(ret_major);
assert(ret_minor);
/* Get a FUSE handle. */
fuse_fd = open("/dev/fuse", O_CLOEXEC|O_RDWR);
if (fuse_fd < 0)
return log_debug_errno(errno, "Failed to open /dev/fuse: %m");
if (asprintf(&opts, "fd=%i,rootmode=40000,user_id=0,group_id=0", fuse_fd) < 0)
return log_oom_debug();
mnt_fd = make_fsmount(LOG_DEBUG, "nspawn-fuse", "fuse.nspawn", 0, opts, -EBADF);
if (mnt_fd < 0)
return mnt_fd;
/* Read a request from the FUSE handle. */
n = read(fuse_fd, &request.unstructured, sizeof request);
if (n < 0)
return log_debug_errno(errno, "Failed to read /dev/fuse: %m");
if ((size_t) n < sizeof request.structured.header ||
(size_t) n < request.structured.header.len)
return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Failed to read /dev/fuse: Short read");
/* Assume that the request is a FUSE_INIT request, and return the version information from it. */
if (request.structured.header.opcode != FUSE_INIT)
return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Initial request from /dev/fuse should have opcode=%i (FUSE_INIT), but has opcode=%"PRIu32,
FUSE_INIT, request.structured.header.opcode);
if (request.structured.header.len < sizeof request.structured)
return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Initial FUSE_INIT request from /dev/fuse is too short");
*ret_major = request.structured.body.major;
*ret_minor = request.structured.body.minor;
return 0;
}
static bool should_enable_fuse(void) {
uint32_t fuse_major, fuse_minor;
int r;
r = get_fuse_version(&fuse_major, &fuse_minor);
if (r < 0) {
if (ERRNO_IS_NEG_DEVICE_ABSENT(r))
log_debug_errno(r, "Disabling FUSE: FUSE appears to be disabled on the host: %m");
else if (r == -ENOSYS)
log_debug_errno(r, "Disabling FUSE: Kernel does not support the fsopen() family of syscalls: %m");
else
log_warning_errno(r, "Disabling FUSE: Failed to determine FUSE version: %m");
return false;
}
/* FUSE is only userns-safe in FUSE version 7.27 and later.
* https://github.com/torvalds/linux/commit/da315f6e03988a7127680bbc26e1028991b899b8 */
if (fuse_major < 7 || (fuse_major == 7 && fuse_minor < 27)) {
log_debug("Disabling FUSE: FUSE version %" PRIu32 ".%" PRIu32 " is too old to support user namespaces",
fuse_major, fuse_minor);
return false;
}
return true;
}
static int copy_devnodes(const char *dest, bool enable_fuse) {
_cleanup_strv_free_ char **devnodes = NULL;
int r = 0;
assert(dest);
devnodes = strv_new("null",
"zero",
"full",
"random",
"urandom",
"tty",
STRV_IFNOTNULL(enable_fuse ? "fuse" : NULL),
"net/tun");
if (!devnodes)
return log_oom();
BLOCK_WITH_UMASK(0000);
/* Create /dev/net, so that we can create /dev/net/tun in it */
if (userns_mkdir(dest, "/dev/net", 0755, 0, 0) < 0)
return log_error_errno(r, "Failed to create /dev/net directory: %m");
NULSTR_FOREACH(d, devnodes) {
STRV_FOREACH(d, devnodes) {
_cleanup_free_ char *from = NULL, *to = NULL;
struct stat st;
from = path_join("/dev/", d);
from = path_join("/dev/", *d);
if (!from)
return log_oom();
@@ -2225,7 +2307,7 @@ static int copy_devnodes(const char *dest) {
if (!prefixed)
return log_oom();
t = path_join("..", d);
t = path_join("..", *d);
if (!t)
return log_oom();
@@ -3805,17 +3887,25 @@ static int outer_child(
_cleanup_(bind_user_context_freep) BindUserContext *bind_user_context = NULL;
_cleanup_strv_free_ char **os_release_pairs = NULL;
_cleanup_close_ int fd = -EBADF, mntns_fd = -EBADF;
bool idmap = false;
bool idmap = false, enable_fuse;
const char *p;
pid_t pid;
ssize_t l;
int r;
/* This is the "outer" child process, i.e the one forked off by the container manager itself. It
* already has its own CLONE_NEWNS namespace (which was created by the clone()). It still lives in
* the host's CLONE_NEWPID, CLONE_NEWUTS, CLONE_NEWIPC, CLONE_NEWUSER and CLONE_NEWNET
* namespaces. After it completed a number of initializations a second child (the "inner" one) is
* forked off it, and it exits. */
/* This is the "outer" child process, i.e the one forked off by the container manager itself. Its
* namespace situation is:
*
* - CLONE_NEWNS : already has its own (created by clone() if arg_privileged, or unshare() if !arg_unprivileged)
* - CLONE_NEWUSER : if arg_privileged: still in the host's
* if !arg_privileged: already has its own (created by nsresource_allocate_userns()->setns(userns_fd))
* - CLONE_NEWPID : still in the host's
* - CLONE_NEWUTS : still in the host's
* - CLONE_NEWIPC : still in the host's
* - CLONE_NEWNET : still in the host's
*
* After it completed a number of initializations a second child (the "inner" one) is forked off it,
* and it exits. */
assert(barrier);
assert(directory);
@@ -4080,7 +4170,12 @@ static int outer_child(
if (r < 0)
return r;
r = copy_devnodes(directory);
enable_fuse = should_enable_fuse();
l = send(fd_outer_socket, &enable_fuse, sizeof enable_fuse, 0);
if (l < 0)
return log_error_errno(errno, "Failed to send whether to enable FUSE: %m");
r = copy_devnodes(directory, enable_fuse);
if (r < 0)
return r;
@@ -5038,6 +5133,7 @@ static int run_container(
ssize_t l;
sigset_t mask_chld;
_cleanup_close_ int child_netns_fd = -EBADF;
bool enable_fuse;
assert_se(sigemptyset(&mask_chld) == 0);
assert_se(sigaddset(&mask_chld, SIGCHLD) == 0);
@@ -5224,6 +5320,12 @@ static int run_container(
l, l == 0 ? " The child is most likely dead." : "");
}
l = recv(fd_outer_socket_pair[0], &enable_fuse, sizeof enable_fuse, 0);
if (l < 0)
return log_error_errno(errno, "Failed to read whether to enable FUSE: %m");
if (l != sizeof enable_fuse)
return log_error_errno(SYNTHETIC_ERRNO(EIO), "Short read while reading whether to enable FUSE.");
/* Wait for the outer child. */
r = wait_for_terminate_and_check("(sd-namespace)", *pid, WAIT_LOG_ABNORMAL);
if (r < 0)
@@ -5374,6 +5476,9 @@ static int run_container(
}
if (arg_register) {
RegisterMachineFlags flags = 0;
SET_FLAG(flags, REGISTER_MACHINE_KEEP_UNIT, arg_keep_unit);
SET_FLAG(flags, REGISTER_MACHINE_ENABLE_FUSE, enable_fuse);
r = register_machine(
bus,
arg_machine,
@@ -5386,13 +5491,15 @@ static int run_container(
arg_kill_signal,
arg_property,
arg_property_message,
arg_keep_unit,
arg_container_service_name,
arg_start_mode);
arg_start_mode,
flags);
if (r < 0)
return r;
} else if (!arg_keep_unit) {
AllocateScopeFlags flags = ALLOCATE_SCOPE_ALLOW_PIDFD;
SET_FLAG(flags, ALLOCATE_SCOPE_ENABLE_FUSE, enable_fuse);
r = allocate_scope(
bus,
arg_machine,
@@ -5402,8 +5509,8 @@ static int run_container(
arg_kill_signal,
arg_property,
arg_property_message,
/* allow_pidfds= */ true,
arg_start_mode);
arg_start_mode,
flags);
if (r < 0)
return r;
+4
View File
@@ -20,6 +20,10 @@ test_append_files() {
# for IPMasquerade=
instmods "=net/netfilter"
generate_module_dependencies
# For unprivileged mountfsd.
if command -v openssl >/dev/null 2>&1; then
inst_binary openssl
fi
# Create a dummy container "template" with a minimal toolset, which we can
# then use as a base for our nspawn/machinectl tests
+152
View File
@@ -42,6 +42,9 @@ at_exit() {
mountpoint -q /var/lib/machines && umount --recursive /var/lib/machines
rm -f /run/systemd/nspawn/*.nspawn
rm -fr /var/tmp/TEST-13-NSPAWN.*
rm -f /run/verity.d/test-13-nspawn-*.crt
}
trap at_exit EXIT
@@ -1031,4 +1034,153 @@ EOF
rm -fr "$root"
}
can_do_rootless_nspawn() {
# Our create_dummy_ddi() uses squashfs and openssl.
command -v mksquashfs &&
command -v openssl &&
# mountfsd must be enabled...
[[ -S /run/systemd/io.systemd.MountFileSystem ]] &&
# ...and have pidfd support for unprivileged operation.
systemd-analyze compare-versions "$(uname -r)" ge 6.5 &&
systemd-analyze compare-versions "$(pkcheck --version | awk '{print $3}')" ge 124 &&
# nsresourced must be enabled...
[[ -S /run/systemd/userdb/io.systemd.NamespaceResource ]] &&
# ...and must support the UserNamespaceInterface.
! (SYSTEMD_LOG_TARGET=console varlinkctl call \
/run/systemd/userdb/io.systemd.NamespaceResource \
io.systemd.NamespaceResource.AllocateUserRange \
'{"name":"test-supported","size":65536,"userNamespaceFileDescriptor":0}' \
2>&1 || true) |
grep -q "io.systemd.NamespaceResource.UserNamespaceInterfaceNotSupported"
}
create_dummy_ddi() {
local outdir="${1:?}"
local container_name="${2:?}"
cat >"$outdir"/openssl.conf <<EOF
[ req ]
prompt = no
distinguished_name = req_distinguished_name
[ req_distinguished_name ]
C = DE
ST = Test State
L = Test Locality
O = Org Name
OU = Org Unit Name
CN = Common Name
emailAddress = test@email.com
EOF
openssl req -config "$outdir"/openssl.conf -subj="/CN=waldo" \
-x509 -sha256 -nodes -days 365 -newkey rsa:4096 \
-keyout "${outdir}/${container_name}.key" -out "${outdir}/${container_name}.crt"
mkdir -p /run/verity.d
cp "${outdir}/${container_name}.crt" "/run/verity.d/test-13-nspawn-${container_name}.crt"
SYSTEMD_REPART_OVERRIDE_FSTYPE=squashfs \
systemd-repart --make-ddi=portable \
--copy-source=/usr/share/TEST-13-NSPAWN-container-template \
--certificate="${outdir}/${container_name}.crt" \
--private-key="${outdir}/${container_name}.key" \
"${outdir}/${container_name}.raw"
}
testcase_unpriv() {
if ! can_do_rootless_nspawn; then
echo "Skipping rootless test..."
return 0
fi
local tmpdir name
tmpdir="$(mktemp -d /var/tmp/TEST-13-NSPAWN.unpriv.XXX)"
name="unpriv-${tmpdir##*.}"
trap 'rm -fr ${tmpdir@Q} || true; rm -f /run/verity.d/test-13-nspawn-${name@Q} || true' RETURN ERR
create_dummy_ddi "$tmpdir" "$name"
chown --recursive testuser: "$tmpdir"
systemd-run \
--pipe \
--uid=testuser \
--property=Delegate=yes \
-- \
systemd-nspawn --pipe --private-network --register=no --keep-unit --image="$tmpdir/$name.raw" echo hello >"$tmpdir/stdout.txt"
echo hello | cmp "$tmpdir/stdout.txt" -
}
testcase_fuse() {
if [[ "$(cat <>/dev/fuse 2>&1)" != 'cat: -: Operation not permitted' ]]; then
echo "FUSE is not supported, skipping the test..."
return 0
fi
# Assume that the tests are running on a kernel that is new enough for FUSE
# to have user-namespace support; and so we should expect that nspawn
# enables FUSE. This test does not validate that the version check
# disables FUSE on old kernels.
local root
root="$(mktemp -d /var/lib/machines/TEST-13-NSPAWN.fuse.XXX)"
create_dummy_container "$root"
# To avoid adding any complex dependencies to the test, we simply check
# that /dev/fuse can be opened for reading and writing (O_RDWR), but that
# actually reading from it fails with EPERM. This can be done with a
# simple Bash script: run `cat <>/dev/fuse` and if the EPERM error message
# comes from "bash" then we know it couldn't be opened, while if it comes
# from "cat" then we know that it was opened but not read. If we are able
# to read from the file, then this indicates that it's not a real FUSE
# device (which requires us to mount a type="fuse" filesystem with the
# option string "fd=${num}" for /dev/fuse FD before reading from it will
# return anything other than EPERM); if this happens then most likely
# nspawn didn't create the file at all and Bash "<>" simply created a new
# normal file.
#
# "cat: -: Operation not permitted" # pass the test; opened but not read
# "bash: line 1: /dev/fuse: Operation not permitted" # fail the test; could not open
# "" # fail the test; reading worked
[[ "$(systemd-nspawn --pipe --directory="$root" \
bash -c 'cat <>/dev/fuse' 2>&1)" == 'cat: -: Operation not permitted' ]]
rm -fr "$root"
}
testcase_unpriv_fuse() {
# Same as above, but for unprivileged operation.
if [[ "$(cat <>/dev/fuse 2>&1)" != 'cat: -: Operation not permitted' ]]; then
echo "FUSE is not supported, skipping the test..."
return 0
fi
if ! can_do_rootless_nspawn; then
echo "Skipping rootless test..."
return 0
fi
local tmpdir name
tmpdir="$(mktemp -d /var/tmp/TEST-13-NSPAWN.unpriv-fuse.XXX)"
# $name must be such that len("ns-$(id -u testuser)-nspawn-${name}-65535")
# <= 31, or nsresourced will reject the request for a namespace.
# Therefore; len($name) <= 10 bytes.
name="ufuse-${tmpdir##*.}"
trap 'rm -fr ${tmpdir@Q} || true; rm -f /run/verity.d/test-13-nspawn-${name@Q} || true' RETURN ERR
create_dummy_ddi "$tmpdir" "$name"
chown --recursive testuser: "$tmpdir"
[[ "$(systemd-run \
--pipe \
--uid=testuser \
--property=Delegate=yes \
--setenv=SYSTEMD_LOG_LEVEL \
--setenv=SYSTEMD_LOG_TARGET \
-- \
systemd-nspawn --pipe --private-network --register=no --keep-unit --image="$tmpdir/$name.raw" \
bash -c 'cat <>/dev/fuse' 2>&1)" == *'cat: -: Operation not permitted' ]]
}
run_testcases
+7 -3
View File
@@ -30,12 +30,16 @@ CoredumpReceive=yes
TasksMax=16384
{{SERVICE_WATCHDOG}}
{# Enforce a strict device policy, similar to the one nspawn configures when it
# allocates its own scope unit. Make sure to keep these policies in sync if you
# change them! #}
{# Enforce a strict device policy, similar to the one nspawn configures (in
# nspawn-register.c:append_machine_properties()) when it allocates its own
# scope unit. Make sure to keep these policies in sync if you change them! #}
DevicePolicy=closed
DeviceAllow=/dev/net/tun rwm
DeviceAllow=char-pts rw
{# /dev/fuse gets 'm' here even though it doesn't in nspawn-register.c, since
# efedb6b0f3 (nspawn: refuse to bind mount device node from host when
# --private-users= is specified, 2024-09-05) #}
DeviceAllow=/dev/fuse rwm
# nspawn itself needs access to /dev/loop-control and /dev/loop, to implement
# the --image= option. Add these here, too.