core: add FileDescriptorStorePreserve=on-success option (#42160)

Currently with FileDescriptorStorePreserve=yes the FD store is kept
around
regardless of what happens to a unit, which is useful in many cases. But
in
some cases, for example when complex services crash horribly, it's hard
to
reason about what was in the intermediate state, and it's better to
start
fresh.

Add a new 'on-success' option for the FileDescriptorStorePreserve=
setting
that keeps it around only for as long as the unit doesn't go to a
persistently
failed state.

This is especially useful in combination with LUO, where we don't want
to
keep around LUO sessions created by units that then proceeded to crash
and
burn, and might be in a bad state afterwards.
This commit is contained in:
Luca Boccassi
2026-05-21 12:08:33 +01:00
committed by GitHub
10 changed files with 150 additions and 36 deletions
+18 -4
View File
@@ -336,14 +336,14 @@ systemd-dissect --shift /home/testuser/.local/state/machines/fdstore foreign
run0 -u testuser mkdir -p .config/systemd/nspawn/
run0 -u testuser -i "cat >.config/systemd/nspawn/fdstore.nspawn <<EOF
[Exec]
KillSignal=SIGKILL
KillSignal=SIGTERM
EOF"
run0 -u testuser mkdir -p ".config/systemd/user/systemd-nspawn@fdstore.service.d/"
run0 -u testuser -i "cat >.config/systemd/user/systemd-nspawn@fdstore.service.d/fdstore.conf <<EOF
[Service]
FileDescriptorStoreMax=8
FileDescriptorStorePreserve=yes
FileDescriptorStorePreserve=on-success
EOF"
run0 -u testuser systemctl --user daemon-reload
@@ -362,7 +362,7 @@ timeout 30s bash -c \
n_user_at_fds=$(systemctl show -P NFileDescriptorStore "user@${TESTUSER_UID}.service")
test "${n_user_at_fds}" -ge 2
# 3) Stop the nspawn service: payload is gone but FileDescriptorStorePreserve=yes
# 3) Stop the nspawn service: payload is gone but FileDescriptorStorePreserve=on-success
# must keep the fds in the user-side fdstore (and propagated copy in PID 1).
run0 -u testuser systemctl --user stop systemd-nspawn@fdstore.service
n_nspawn_fds=$(run0 -u testuser systemctl --user show -P NFileDescriptorStore systemd-nspawn@fdstore.service)
@@ -388,7 +388,21 @@ timeout 30s bash -c \
run0 -u testuser systemctl --user start systemd-nspawn@fdstore.service
run0 -u testuser systemctl is-active --user systemd-nspawn@fdstore.service
run0 -u testuser systemctl --user stop systemd-nspawn@fdstore.service
# 7) Failure case: with FileDescriptorStorePreserve=on-success, the fdstore must
# be dropped once the unit enters the permanent failed state (i.e. once all
# automated restart attempts driven by Restart= are exhausted). The
# systemd-nspawn@.service template doesn't set Restart=, so killing the inner
# payload with SIGKILL forces the unit straight into 'failed'.
timeout 30s bash -c \
"until [[ \"\$(run0 -u testuser systemctl --user show -P NFileDescriptorStore systemd-nspawn@fdstore.service)\" -ge 2 ]]; do sleep 0.5; done"
run0 -u testuser systemctl --user kill --kill-whom=all -s SIGKILL systemd-nspawn@fdstore.service
timeout 30s bash -c \
"until [[ \"\$(run0 -u testuser systemctl --user show -P ActiveState systemd-nspawn@fdstore.service)\" == failed ]]; do sleep 0.5; done"
# The fdstore must be discarded once the failed state is reached.
assert_eq "$(run0 -u testuser systemctl --user show -P NFileDescriptorStore systemd-nspawn@fdstore.service)" 0
assert_eq "$(run0 -u testuser systemctl --user show -P SubState systemd-nspawn@fdstore.service)" failed
run0 -u testuser systemctl --user reset-failed systemd-nspawn@fdstore.service
machinectl terminate fdstore 2>/dev/null || true
loginctl disable-linger testuser
+74 -2
View File
@@ -92,7 +92,7 @@ EOF
mkdir -p /run/systemd/nspawn
cat >/run/systemd/nspawn/fdstore.nspawn <<EOF
[Exec]
KillSignal=SIGKILL
KillSignal=SIGTERM
EOF
n_nspawn_fds=$(systemctl show -P NFileDescriptorStore systemd-nspawn@fdstore.service)
test "${n_nspawn_fds}" -ge 2
@@ -138,6 +138,78 @@ EOF
n_fds=$(systemctl show -P NFileDescriptorStore TEST-91-LIVEUPDATE-late-zerofds.service)
test "$n_fds" -eq 0
systemctl start TEST-91-LIVEUPDATE-late-zerofds.service
# Verify that with FileDescriptorStorePreserve=on-success the fdstore is
# discarded once the unit enters the permanent failed state, while still
# being preserved across the transitionary failed states that precede
# each automated auto-restart attempt. Use Restart=on-failure with
# StartLimitBurst=2 so the manager runs the helper twice before
# giving up. The helper:
# - on the first attempt pushes an fd into the fdstore, becomes ready,
# and then crashes,
# - on subsequent attempts asserts that the previously stored fd is
# handed back via $LISTEN_FDS (proving the fdstore survived the
# auto-restart) and then crashes again.
# When the start-limit is hit the unit lands in the permanent failed
# state, at which point the fdstore must be empty.
cat >/run/TEST-91-LIVEUPDATE-failure.sh <<'EOF'
#!/usr/bin/env bash
set -eux
state_file=/run/TEST-91-LIVEUPDATE-failure.attempt
attempt=$(cat "$state_file" 2>/dev/null || echo 0)
attempt=$((attempt + 1))
echo "$attempt" > "$state_file"
if [[ "$attempt" -eq 1 ]]; then
systemd-notify --fd=0 --fdname=mem </dev/zero
else
# On any restart attempt the fdstore must have been preserved across the
# transitionary failed state and handed back to us via $LISTEN_FDS. Drop a
# marker file when the invariant is broken so the outer test can detect it.
if [[ "${LISTEN_FDS:-0}" -lt 1 ]]; then
touch /run/TEST-91-LIVEUPDATE-failure.preserve-broken
fi
fi
systemd-notify --ready
# Give PID 1 a chance to process the FDSTORE=1/READY=1 notifications before
# we exit, so the fdstore add is observed by the manager.
sleep 0.5
exit 1
EOF
chmod +x /run/TEST-91-LIVEUPDATE-failure.sh
rm -f /run/TEST-91-LIVEUPDATE-failure.attempt \
/run/TEST-91-LIVEUPDATE-failure.preserve-broken
cat >/run/systemd/system/TEST-91-LIVEUPDATE-failure.service <<EOF
[Unit]
StartLimitIntervalSec=60
StartLimitBurst=2
[Service]
Type=notify
NotifyAccess=all
FileDescriptorStoreMax=4
FileDescriptorStorePreserve=on-success
Restart=on-failure
RestartSec=100ms
ExecStart=/run/TEST-91-LIVEUPDATE-failure.sh
EOF
systemctl daemon-reload
systemctl start TEST-91-LIVEUPDATE-failure.service || true
timeout 60s bash -c \
"until [[ \"\$(systemctl show -P ActiveState TEST-91-LIVEUPDATE-failure.service)\" == failed ]]; do sleep 0.5; done"
# Sanity: the helper ran more than once, proving the fdstore was preserved
# across at least one auto-restart attempt.
test "$(cat /run/TEST-91-LIVEUPDATE-failure.attempt)" -ge 2
# And the in-flight preservation invariant must hold for every restart.
test ! -e /run/TEST-91-LIVEUPDATE-failure.preserve-broken
# And the fdstore must be empty now that the permanent failed state was
# reached, since FileDescriptorStorePreserve=on-success is set.
n_fds=$(systemctl show -P NFileDescriptorStore TEST-91-LIVEUPDATE-failure.service)
test "$n_fds" -eq 0
systemctl reset-failed TEST-91-LIVEUPDATE-failure.service
rm -f /run/systemd/system/TEST-91-LIVEUPDATE-failure.service \
/run/TEST-91-LIVEUPDATE-failure.sh \
/run/TEST-91-LIVEUPDATE-failure.attempt \
/run/TEST-91-LIVEUPDATE-failure.preserve-broken
systemctl daemon-reload
else
# Create memfds with known content and push them to our fd store.
# Also request a LUO session, store a memfd in it, and push the session fd to the fd store.
@@ -170,7 +242,7 @@ EOF
mkdir -p /run/systemd/nspawn
cat >/run/systemd/nspawn/fdstore.nspawn <<EOF
[Exec]
KillSignal=SIGKILL
KillSignal=SIGTERM
EOF
systemctl start systemd-nspawn@fdstore.service