core: add FileDescriptorStorePreserve=on-success option (#42160)
Currently with FileDescriptorStorePreserve=yes the FD store is kept around regardless of what happens to a unit, which is useful in many cases. But in some cases, for example when complex services crash horribly, it's hard to reason about what was in the intermediate state, and it's better to start fresh. Add a new 'on-success' option for the FileDescriptorStorePreserve= setting that keeps it around only for as long as the unit doesn't go to a persistently failed state. This is especially useful in combination with LUO, where we don't want to keep around LUO sessions created by units that then proceeded to crash and burn, and might be in a bad state afterwards.
This commit is contained in:
@@ -336,14 +336,14 @@ systemd-dissect --shift /home/testuser/.local/state/machines/fdstore foreign
|
||||
run0 -u testuser mkdir -p .config/systemd/nspawn/
|
||||
run0 -u testuser -i "cat >.config/systemd/nspawn/fdstore.nspawn <<EOF
|
||||
[Exec]
|
||||
KillSignal=SIGKILL
|
||||
KillSignal=SIGTERM
|
||||
EOF"
|
||||
|
||||
run0 -u testuser mkdir -p ".config/systemd/user/systemd-nspawn@fdstore.service.d/"
|
||||
run0 -u testuser -i "cat >.config/systemd/user/systemd-nspawn@fdstore.service.d/fdstore.conf <<EOF
|
||||
[Service]
|
||||
FileDescriptorStoreMax=8
|
||||
FileDescriptorStorePreserve=yes
|
||||
FileDescriptorStorePreserve=on-success
|
||||
EOF"
|
||||
run0 -u testuser systemctl --user daemon-reload
|
||||
|
||||
@@ -362,7 +362,7 @@ timeout 30s bash -c \
|
||||
n_user_at_fds=$(systemctl show -P NFileDescriptorStore "user@${TESTUSER_UID}.service")
|
||||
test "${n_user_at_fds}" -ge 2
|
||||
|
||||
# 3) Stop the nspawn service: payload is gone but FileDescriptorStorePreserve=yes
|
||||
# 3) Stop the nspawn service: payload is gone but FileDescriptorStorePreserve=on-success
|
||||
# must keep the fds in the user-side fdstore (and propagated copy in PID 1).
|
||||
run0 -u testuser systemctl --user stop systemd-nspawn@fdstore.service
|
||||
n_nspawn_fds=$(run0 -u testuser systemctl --user show -P NFileDescriptorStore systemd-nspawn@fdstore.service)
|
||||
@@ -388,7 +388,21 @@ timeout 30s bash -c \
|
||||
run0 -u testuser systemctl --user start systemd-nspawn@fdstore.service
|
||||
run0 -u testuser systemctl is-active --user systemd-nspawn@fdstore.service
|
||||
|
||||
run0 -u testuser systemctl --user stop systemd-nspawn@fdstore.service
|
||||
# 7) Failure case: with FileDescriptorStorePreserve=on-success, the fdstore must
|
||||
# be dropped once the unit enters the permanent failed state (i.e. once all
|
||||
# automated restart attempts driven by Restart= are exhausted). The
|
||||
# systemd-nspawn@.service template doesn't set Restart=, so killing the inner
|
||||
# payload with SIGKILL forces the unit straight into 'failed'.
|
||||
timeout 30s bash -c \
|
||||
"until [[ \"\$(run0 -u testuser systemctl --user show -P NFileDescriptorStore systemd-nspawn@fdstore.service)\" -ge 2 ]]; do sleep 0.5; done"
|
||||
run0 -u testuser systemctl --user kill --kill-whom=all -s SIGKILL systemd-nspawn@fdstore.service
|
||||
timeout 30s bash -c \
|
||||
"until [[ \"\$(run0 -u testuser systemctl --user show -P ActiveState systemd-nspawn@fdstore.service)\" == failed ]]; do sleep 0.5; done"
|
||||
# The fdstore must be discarded once the failed state is reached.
|
||||
assert_eq "$(run0 -u testuser systemctl --user show -P NFileDescriptorStore systemd-nspawn@fdstore.service)" 0
|
||||
assert_eq "$(run0 -u testuser systemctl --user show -P SubState systemd-nspawn@fdstore.service)" failed
|
||||
run0 -u testuser systemctl --user reset-failed systemd-nspawn@fdstore.service
|
||||
|
||||
machinectl terminate fdstore 2>/dev/null || true
|
||||
|
||||
loginctl disable-linger testuser
|
||||
|
||||
@@ -92,7 +92,7 @@ EOF
|
||||
mkdir -p /run/systemd/nspawn
|
||||
cat >/run/systemd/nspawn/fdstore.nspawn <<EOF
|
||||
[Exec]
|
||||
KillSignal=SIGKILL
|
||||
KillSignal=SIGTERM
|
||||
EOF
|
||||
n_nspawn_fds=$(systemctl show -P NFileDescriptorStore systemd-nspawn@fdstore.service)
|
||||
test "${n_nspawn_fds}" -ge 2
|
||||
@@ -138,6 +138,78 @@ EOF
|
||||
n_fds=$(systemctl show -P NFileDescriptorStore TEST-91-LIVEUPDATE-late-zerofds.service)
|
||||
test "$n_fds" -eq 0
|
||||
systemctl start TEST-91-LIVEUPDATE-late-zerofds.service
|
||||
|
||||
# Verify that with FileDescriptorStorePreserve=on-success the fdstore is
|
||||
# discarded once the unit enters the permanent failed state, while still
|
||||
# being preserved across the transitionary failed states that precede
|
||||
# each automated auto-restart attempt. Use Restart=on-failure with
|
||||
# StartLimitBurst=2 so the manager runs the helper twice before
|
||||
# giving up. The helper:
|
||||
# - on the first attempt pushes an fd into the fdstore, becomes ready,
|
||||
# and then crashes,
|
||||
# - on subsequent attempts asserts that the previously stored fd is
|
||||
# handed back via $LISTEN_FDS (proving the fdstore survived the
|
||||
# auto-restart) and then crashes again.
|
||||
# When the start-limit is hit the unit lands in the permanent failed
|
||||
# state, at which point the fdstore must be empty.
|
||||
cat >/run/TEST-91-LIVEUPDATE-failure.sh <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -eux
|
||||
state_file=/run/TEST-91-LIVEUPDATE-failure.attempt
|
||||
attempt=$(cat "$state_file" 2>/dev/null || echo 0)
|
||||
attempt=$((attempt + 1))
|
||||
echo "$attempt" > "$state_file"
|
||||
if [[ "$attempt" -eq 1 ]]; then
|
||||
systemd-notify --fd=0 --fdname=mem </dev/zero
|
||||
else
|
||||
# On any restart attempt the fdstore must have been preserved across the
|
||||
# transitionary failed state and handed back to us via $LISTEN_FDS. Drop a
|
||||
# marker file when the invariant is broken so the outer test can detect it.
|
||||
if [[ "${LISTEN_FDS:-0}" -lt 1 ]]; then
|
||||
touch /run/TEST-91-LIVEUPDATE-failure.preserve-broken
|
||||
fi
|
||||
fi
|
||||
systemd-notify --ready
|
||||
# Give PID 1 a chance to process the FDSTORE=1/READY=1 notifications before
|
||||
# we exit, so the fdstore add is observed by the manager.
|
||||
sleep 0.5
|
||||
exit 1
|
||||
EOF
|
||||
chmod +x /run/TEST-91-LIVEUPDATE-failure.sh
|
||||
rm -f /run/TEST-91-LIVEUPDATE-failure.attempt \
|
||||
/run/TEST-91-LIVEUPDATE-failure.preserve-broken
|
||||
cat >/run/systemd/system/TEST-91-LIVEUPDATE-failure.service <<EOF
|
||||
[Unit]
|
||||
StartLimitIntervalSec=60
|
||||
StartLimitBurst=2
|
||||
[Service]
|
||||
Type=notify
|
||||
NotifyAccess=all
|
||||
FileDescriptorStoreMax=4
|
||||
FileDescriptorStorePreserve=on-success
|
||||
Restart=on-failure
|
||||
RestartSec=100ms
|
||||
ExecStart=/run/TEST-91-LIVEUPDATE-failure.sh
|
||||
EOF
|
||||
systemctl daemon-reload
|
||||
systemctl start TEST-91-LIVEUPDATE-failure.service || true
|
||||
timeout 60s bash -c \
|
||||
"until [[ \"\$(systemctl show -P ActiveState TEST-91-LIVEUPDATE-failure.service)\" == failed ]]; do sleep 0.5; done"
|
||||
# Sanity: the helper ran more than once, proving the fdstore was preserved
|
||||
# across at least one auto-restart attempt.
|
||||
test "$(cat /run/TEST-91-LIVEUPDATE-failure.attempt)" -ge 2
|
||||
# And the in-flight preservation invariant must hold for every restart.
|
||||
test ! -e /run/TEST-91-LIVEUPDATE-failure.preserve-broken
|
||||
# And the fdstore must be empty now that the permanent failed state was
|
||||
# reached, since FileDescriptorStorePreserve=on-success is set.
|
||||
n_fds=$(systemctl show -P NFileDescriptorStore TEST-91-LIVEUPDATE-failure.service)
|
||||
test "$n_fds" -eq 0
|
||||
systemctl reset-failed TEST-91-LIVEUPDATE-failure.service
|
||||
rm -f /run/systemd/system/TEST-91-LIVEUPDATE-failure.service \
|
||||
/run/TEST-91-LIVEUPDATE-failure.sh \
|
||||
/run/TEST-91-LIVEUPDATE-failure.attempt \
|
||||
/run/TEST-91-LIVEUPDATE-failure.preserve-broken
|
||||
systemctl daemon-reload
|
||||
else
|
||||
# Create memfds with known content and push them to our fd store.
|
||||
# Also request a LUO session, store a memfd in it, and push the session fd to the fd store.
|
||||
@@ -170,7 +242,7 @@ EOF
|
||||
mkdir -p /run/systemd/nspawn
|
||||
cat >/run/systemd/nspawn/fdstore.nspawn <<EOF
|
||||
[Exec]
|
||||
KillSignal=SIGKILL
|
||||
KillSignal=SIGTERM
|
||||
EOF
|
||||
|
||||
systemctl start systemd-nspawn@fdstore.service
|
||||
|
||||
Reference in New Issue
Block a user