systemd/test/units/TEST-03-JOBS.sh

#!/usr/bin/env bash
# SPDX-License-Identifier: LGPL-2.1-or-later
# shellcheck disable=SC2016

set -eux
set -o pipefail

# shellcheck source=test/units/util.sh
. "$(dirname "$0")"/util.sh

# Simple test for that daemon-reexec works in container.
# See: https://github.com/systemd/systemd/pull/23883
systemctl daemon-reexec

# Test merging of a --job-mode=ignore-dependencies job into a previously
# installed job.

systemctl start --no-block hello-after-sleep.target

timeout 10 bash -c "until systemctl list-jobs | tee /root/list-jobs.txt | grep 'sleep\.service.*running'; do sleep .1; done"
grep 'hello\.service.*waiting' /root/list-jobs.txt

# This is supposed to finish quickly, not wait for sleep to finish.
timeout 10 systemctl start --job-mode=ignore-dependencies hello

# sleep should still be running, hello not.
systemctl list-jobs >/root/list-jobs.txt
grep 'sleep\.service.*running' /root/list-jobs.txt
(! grep 'hello\.service' /root/list-jobs.txt)
systemctl stop sleep.service hello-after-sleep.target

# Some basic testing that --show-transaction does something useful
(! systemctl is-active systemd-importd)
systemctl -T start systemd-importd
systemctl is-active systemd-importd
systemctl --show-transaction stop systemd-importd
(! systemctl is-active systemd-importd)

# Test for a crash when enqueuing a JOB_NOP when other job already exists
systemctl start --no-block hello-after-sleep.target
# hello.service should still be waiting, so these try-restarts will collapse
# into NOPs.
systemctl try-restart --job-mode=fail hello.service
systemctl try-restart hello.service
systemctl stop hello.service sleep.service hello-after-sleep.target

# TODO: add more job queueing/merging tests here.

# Test that restart propagates to activating units
systemctl -T --no-block start always-activating.service
systemctl list-jobs | grep 'always-activating.service'
ACTIVATING_ID_PRE=$(systemctl show -P InvocationID always-activating.service)
systemctl -T start always-activating.socket # Wait for the socket to come up
systemctl -T restart always-activating.socket
ACTIVATING_ID_POST=$(systemctl show -P InvocationID always-activating.service)
[[ "$ACTIVATING_ID_PRE" != "$ACTIVATING_ID_POST" ]]

# Test for irreversible jobs
systemctl start unstoppable.service

# This is expected to fail with 'job cancelled'
(! systemctl stop unstoppable.service)
# But this should succeed
systemctl stop --job-mode=replace-irreversibly unstoppable.service

# We're going to shutdown soon. Let's see if it succeeds when
# there's an active service that tries to be unstoppable.
# Shutdown of the container/VM will hang if not.
systemctl start unstoppable.service

# Test waiting for a started units to terminate again
cat <<EOF >/run/systemd/system/wait2.service
[Unit]
Description=Wait for 2 seconds
[Service]
ExecStart=bash -ec 'sleep 2'
EOF
cat <<EOF >/run/systemd/system/wait5fail.service
[Unit]
Description=Wait for 5 seconds and fail
[Service]
ExecStart=bash -ec 'sleep 5; false'
EOF

# wait2 succeeds
START_SEC=$(date -u '+%s')
timeout 10 systemctl start --wait wait2.service
END_SEC=$(date -u '+%s')
ELAPSED=$((END_SEC-START_SEC))
[[ "$ELAPSED" -ge 2 ]]

# wait5fail fails, so systemctl should fail
START_SEC=$(date -u '+%s')
(! systemctl start --wait wait2.service wait5fail.service)
END_SEC=$(date -u '+%s')
ELAPSED=$((END_SEC-START_SEC))
[[ "$ELAPSED" -ge 5 ]]

# Test time-limited scopes
START_SEC=$(date -u '+%s')
(! systemd-run --scope --property=RuntimeMaxSec=3s sleep 30)
END_SEC=$(date -u '+%s')
ELAPSED=$((END_SEC-START_SEC))
[[ "$ELAPSED" -ge 3 ]]
[[ "$ELAPSED" -le 10 ]]

# Test transactions with cycles
# Provides coverage for issues like https://github.com/systemd/systemd/issues/26872
for i in {0..19}; do
    cat >"/run/systemd/system/transaction-cycle$i.service" <<EOF
[Unit]
After=transaction-cycle$(((i + 1) % 20)).service
Requires=transaction-cycle$(((i + 1) % 20)).service

[Service]
ExecStart=true
EOF
done
systemctl daemon-reload
for i in {0..19}; do
    # This intentionally fails with:
    #   Failed to start transaction-cycle0.service: Transaction order is cyclic. See system logs for details.
    systemctl start "transaction-cycle$i.service" || :
done

IDS_FILE="/tmp/TEST-03-JOBS-CYCLE-IDS-$RANDOM"
varlinkctl call /run/systemd/io.systemd.Manager io.systemd.Manager.Describe '{}' | jq '.runtime.TransactionsWithOrderingCycle' >"$IDS_FILE"
[[ "$(jq length "$IDS_FILE")" -ge 20 ]]
journalctl --sync
for i in {0..19}; do
    journalctl -b TRANSACTION_ID="$(jq -r ".[$i]" "$IDS_FILE")" --grep "cycle starting with"
done

# Test PropagatesStopTo= when restart (issue #26839)
systemctl start propagatestopto-and-pullin.target
systemctl --quiet is-active propagatestopto-and-pullin.target

systemctl restart propagatestopto-and-pullin.target
systemctl --quiet is-active propagatestopto-and-pullin.target
systemctl --quiet is-active sleep-infinity-simple.service

systemctl start propagatestopto-only.target
systemctl --quiet is-active propagatestopto-only.target
systemctl --quiet is-active sleep-infinity-simple.service

systemctl restart propagatestopto-only.target
assert_rc 3 systemctl --quiet is-active sleep-infinity-simple.service

systemctl start propagatestopto-indirect.target propagatestopto-and-pullin.target
systemctl --quiet is-active propagatestopto-indirect.target
systemctl --quiet is-active propagatestopto-and-pullin.target

systemctl restart propagatestopto-indirect.target
assert_rc 3 systemctl --quiet is-active propagatestopto-and-pullin.target
assert_rc 3 systemctl --quiet is-active sleep-infinity-simple.service

# Test restart mode direct

systemctl start succeeds-on-restart-restartdirect.target
assert_rc 0 systemctl --quiet is-active succeeds-on-restart-restartdirect.target

systemctl start fails-on-restart-restartdirect.target || :
assert_rc 3 systemctl --quiet is-active fails-on-restart-restartdirect.target

systemctl start succeeds-on-restart.target || :
assert_rc 3 systemctl --quiet is-active succeeds-on-restart.target

systemctl start fails-on-restart.target || :
assert_rc 3 systemctl --quiet is-active fails-on-restart.target

systemctl stop fails-on-restart.service

COUNTER_FILE=/tmp/test-03-restart-counter
export FAILURE_FLAG_FILE=/tmp/test-03-restart-failure-flag

assert_rc 3 systemctl --quiet is-active sleep-infinity-restart-normal.service
assert_rc 3 systemctl --quiet is-active sleep-infinity-restart-direct.service
assert_rc 3 systemctl --quiet is-active counter.service
echo 0 >"$COUNTER_FILE"
rm -f "$FAILURE_FLAG_FILE"

systemctl start counter.service
assert_eq "$(cat "$COUNTER_FILE")" "1"
systemctl --quiet is-active sleep-infinity-restart-normal.service
systemctl --quiet is-active sleep-infinity-restart-direct.service
systemctl --quiet is-active counter.service

# RestartMode=direct + restart: explicit restart should get propagated as TRY_RESTART to the still active
# counter.service
systemctl restart sleep-infinity-restart-direct.service
timeout 10 bash -c 'while ! systemctl --quiet is-active counter.service; do sleep .5; done'
assert_eq "$(cat "$COUNTER_FILE")" "2"
[[ ! -f "$FAILURE_FLAG_FILE" ]]

# RestartMode=direct + kill: the fail/inactive state shouldn't get propagated to the counter.service
systemctl kill --signal=KILL sleep-infinity-restart-direct.service
systemctl --quiet is-active counter.service
assert_eq "$(cat "$COUNTER_FILE")" "2"
[[ ! -f "$FAILURE_FLAG_FILE" ]]

# RestartMode=normal + restart: explicit restart should get propagated as TRY_RESTART to the still active
# counter.service
systemctl restart sleep-infinity-restart-normal.service
timeout 10 bash -c 'while ! systemctl --quiet is-active counter.service; do sleep .5; done'
assert_eq "$(cat "$COUNTER_FILE")" "3"
[[ ! -f "$FAILURE_FLAG_FILE" ]]

# RestartMode=normal + kill: the fail/inactive state should get propagated to the counter.service, which in
# turn should be stopped
systemctl kill --signal=KILL sleep-infinity-restart-normal.service
timeout 10 bash -c 'while [[ ! -f $FAILURE_FLAG_FILE ]]; do sleep .5; done'
timeout 10 bash -c 'while systemctl --quiet is-active counter.service; do sleep .5; done'
assert_eq "$(cat "$COUNTER_FILE")" "3"

# Test shortcutting auto restart

export UNIT_NAME="TEST-03-JOBS-shortcut-restart.service"
TMP_FILE="/tmp/test-03-shortcut-restart-test$RANDOM"

cat >"/run/systemd/system/$UNIT_NAME" <<EOF
[Service]
Type=oneshot
ExecStart=rm -v "$TMP_FILE"
Restart=on-failure
RestartSec=1d
RemainAfterExit=yes
EOF

(! systemctl start "$UNIT_NAME")
timeout 10 bash -c 'while [[ "$(systemctl show "$UNIT_NAME" -P SubState)" != "auto-restart" ]]; do sleep .5; done'
touch "$TMP_FILE"
assert_eq "$(systemctl show "$UNIT_NAME" -P SubState)" "auto-restart"

timeout 30 systemctl start "$UNIT_NAME"
systemctl --quiet is-active "$UNIT_NAME"
assert_eq "$(systemctl show "$UNIT_NAME" -P NRestarts)" "1"
[[ ! -f "$TMP_FILE" ]]

rm /run/systemd/system/"$UNIT_NAME"

# Test RestartRandomizedDelaySec=

export UNIT_NAME="TEST-03-JOBS-restart-randomized-delay.service"

cat >"/run/systemd/system/$UNIT_NAME" <<EOF
[Service]
Type=simple
ExecStart=false
Restart=on-failure
RestartSec=1
RestartRandomizedDelaySec=1
StartLimitIntervalSec=0
EOF

systemctl daemon-reload

# The option should be parsed and exposed on the bus in usec.
assert_eq "$(systemctl show "$UNIT_NAME" -P RestartRandomizedDelayUSec)" "1s"

# The chosen delay is logged at debug level when the unit enters auto-restart, so we can read it without
# waiting for the delay to elapse.
PREV_LOG_LEVEL="$(systemctl log-level)"

restart_randomized_delay_cleanup() {
    set +e
    systemctl log-level "$PREV_LOG_LEVEL"
    systemctl stop "$UNIT_NAME"
    rm -f /run/systemd/system/"$UNIT_NAME"
    systemctl daemon-reload
}
trap restart_randomized_delay_cleanup EXIT

systemctl log-level debug

get_restart_interval() {
    # Enter auto-restart once, read the logged "<total>|<delay>", then stop again so it never has to elapse.
    systemctl start --no-block "$UNIT_NAME"
    timeout 10 bash -c 'while [[ "$(systemctl show "'"$UNIT_NAME"'" -P SubState)" != "auto-restart" ]]; do sleep .2; done'
    systemctl stop "$UNIT_NAME"
    journalctl --sync
    # needed because of -o pipefail
    { journalctl -q --no-pager -o cat -b -u "$UNIT_NAME" --grep="Next restart interval calculated as" || true; } |
        sed -n 's/.*calculated as: \(.*\) (randomized delay: \(.*\))$/\1|\2/p' | tail -n1
}

# Several samples + "not all equal": two draws could rarely render identically (~1e-6) and falsely fail.
DELAYS=()
TOTALS=()
for _ in {1..4}; do
    IFS='|' read -r total delay <<<"$(get_restart_interval)"
    TOTALS+=("$total")
    DELAYS+=("$delay")
done

systemctl log-level "$PREV_LOG_LEVEL"

: "Chosen randomized restart delays: ${DELAYS[*]} (totals: ${TOTALS[*]})"
for delay in "${DELAYS[@]}"; do
    assert_neq "$delay" ""
    # Within bound: a value below 1s never renders a bare "<digit>s" token (only ms/us).
    if [[ "$delay" =~ [0-9]s ]]; then
        echo "FAIL: randomized restart delay '$delay' exceeds the configured 1s bound" >&2
        exit 1
    fi
done
# Total must vary, proving the jitter is folded into the armed timer (not merely logged).
all_equal=1
for total in "${TOTALS[@]}"; do
    [[ "$total" == "${TOTALS[0]}" ]] || all_equal=0
done
assert_eq "$all_equal" "0"

touch /testok