From 79ddf3c6f364d5c8b94aa11955c108628813d540 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Mon, 25 May 2026 11:28:36 +0100 Subject: [PATCH 1/3] Revert "test: switch TEST-55-OOMD stress-ng --vm-method to lfsr32" This reverts commit 4ac23697280bf54bb768f0aa7a5c7d7d0bcf3f6b. --- .../TEST-55-OOMD-testbloat.service | 7 ++++--- .../TEST-55-OOMD-testmunch.service | 7 ++++--- test/units/TEST-55-OOMD.sh | 15 +++++++++------ 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testbloat.service b/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testbloat.service index e4109fd60d..22bbd210e9 100644 --- a/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testbloat.service +++ b/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testbloat.service @@ -7,6 +7,7 @@ Description=Create a lot of memory pressure # to throttle and be put under heavy pressure. MemoryHigh=3M Slice=TEST-55-OOMD-workload.slice -# Pin --vm-method=lfsr32: the only stress-ng vm method without TARGET_CLONES, -# so it can't dispatch to an AVX-512 clone and SIGILL on CPUs lacking it. -ExecStart=stress-ng --timeout 3m --vm 10 --vm-bytes 200M --vm-keep --vm-method=lfsr32 +# Pin --vm-method to a portable method (zero-one): the default 'all' cycles +# through methods, including newer ones using AVX-512 instructions that SIGILL +# on CPUs without AVX-512 (e.g. AMD Zen 1-3), making the test flaky. +ExecStart=stress-ng --timeout 3m --vm 10 --vm-bytes 200M --vm-keep --vm-method=zero-one diff --git a/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testmunch.service b/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testmunch.service index 81305c7e97..06eea10b79 100644 --- a/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testmunch.service +++ b/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testmunch.service @@ -5,6 +5,7 @@ Description=Create some memory pressure [Service] MemoryHigh=12M Slice=TEST-55-OOMD-workload.slice -# Pin --vm-method=lfsr32: the only stress-ng vm method without TARGET_CLONES, -# so it can't dispatch to an AVX-512 clone and SIGILL on CPUs lacking it. -ExecStart=stress-ng --timeout 3m --vm 10 --vm-bytes 200M --vm-keep --vm-method=lfsr32 +# Pin --vm-method to a portable method (zero-one): the default 'all' cycles +# through methods, including newer ones using AVX-512 instructions that SIGILL +# on CPUs without AVX-512 (e.g. AMD Zen 1-3), making the test flaky. +ExecStart=stress-ng --timeout 3m --vm 10 --vm-bytes 200M --vm-keep --vm-method=zero-one diff --git a/test/units/TEST-55-OOMD.sh b/test/units/TEST-55-OOMD.sh index 84baee12a4..b7311e83dc 100755 --- a/test/units/TEST-55-OOMD.sh +++ b/test/units/TEST-55-OOMD.sh @@ -366,12 +366,13 @@ EOF systemctl reload systemd-oomd.service # Run a transient service with OOMRules=testrule that generates memory pressure. - # Pin --vm-method=lfsr32: the only stress-ng vm method without TARGET_CLONES, - # so it can't dispatch to an AVX-512 clone and SIGILL on CPUs lacking it. + # Pin --vm-method to a portable method (zero-one): the default 'all' cycles + # through every method, including newer ones using AVX-512 instructions that + # SIGILL on CPUs without AVX-512 (e.g. AMD Zen 1-3), making the test flaky. (! systemd-run --wait --unit=TEST-55-OOMD-testrules \ -p MemoryHigh=3M \ -p OOMRules=testrule \ - stress-ng --timeout 3m --vm 10 --vm-bytes 50M --vm-keep --vm-method=lfsr32) + stress-ng --timeout 3m --vm 10 --vm-bytes 50M --vm-keep --vm-method=zero-one) # Verify in the journal that the rule triggered journalctl --sync @@ -456,12 +457,14 @@ EOF # Start the unit without --wait so we can check mid-run state. The # stress-ng timeout bounds the test if anything goes wrong. - # Pin --vm-method=lfsr32: the only stress-ng vm method without TARGET_CLONES, - # so it can't SIGILL on AVX-512-less CPUs and exit before the 6 s wait below. + # Pin --vm-method to a portable method (zero-one): the default 'all' cycles + # through every method, including newer ones using AVX-512 instructions that + # SIGILL on CPUs without AVX-512 (e.g. AMD Zen 1-3) and would cause stress-ng + # to exit before the 6 s wait below elapses, failing the ActiveState check. systemd-run --unit=TEST-55-OOMD-slowrule \ -p MemoryHigh=3M \ -p OOMRules=slowrule \ - stress-ng --timeout 15s --vm 10 --vm-bytes 50M --vm-keep --vm-method=lfsr32 + stress-ng --timeout 15s --vm 10 --vm-bytes 50M --vm-keep --vm-method=zero-one # Wait long enough for oomd's 1s rule-check loop to evaluate the condition # many times. With LastingSec=1h the kill must not fire. From aac19fe4685906a2166464cb0e9e600ebfe850a1 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Mon, 25 May 2026 11:28:37 +0100 Subject: [PATCH 2/3] Revert "test: pin stress-ng --vm-method to a portable scalar method in TEST-55-OOMD" This reverts commit 881e4717c7981b274853309e68b39153e3b292f4. --- .../TEST-55-OOMD-testbloat.service | 5 +---- .../TEST-55-OOMD-testmunch.service | 5 +---- test/units/TEST-55-OOMD.sh | 13 +++---------- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testbloat.service b/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testbloat.service index 22bbd210e9..70c87727c8 100644 --- a/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testbloat.service +++ b/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testbloat.service @@ -7,7 +7,4 @@ Description=Create a lot of memory pressure # to throttle and be put under heavy pressure. MemoryHigh=3M Slice=TEST-55-OOMD-workload.slice -# Pin --vm-method to a portable method (zero-one): the default 'all' cycles -# through methods, including newer ones using AVX-512 instructions that SIGILL -# on CPUs without AVX-512 (e.g. AMD Zen 1-3), making the test flaky. -ExecStart=stress-ng --timeout 3m --vm 10 --vm-bytes 200M --vm-keep --vm-method=zero-one +ExecStart=stress-ng --timeout 3m --vm 10 --vm-bytes 200M --vm-keep diff --git a/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testmunch.service b/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testmunch.service index 06eea10b79..79bd01838e 100644 --- a/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testmunch.service +++ b/test/integration-tests/TEST-55-OOMD/TEST-55-OOMD.units/TEST-55-OOMD-testmunch.service @@ -5,7 +5,4 @@ Description=Create some memory pressure [Service] MemoryHigh=12M Slice=TEST-55-OOMD-workload.slice -# Pin --vm-method to a portable method (zero-one): the default 'all' cycles -# through methods, including newer ones using AVX-512 instructions that SIGILL -# on CPUs without AVX-512 (e.g. AMD Zen 1-3), making the test flaky. -ExecStart=stress-ng --timeout 3m --vm 10 --vm-bytes 200M --vm-keep --vm-method=zero-one +ExecStart=stress-ng --timeout 3m --vm 10 --vm-bytes 200M --vm-keep diff --git a/test/units/TEST-55-OOMD.sh b/test/units/TEST-55-OOMD.sh index b7311e83dc..6689bbdd73 100755 --- a/test/units/TEST-55-OOMD.sh +++ b/test/units/TEST-55-OOMD.sh @@ -365,14 +365,11 @@ EOF systemctl reload systemd-oomd.service - # Run a transient service with OOMRules=testrule that generates memory pressure. - # Pin --vm-method to a portable method (zero-one): the default 'all' cycles - # through every method, including newer ones using AVX-512 instructions that - # SIGILL on CPUs without AVX-512 (e.g. AMD Zen 1-3), making the test flaky. + # Run a transient service with OOMRules=testrule that generates memory pressure (! systemd-run --wait --unit=TEST-55-OOMD-testrules \ -p MemoryHigh=3M \ -p OOMRules=testrule \ - stress-ng --timeout 3m --vm 10 --vm-bytes 50M --vm-keep --vm-method=zero-one) + stress-ng --timeout 3m --vm 10 --vm-bytes 50M --vm-keep) # Verify in the journal that the rule triggered journalctl --sync @@ -457,14 +454,10 @@ EOF # Start the unit without --wait so we can check mid-run state. The # stress-ng timeout bounds the test if anything goes wrong. - # Pin --vm-method to a portable method (zero-one): the default 'all' cycles - # through every method, including newer ones using AVX-512 instructions that - # SIGILL on CPUs without AVX-512 (e.g. AMD Zen 1-3) and would cause stress-ng - # to exit before the 6 s wait below elapses, failing the ActiveState check. systemd-run --unit=TEST-55-OOMD-slowrule \ -p MemoryHigh=3M \ -p OOMRules=slowrule \ - stress-ng --timeout 15s --vm 10 --vm-bytes 50M --vm-keep --vm-method=zero-one + stress-ng --timeout 15s --vm 10 --vm-bytes 50M --vm-keep # Wait long enough for oomd's 1s rule-check loop to evaluate the condition # many times. With LastingSec=1h the kill must not fire. From a17efef1375c07df89e7f02e87fb881dfced53fb Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Mon, 25 May 2026 12:11:23 +0100 Subject: [PATCH 3/3] test: try to detect SIGILL in stress-ng and skip TEST-55-OOMD gracefully The attempt to fix SIGILL in TEST-55-OOMD actually made the test more flaky, as the chosen stress-ng method generates less pressure, so it often fails. Try a different approach: try to detect that SIGILL caused the unit calling stress-ng to fail, and skip gracefully in that case. --- test/units/TEST-55-OOMD.sh | 54 +++++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/test/units/TEST-55-OOMD.sh b/test/units/TEST-55-OOMD.sh index 6689bbdd73..9993e98937 100755 --- a/test/units/TEST-55-OOMD.sh +++ b/test/units/TEST-55-OOMD.sh @@ -94,6 +94,19 @@ else systemd-run -t -p MemoryMax=10M -p MemorySwapMax=0 -p MemoryZSwapMax=0 true fi +# stress-ng can fail with SIGILL due to trying to use AVX-512 on older CPUs, try to detect and avoid failing +stress_ng_sigilled() { + local result status sigill + local unit="${1:?}" + shift + + result=$(systemctl "$@" show "$unit" -P Result) + status=$(systemctl "$@" show "$unit" -P ExecMainStatus) + sigill=$(kill -l ILL) + + [[ "$status" == "$sigill" && ( "$result" == "signal" || "$result" == "core-dump" ) ]] +} + test_basic() { local cgroup_path="${1:?}" shift @@ -123,7 +136,11 @@ test_basic() { if systemctl "$@" status TEST-55-OOMD-testbloat.service; then exit 42; fi if ! systemctl "$@" status TEST-55-OOMD-testchill.service; then exit 24; fi - assert_eq "$(systemctl "$@" show TEST-55-OOMD-testbloat.service -P ManagedOOMKills)" "1" + if stress_ng_sigilled TEST-55-OOMD-testbloat.service "$@"; then + echo "stress-ng died with SIGILL, skipping ManagedOOMKills assertion" + else + assert_eq "$(systemctl "$@" show TEST-55-OOMD-testbloat.service -P ManagedOOMKills)" "1" + fi systemctl "$@" kill --signal=KILL TEST-55-OOMD-testbloat.service || : systemctl "$@" stop TEST-55-OOMD-testbloat.service @@ -171,10 +188,14 @@ EOF sleep 2 done - # testmunch should be killed since testbloat had the avoid xattr on it - if ! systemctl status TEST-55-OOMD-testbloat.service; then exit 25; fi - if systemctl status TEST-55-OOMD-testmunch.service; then exit 43; fi - if ! systemctl status TEST-55-OOMD-testchill.service; then exit 24; fi + if stress_ng_sigilled TEST-55-OOMD-testbloat.service || stress_ng_sigilled TEST-55-OOMD-testmunch.service; then + echo "stress-ng died with SIGILL, skipping testcase_preference_avoid assertions" + else + # testmunch should be killed since testbloat had the avoid xattr on it + if ! systemctl status TEST-55-OOMD-testbloat.service; then exit 25; fi + if systemctl status TEST-55-OOMD-testmunch.service; then exit 43; fi + if ! systemctl status TEST-55-OOMD-testchill.service; then exit 24; fi + fi systemctl kill --signal=KILL TEST-55-OOMD-testbloat.service || : systemctl kill --signal=KILL TEST-55-OOMD-testmunch.service || : @@ -249,8 +270,12 @@ EOF sleep 2 done - if systemctl status TEST-55-OOMD-testmunch.service; then exit 44; fi - if ! systemctl status TEST-55-OOMD-testchill.service; then exit 23; fi + if stress_ng_sigilled TEST-55-OOMD-testmunch.service; then + echo "stress-ng died with SIGILL, skipping testcase_duration_override assertions" + else + if systemctl status TEST-55-OOMD-testmunch.service; then exit 44; fi + if ! systemctl status TEST-55-OOMD-testchill.service; then exit 23; fi + fi systemctl kill --signal=KILL TEST-55-OOMD-testmunch.service || : systemctl stop TEST-55-OOMD-testmunch.service @@ -463,9 +488,13 @@ EOF # many times. With LastingSec=1h the kill must not fire. sleep 6 - # Unit must still be active — if it were killed, Result= would be oom-kill. - assert_eq "$(systemctl show TEST-55-OOMD-slowrule.service -P ActiveState)" "active" - assert_eq "$(systemctl show TEST-55-OOMD-slowrule.service -P Result)" "success" + if stress_ng_sigilled TEST-55-OOMD-slowrule.service; then + echo "stress-ng died with SIGILL, skipping testcase_oom_rulesets_lasting_sec assertions" + else + # Unit must still be active. If it were killed, Result= would be oom-kill. + assert_eq "$(systemctl show TEST-55-OOMD-slowrule.service -P ActiveState)" "active" + assert_eq "$(systemctl show TEST-55-OOMD-slowrule.service -P Result)" "success" + fi systemctl stop TEST-55-OOMD-slowrule.service 2>/dev/null || true @@ -484,6 +513,11 @@ EOF systemctl reload systemd-oomd.service ! systemctl start --wait TEST-55-OOMD-testbloat.service || exit 1 + if stress_ng_sigilled TEST-55-OOMD-testbloat.service; then + echo "stress-ng died with SIGILL, skipping testcase_prekill_hook" + return 0 + fi + # one hook mkdir -p /run/systemd/oomd.prekill.hook/ ncat --recv-only -kUl /run/systemd/oomd.prekill.hook/althook >/tmp/oomd_event.json &