diff --git a/test/units/TEST-07-PID1.transient-unit-container.sh b/test/units/TEST-07-PID1.transient-unit-container.sh
index af2e1e9315..86a5f4ee71 100755
--- a/test/units/TEST-07-PID1.transient-unit-container.sh
+++ b/test/units/TEST-07-PID1.transient-unit-container.sh
@@ -188,7 +188,9 @@ file_write_cleanup() {
     # Remove all the mounts and directories we created
     # These variables reset to empty arrays when the subprocess concludes.
     umount "${CLEANUP_MOUNTS[@]}"
-    rm -rf "${CLEANUP_PATHS[@]}"
+    rm -rf --one-file-system "${CLEANUP_PATHS[@]}"
+
+    set -e
 }
 
 run_testcases
diff --git a/test/units/TEST-23-UNIT-FILE.RuntimeDirectory.sh b/test/units/TEST-23-UNIT-FILE.RuntimeDirectory.sh
index d29331e99f..18e6ff16f0 100755
--- a/test/units/TEST-23-UNIT-FILE.RuntimeDirectory.sh
+++ b/test/units/TEST-23-UNIT-FILE.RuntimeDirectory.sh
@@ -8,6 +8,7 @@ set -o pipefail
 at_exit() {
     set +e
 
+    systemctl stop tmp-aaa.mount
     rm -fr /run/hoge /tmp/aaa
 }
 
diff --git a/test/units/TEST-45-TIMEDATE.sh b/test/units/TEST-45-TIMEDATE.sh
index 8b2d027982..063aeaf45d 100755
--- a/test/units/TEST-45-TIMEDATE.sh
+++ b/test/units/TEST-45-TIMEDATE.sh
@@ -423,7 +423,7 @@ EOF
 }
 
 teardown_timesyncd_resolve() {
-    set +eu
+    set +e
 
     systemctl start systemd-resolved-varlink.socket
     systemctl start systemd-resolved.service
@@ -431,6 +431,8 @@ teardown_timesyncd_resolve() {
     rm -rf /run/systemd/system/systemd-timesyncd.service.d
     systemctl daemon-reload
     systemctl stop systemd-timesyncd
+
+    set -e
 }
 
 testcase_timesyncd_resolve() {
@@ -481,12 +483,14 @@ EOF
 }
 
 teardown_timedated_alternate_paths() {
-    set +eu
+    set +e
 
     rm -rf /run/systemd/system/systemd-timedated.service.d
     rm -rf /run/alternate-path
     systemctl daemon-reload
     systemctl restart systemd-timedated
+
+    set -e
 }
 
 testcase_timedated_alternate_paths() {
diff --git a/test/units/TEST-71-HOSTNAME.sh b/test/units/TEST-71-HOSTNAME.sh
index 3563f9ae00..c758f9c03d 100755
--- a/test/units/TEST-71-HOSTNAME.sh
+++ b/test/units/TEST-71-HOSTNAME.sh
@@ -321,9 +321,10 @@ testcase_wildcard_words() {
 }
 
 teardown_hostnamed_alternate_paths() {
-    set +eu
+    set +e
 
     rm -rf /run/systemd/system/systemd-hostnamed.service.d
+    rm -rf /run/alternate-path
     systemctl daemon-reload
     systemctl restart systemd-hostnamed
     if [[ -f /etc/hostname ]]; then
@@ -332,6 +333,8 @@ teardown_hostnamed_alternate_paths() {
             hostnamectl hostname "${orig}"
         fi
     fi
+
+    set -e
 }
 
 testcase_hostnamed_alternate_paths() {
diff --git a/test/units/TEST-73-LOCALE.sh b/test/units/TEST-73-LOCALE.sh
index b1f19f07b3..bdfb5637f4 100755
--- a/test/units/TEST-73-LOCALE.sh
+++ b/test/units/TEST-73-LOCALE.sh
@@ -652,11 +652,14 @@ testcase_locale_gen_leading_space() {
 }
 
 teardown_localed_alternate_paths() {
-    set +eu
+    set +e
 
     rm -rf /run/systemd/system/systemd-localed.service.d
+    rm -rf /run/alternate-path
     systemctl daemon-reload
     systemctl restart systemd-localed
+
+    set -e
 }
 
 testcase_localed_alternate_paths() {
diff --git a/test/units/TEST-87-AUX-UTILS-VM.sysinstall.sh b/test/units/TEST-87-AUX-UTILS-VM.sysinstall.sh
index 51be486995..b114c97bab 100755
--- a/test/units/TEST-87-AUX-UTILS-VM.sysinstall.sh
+++ b/test/units/TEST-87-AUX-UTILS-VM.sysinstall.sh
@@ -54,7 +54,12 @@ cleanup() {
         systemd-dissect --detach "$LOOPDEV"
         LOOPDEV=""
     fi
-    rm -rf "$WORKDIR"
+    if [[ -n "${WORKDIR:-}" ]]; then
+        rm -rf "$WORKDIR"
+        WORKDIR=""
+    fi
+
+    set -e
 }
 
 create_fake_os_source_tree() {
@@ -199,8 +204,6 @@ testcase_sysinstall_basic() {
         "$WORKDIR/target.img"
 
     validate_image
-
-    cleanup
 }
 
 testcase_sysinstall_varlink_basic() {