From 63bafef12ba867010507963d5aac67f4d64d85e9 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 2 Aug 2024 13:45:41 +0200 Subject: [PATCH 1/7] mkosi: Drop locale tmpfiles snippet Not required anymore as we don't do /usr only anymore and the symlink will be created by the debian systemd package. --- mkosi.extra/usr/lib/tmpfiles.d/locale.conf | 1 - 1 file changed, 1 deletion(-) delete mode 100644 mkosi.extra/usr/lib/tmpfiles.d/locale.conf diff --git a/mkosi.extra/usr/lib/tmpfiles.d/locale.conf b/mkosi.extra/usr/lib/tmpfiles.d/locale.conf deleted file mode 100644 index e1a8e8171a..0000000000 --- a/mkosi.extra/usr/lib/tmpfiles.d/locale.conf +++ /dev/null @@ -1 +0,0 @@ -L /etc/default/locale - - - - ../locale.conf From 5f5b6fa9017c7c5de1e8a3e6b8888fd71a1d3862 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 2 Aug 2024 15:18:45 +0200 Subject: [PATCH 2/7] docs: Update upgrade commands in HACKING.md - Add the required options to make the package managers non interactive - Use apt-get instead of apt - Remove --reinstall from apt-get command so we only install newer packages - Add --needed to pacman command so we only install newer packages --- docs/HACKING.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/HACKING.md b/docs/HACKING.md index b17875f62c..4731dc05d4 100644 --- a/docs/HACKING.md +++ b/docs/HACKING.md @@ -69,10 +69,10 @@ another terminal on your host (choose the right one depending on the distribution of the container or virtual machine): ```sh -mkosi -t none && mkosi ssh dnf upgrade --disablerepo="*" "/work/build/*.rpm" # CentOS/Fedora -mkosi -t none && mkosi ssh apt install --reinstall "/work/build/*.deb" # Debian/Ubuntu -mkosi -t none && mkosi ssh pacman -U "/work/build/*.pkg.tar" # Arch Linux -mkosi -t none && mkosi ssh zypper install --allow-unsigned-rpm "/work/build/*.rpm" # OpenSUSE +mkosi -t none && mkosi ssh dnf upgrade --disablerepo="*" --assumeyes "/work/build/*.rpm" # CentOS/Fedora +mkosi -t none && mkosi ssh apt-get install "/work/build/*.deb" # Debian/Ubuntu +mkosi -t none && mkosi ssh pacman --upgrade --needed --noconfirm "/work/build/*.pkg.tar" # Arch Linux +mkosi -t none && mkosi ssh zypper --non-interactive install --allow-unsigned-rpm "/work/build/*.rpm" # OpenSUSE ``` and optionally restart the daemon(s) you're working on using From 33f400a9e05f3a09d28ac3d8251cf82592a207c6 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 2 Aug 2024 16:16:14 +0200 Subject: [PATCH 3/7] test: Rename INTERACTIVE_DEBUG to TEST_SHELL --- test/README.testsuite | 6 +++--- test/TEST-04-JOURNAL/test.sh | 2 +- test/test-functions | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/README.testsuite b/test/README.testsuite index 22da1cd1a1..c1b048a93d 100644 --- a/test/README.testsuite +++ b/test/README.testsuite @@ -181,7 +181,7 @@ sec). `NSPAWN_TIMEOUT=infinity`: Set a timeout for tests under systemd-nspawn (defaults to 1800 sec). -`INTERACTIVE_DEBUG=1`: Configure the machine to be more *user-friendly* for +`TEST_SHELL=1`: Configure the machine to be more *user-friendly* for interactive debugging (e.g. by setting a usable default terminal, suppressing the shutdown after the test, etc.). @@ -200,11 +200,11 @@ one with `$QEMU_BIN`. ### Debugging the qemu image -If you want to log in the testsuite virtual machine, use `INTERACTIVE_DEBUG=1` +If you want to log in the testsuite virtual machine, use `TEST_SHELL=1` and log in as root: ```shell -$ sudo make -C test/TEST-01-BASIC INTERACTIVE_DEBUG=1 run +$ sudo make -C test/TEST-01-BASIC TEST_SHELL=1 run ``` The root password is empty. diff --git a/test/TEST-04-JOURNAL/test.sh b/test/TEST-04-JOURNAL/test.sh index a7aa71fce3..01d0b66ee7 100755 --- a/test/TEST-04-JOURNAL/test.sh +++ b/test/TEST-04-JOURNAL/test.sh @@ -19,7 +19,7 @@ test_append_files() { # Since we nuke the journal repeatedly during this test, let's redirect # stdout/stderr to the console as well to make the test a bit more debug-able. - if ! get_bool "${INTERACTIVE_DEBUG:-}"; then + if ! get_bool "${TEST_SHELL:-}"; then dropin_dir="${workspace:?}/etc/systemd/system/TEST-04-JOURNAL.service.d/" mkdir -p "$dropin_dir" printf '[Service]\nStandardOutput=journal+console\nStandardError=journal+console' >"$dropin_dir/99-stdout.conf" diff --git a/test/test-functions b/test/test-functions index 3c6f763973..e41e5792a3 100644 --- a/test/test-functions +++ b/test/test-functions @@ -571,7 +571,7 @@ run_qemu() { ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"} ) - if ! get_bool "$INTERACTIVE_DEBUG" && ! get_bool "$TEST_SKIP_SHUTDOWN"; then + if ! get_bool "$TEST_SHELL" && ! get_bool "$TEST_SKIP_SHUTDOWN"; then kernel_params+=( "panic=1" "softlockup_panic=1" @@ -665,7 +665,7 @@ run_nspawn() { ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"} ) - if get_bool "$INTERACTIVE_DEBUG"; then + if get_bool "$TEST_SHELL"; then nspawn_options+=("--console=interactive") elif ! get_bool "$TEST_SKIP_SHUTDOWN"; then kernel_params+=("systemd.wants=end.service") @@ -1985,7 +1985,7 @@ install_debug_tools() { dinfo "Install debug tools" image_install -o "${DEBUGTOOLS[@]}" - if get_bool "$INTERACTIVE_DEBUG"; then + if get_bool "$TEST_SHELL"; then # Set default TERM from vt220 to linux, so at least basic key shortcuts work local getty_override="${initdir:?}/etc/systemd/system/serial-getty@.service.d" mkdir -p "$getty_override" From dd1c01b20fb8eef6a4d41ac7f0d3d83399e27593 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 2 Aug 2024 15:46:41 +0200 Subject: [PATCH 4/7] test: Implement TEST_SHELL for mkosi based test runner TEST_SHELL can be used to get a shell in the integration test environment without actually immediately starting the test. --- test/integration-test-wrapper.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/test/integration-test-wrapper.py b/test/integration-test-wrapper.py index 596761aa89..783c05f90b 100755 --- a/test/integration-test-wrapper.py +++ b/test/integration-test-wrapper.py @@ -62,20 +62,30 @@ def main(): exit(77) keep_journal = os.getenv("TEST_SAVE_JOURNAL", "fail") + shell = bool(int(os.getenv("TEST_SHELL", "0"))) + + if shell and not sys.stderr.isatty(): + print(f"--interactive must be passed to meson test to use TEST_SHELL=1", file=sys.stderr) + exit(1) name = args.name + (f"-{i}" if (i := os.getenv("MESON_TEST_ITERATION")) else "") dropin = textwrap.dedent( """\ - [Unit] - SuccessAction=exit - SuccessActionExitStatus=123 - [Service] StandardOutput=journal+console """ ) + if not shell: + dropin += textwrap.dedent( + f""" + [Unit] + SuccessAction=exit + SuccessActionExitStatus=123 + """ + ) + if os.getenv("TEST_MATCH_SUBTEST"): dropin += textwrap.dedent( f""" @@ -92,6 +102,7 @@ def main(): """ ) + journal_file = None if not sys.stderr.isatty(): dropin += textwrap.dedent( """ @@ -102,14 +113,13 @@ def main(): journal_file = (args.meson_build_dir / (f"test/journal/{name}.journal")).absolute() journal_file.unlink(missing_ok=True) - else: + elif not shell: dropin += textwrap.dedent( """ [Unit] Wants=multi-user.target """ ) - journal_file = None cmd = [ args.mkosi, @@ -140,7 +150,7 @@ def main(): ' '.join([ 'systemd.hostname=H', f"SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/{args.name}.units:/usr/lib/systemd/tests/testdata/units:", - f"systemd.unit={args.unit}", + *([f"systemd.unit={args.unit}"] if not shell else []), 'systemd.mask=systemd-networkd-wait-online.service', *( [ @@ -163,8 +173,8 @@ def main(): if journal_file and (keep_journal == "0" or (result.returncode in (args.exit_code, 77) and keep_journal == "fail")): journal_file.unlink(missing_ok=True) - if result.returncode in (args.exit_code, 77): - exit(0 if result.returncode == args.exit_code else 77) + if shell or result.returncode in (args.exit_code, 77): + exit(0 if shell or result.returncode == args.exit_code else 77) if journal_file: ops = [] From ea696c5f5300c240ae2749f7791aaa78837d556b Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 2 Aug 2024 16:22:37 +0200 Subject: [PATCH 5/7] mkosi: Enable Autologin= again on Debian Debian uses /usr/bin/login from the shadow package instead of util-linux which doesn't support credentials. Let's enable autologin the old fashioned way for now. https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833256 --- mkosi.conf.d/10-debian-ubuntu/mkosi.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mkosi.conf.d/10-debian-ubuntu/mkosi.conf b/mkosi.conf.d/10-debian-ubuntu/mkosi.conf index 43e078117f..d14746f1ed 100644 --- a/mkosi.conf.d/10-debian-ubuntu/mkosi.conf +++ b/mkosi.conf.d/10-debian-ubuntu/mkosi.conf @@ -8,6 +8,10 @@ Distribution=|ubuntu PackageManagerTrees=mkosi-pinning.pref:/etc/apt/preferences.d/mkosi-pinning.pref [Content] +# Debian ships /usr/bin/login from shadow instead of util-linux which doesn't support credentials so we +# enable autologin the old-fashioned way. +Autologin=yes + VolatilePackages= libnss-myhostname libnss-mymachines From edc6592e532806fe9ac28400138d5d4e1492a7d0 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 2 Aug 2024 16:23:40 +0200 Subject: [PATCH 6/7] mkosi: Switch back to btrfs The next commit will introduce a way to iterate on integration tests which depends on btrfs specific features. We leave CentOS on ext4 as its kernel does not support btrfs. --- mkosi.conf | 3 --- mkosi.conf.d/10-centos/mkosi.conf | 5 +++-- mkosi.repart/10-root.conf | 1 + 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/mkosi.conf b/mkosi.conf index e6b0e41135..6545e1a41f 100644 --- a/mkosi.conf +++ b/mkosi.conf @@ -46,9 +46,6 @@ ExtraTrees= Initrds=%O/initrd -Environment= - SYSTEMD_REPART_OVERRIDE_FSTYPE_ROOT=%F - # Disable relabeling by default as it only matters for TEST-06-SELINUX, takes a non-trivial amount of time # and results in lots of errors when building images as a regular user. SELinuxRelabel=no diff --git a/mkosi.conf.d/10-centos/mkosi.conf b/mkosi.conf.d/10-centos/mkosi.conf index d97b08138e..dd462efcee 100644 --- a/mkosi.conf.d/10-centos/mkosi.conf +++ b/mkosi.conf.d/10-centos/mkosi.conf @@ -8,8 +8,9 @@ Release=10 [Content] Environment= - # We'd prefer to use XFS here but it fails to mount on duplicate filesystem UUIDs which - # happens when running tests in parallel so we use ext4 instead. + # CentOS doesn't support btrfs so we have to use something else. We'd prefer to use XFS here but it + # fails to mount on duplicate filesystem UUIDs which happens when running tests in parallel so we use + # ext4 instead. SYSTEMD_REPART_OVERRIDE_FSTYPE_ROOT=ext4 # The kernel versions in CentOS Stream 9 doesn't support orphan_file, but later versions of # mkfs.ext4 enabled it by default, so we disable it explicitly. diff --git a/mkosi.repart/10-root.conf b/mkosi.repart/10-root.conf index c774086fb4..3c25dbfb14 100644 --- a/mkosi.repart/10-root.conf +++ b/mkosi.repart/10-root.conf @@ -2,6 +2,7 @@ [Partition] Type=root +Format=btrfs CopyFiles=/ SizeMinBytes=8G SizeMaxBytes=8G From af153e36ae67c242251951c12d6d6b6ae4783845 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 2 Aug 2024 16:25:03 +0200 Subject: [PATCH 7/7] test: Add a way to quickly iterate on an integration test Rebuilding the integration test every time is very slow. Let's introduce a way to iterate on an integration test without rebuilding the image every time. By making a btrfs snapshot before we run the integration test, we can then systemctl soft-reboot after running the test to restore the rootfs to a pristine state before running the test again. As /run/nextroot will get nuked on reboot or soft-reboot, we introduce a tmpfiles snippet to make sure it is recreated every (soft-)reboot and adapt the existing tests to deal with this new symlink. --- mkosi.extra/usr/lib/tmpfiles.d/snapshot.conf | 3 ++ test/README.testsuite | 39 ++++++++++++++ test/integration-test-setup.sh | 56 ++++++++++++++++++++ test/meson.build | 8 +-- test/test.service.in | 3 ++ test/units/TEST-09-REBOOT.sh | 4 ++ test/units/TEST-82-SOFTREBOOT.sh | 15 ++++++ 7 files changed, 125 insertions(+), 3 deletions(-) create mode 100644 mkosi.extra/usr/lib/tmpfiles.d/snapshot.conf create mode 100755 test/integration-test-setup.sh diff --git a/mkosi.extra/usr/lib/tmpfiles.d/snapshot.conf b/mkosi.extra/usr/lib/tmpfiles.d/snapshot.conf new file mode 100644 index 0000000000..fea660e71e --- /dev/null +++ b/mkosi.extra/usr/lib/tmpfiles.d/snapshot.conf @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +L /run/nextroot - - - - /snapshot diff --git a/test/README.testsuite b/test/README.testsuite index c1b048a93d..585bddd3dd 100644 --- a/test/README.testsuite +++ b/test/README.testsuite @@ -86,6 +86,45 @@ mkosi in the systemd reposistory, so any local modifications to the mkosi configuration (e.g. in `mkosi.local.conf`) are automatically picked up and used by the integration tests as well. +## Iterating on an integration test + +To iterate on an integration test, let's first get a shell in the integration test environment by running +the following: + +```shell +$ meson compile -C build mkosi && SYSTEMD_INTEGRATION_TESTS=1 TEST_SHELL=1 meson test -C build --no-rebuild -i TEST-01-BASIC +``` + +This will get us a shell in the integration test environment after booting the machine without running the +integration test itself. After booting, we can verify the integration test passes by running it manually, +for example with `systemctl start TEST-01-BASIC`. + +Now you can extend the test in whatever way you like to add more coverage of existing features or to add +coverage for a new feature. Once you've finished writing the logic and want to rerun the test, run the +the following on the host: + +```shell +$ mkosi -t none +``` + +This will rebuild the distribution packages without rebuilding the entire integration test image. Next, run +the following in the integration test machine: + +```shell +$ systemctl soft-reboot +$ systemctl start TEST-01-BASIC +``` + +A soft-reboot is required to make sure all the leftover state from the previous run of the test is cleaned +up by soft-rebooting into the btrfs snapshot we made before running the test. After the soft-reboot, +re-running the test will first install the new packages we just built, make a new snapshot and finally run +the test again. You can keep running the loop of `mkosi -t none`, `systemctl soft-reboot` and +`systemctl start ...` until the changes to the integration test are working. + +If you're debugging a failing integration test (running `meson test --interactive` without `TEST_SHELL`), +there's no need to run `systemctl start ...`, running `systemctl soft-reboot` on its own is sufficient to +rerun the test. + ## Running the integration tests the old fashioned way The extended testsuite only works with UID=0. It consists of the subdirectories diff --git a/test/integration-test-setup.sh b/test/integration-test-setup.sh new file mode 100755 index 0000000000..71f576f61f --- /dev/null +++ b/test/integration-test-setup.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# SPDX-License-Identifier: LGPL-2.1-or-later +set -e + +case "$1" in + setup) + if [[ -f "$STATE_DIRECTORY/inprogress" ]]; then + exit 0 + fi + + if [[ -d /snapshot ]]; then + echo "Run systemctl soft-reboot first to make sure the test runs within a pristine rootfs" >&2 + exit 1 + fi + + . /usr/lib/os-release + + if test -n "$(shopt -s nullglob; echo /work/build/*.{rpm,deb,pkg.tar})"; then + case "$ID" in + arch) + pacman --upgrade --needed --noconfirm /work/build/*.pkg.tar + ;; + debian|ubuntu) + apt-get install /work/build/*.deb + ;; + opensuse*) + zypper --non-interactive install --allow-unsigned-rpm /work/build/*.rpm + ;; + centos|fedora) + dnf upgrade --assumeyes --disablerepo="*" /work/build/*.rpm + ;; + *) + echo "Unknown distribution $ID" >&2 + exit 1 + esac + fi + + # TODO: Use a proper flat btrfs subvolume layout once we can create subvolumes without privileged in + # systemd-repart (see https://github.com/systemd/systemd/pull/33498). Until that's possible, we nest + # snapshots within each other. + if command -v btrfs >/dev/null && [[ "$(stat --file-system --format %T /)" == "btrfs" ]]; then + btrfs subvolume snapshot / /snapshot + fi + + touch "$STATE_DIRECTORY/inprogress" + ;; + finalize) + # If we're rebooting, the test does a reboot as part of its execution and we shouldn't remove /inprogress. + if ! [[ "$(systemctl list-jobs)" =~ reboot.target|kexec.target|soft-reboot.target ]]; then + rm -f "$STATE_DIRECTORY/inprogress" + fi + ;; + *) + echo "Unknown verb $1" >&2 + exit 1 +esac diff --git a/test/meson.build b/test/meson.build index 173d90cc8e..6acff37508 100644 --- a/test/meson.build +++ b/test/meson.build @@ -142,9 +142,11 @@ endif ############################################################ if install_tests - install_data('run-unit-tests.py', - install_mode : 'rwxr-xr-x', - install_dir : testsdir) + foreach script : ['integration-test-setup.sh', 'run-unit-tests.py'] + install_data(script, + install_mode : 'rwxr-xr-x', + install_dir : testsdir) + endforeach endif ############################################################ diff --git a/test/test.service.in b/test/test.service.in index 790c513da4..ef9dba30c4 100644 --- a/test/test.service.in +++ b/test/test.service.in @@ -4,9 +4,12 @@ Description=%N Wants=basic.target network.target @wants@ After=basic.target network.target @after@ Before=getty-pre.target +StateDirectory=%N [Service] ExecStartPre=rm -f /failed /testok +ExecStartPre=/usr/lib/systemd/tests/integration-test-setup.sh setup ExecStart=@command@ +ExecStopPost=/usr/lib/systemd/tests/integration-test-setup.sh finalize Type=oneshot MemoryAccounting=@memory-accounting@ diff --git a/test/units/TEST-09-REBOOT.sh b/test/units/TEST-09-REBOOT.sh index 85630b618b..014ea31db6 100755 --- a/test/units/TEST-09-REBOOT.sh +++ b/test/units/TEST-09-REBOOT.sh @@ -17,7 +17,11 @@ systemd-cat journalctl --list-boots run_subtests if [[ "$REBOOT_COUNT" -lt "$NUM_REBOOT" ]]; then + SYSTEMCTL_SKIP_AUTO_SOFT_REBOOT=1 + export SYSTEMCTL_SKIP_AUTO_SOFT_REBOOT systemctl_final reboot + # Now block until the reboot killing spree kills us. + exec sleep infinity elif [[ "$REBOOT_COUNT" -gt "$NUM_REBOOT" ]]; then assert_not_reached fi diff --git a/test/units/TEST-82-SOFTREBOOT.sh b/test/units/TEST-82-SOFTREBOOT.sh index f86bc929d0..9f3d4066c3 100755 --- a/test/units/TEST-82-SOFTREBOOT.sh +++ b/test/units/TEST-82-SOFTREBOOT.sh @@ -19,6 +19,21 @@ at_exit() { trap at_exit EXIT +# Because this test tests soft-reboot, we have to get rid of the symlink we put at +# /run/nextroot to allow rebooting into the previous snapshot if the test fails for +# the duration of the test. However, let's make sure we put the symlink back in place +# if the test fails. +if [[ -L /run/nextroot ]]; then + at_error() { + mountpoint -q /run/nextroot && umount -R /run/nextroot + rm -rf /run/nextroot + ln -sf /snapshot /run/nextroot + } + + trap at_error ERR + rm -f /run/nextroot +fi + systemd-analyze log-level debug export SYSTEMD_LOG_LEVEL=debug