Merge pull request #33912 from DaanDeMeyer/mkosi

test: Add a way to quickly iterate on an integration test
2024-08-05 16:21:48 +02:00
parent 857e4528f5 af153e36ae
commit bec6b53f3c
16 changed files with 163 additions and 29 deletions
@@ -69,10 +69,10 @@ another terminal on your host (choose the right one depending on the
 distribution of the container or virtual machine):

 ```sh
-mkosi -t none && mkosi ssh dnf upgrade --disablerepo="*" "/work/build/*.rpm" # CentOS/Fedora
-mkosi -t none && mkosi ssh apt install --reinstall "/work/build/*.deb" # Debian/Ubuntu
-mkosi -t none && mkosi ssh pacman -U "/work/build/*.pkg.tar" # Arch Linux
-mkosi -t none && mkosi ssh zypper install --allow-unsigned-rpm "/work/build/*.rpm" # OpenSUSE
+mkosi -t none && mkosi ssh dnf upgrade --disablerepo="*" --assumeyes "/work/build/*.rpm" # CentOS/Fedora
+mkosi -t none && mkosi ssh apt-get install "/work/build/*.deb" # Debian/Ubuntu
+mkosi -t none && mkosi ssh pacman --upgrade --needed --noconfirm "/work/build/*.pkg.tar" # Arch Linux
+mkosi -t none && mkosi ssh zypper --non-interactive install --allow-unsigned-rpm "/work/build/*.rpm" # OpenSUSE
 ```

 and optionally restart the daemon(s) you're working on using
@@ -46,9 +46,6 @@ ExtraTrees=

 Initrds=%O/initrd

-Environment=
-        SYSTEMD_REPART_OVERRIDE_FSTYPE_ROOT=%F
-
 # Disable relabeling by default as it only matters for TEST-06-SELINUX, takes a non-trivial amount of time
 # and results in lots of errors when building images as a regular user.
 SELinuxRelabel=no
@@ -8,8 +8,9 @@ Release=10

 [Content]
 Environment=
-        # We'd prefer to use XFS here but it fails to mount on duplicate filesystem UUIDs which
-        # happens when running tests in parallel so we use ext4 instead.
+        # CentOS doesn't support btrfs so we have to use something else. We'd prefer to use XFS here but it
+        # fails to mount on duplicate filesystem UUIDs which happens when running tests in parallel so we use
+        # ext4 instead.
        SYSTEMD_REPART_OVERRIDE_FSTYPE_ROOT=ext4
        # The kernel versions in CentOS Stream 9 doesn't support orphan_file, but later versions of
        # mkfs.ext4 enabled it by default, so we disable it explicitly.
@@ -8,6 +8,10 @@ Distribution=|ubuntu
 PackageManagerTrees=mkosi-pinning.pref:/etc/apt/preferences.d/mkosi-pinning.pref

 [Content]
+# Debian ships /usr/bin/login from shadow instead of util-linux which doesn't support credentials so we
+# enable autologin the old-fashioned way.
+Autologin=yes
+
 VolatilePackages=
        libnss-myhostname
        libnss-mymachines
@@ -1 +0,0 @@
-L /etc/default/locale - - - - ../locale.conf
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+L /run/nextroot - - - - /snapshot
@@ -2,6 +2,7 @@

 [Partition]
 Type=root
+Format=btrfs
 CopyFiles=/
 SizeMinBytes=8G
 SizeMaxBytes=8G
@@ -86,6 +86,45 @@ mkosi in the systemd reposistory, so any local modifications to the mkosi
 configuration (e.g. in `mkosi.local.conf`) are automatically picked up and used
 by the integration tests as well.

+## Iterating on an integration test
+
+To iterate on an integration test, let's first get a shell in the integration test environment by running
+the following:
+
+```shell
+$ meson compile -C build mkosi && SYSTEMD_INTEGRATION_TESTS=1 TEST_SHELL=1 meson test -C build --no-rebuild -i TEST-01-BASIC
+```
+
+This will get us a shell in the integration test environment after booting the machine without running the
+integration test itself. After booting, we can verify the integration test passes by running it manually,
+for example with `systemctl start TEST-01-BASIC`.
+
+Now you can extend the test in whatever way you like to add more coverage of existing features or to add
+coverage for a new feature. Once you've finished writing the logic and want to rerun the test, run the
+the following on the host:
+
+```shell
+$ mkosi -t none
+```
+
+This will rebuild the distribution packages without rebuilding the entire integration test image. Next, run
+the following in the integration test machine:
+
+```shell
+$ systemctl soft-reboot
+$ systemctl start TEST-01-BASIC
+```
+
+A soft-reboot is required to make sure all the leftover state from the previous run of the test is cleaned
+up by soft-rebooting into the btrfs snapshot we made before running the test. After the soft-reboot,
+re-running the test will first install the new packages we just built, make a new snapshot and finally run
+the test again. You can keep running the loop of `mkosi -t none`, `systemctl soft-reboot` and
+`systemctl start ...` until the changes to the integration test are working.
+
+If you're debugging a failing integration test (running `meson test --interactive` without `TEST_SHELL`),
+there's no need to run `systemctl start ...`, running `systemctl soft-reboot` on its own is sufficient to
+rerun the test.
+
 ## Running the integration tests the old fashioned way

 The extended testsuite only works with UID=0. It consists of the subdirectories
@@ -181,7 +220,7 @@ sec).
 `NSPAWN_TIMEOUT=infinity`: Set a timeout for tests under systemd-nspawn
 (defaults to 1800 sec).

-`INTERACTIVE_DEBUG=1`: Configure the machine to be more *user-friendly* for
+`TEST_SHELL=1`: Configure the machine to be more *user-friendly* for
 interactive debugging (e.g. by setting a usable default terminal, suppressing
 the shutdown after the test, etc.).

@@ -200,11 +239,11 @@ one with `$QEMU_BIN`.

 ### Debugging the qemu image

-If you want to log in the testsuite virtual machine, use `INTERACTIVE_DEBUG=1`
+If you want to log in the testsuite virtual machine, use `TEST_SHELL=1`
 and log in as root:

 ```shell
-$ sudo make -C test/TEST-01-BASIC INTERACTIVE_DEBUG=1 run
+$ sudo make -C test/TEST-01-BASIC TEST_SHELL=1 run
 ```

 The root password is empty.
@@ -19,7 +19,7 @@ test_append_files() {

    # Since we nuke the journal repeatedly during this test, let's redirect
    # stdout/stderr to the console as well to make the test a bit more debug-able.
-    if ! get_bool "${INTERACTIVE_DEBUG:-}"; then
+    if ! get_bool "${TEST_SHELL:-}"; then
        dropin_dir="${workspace:?}/etc/systemd/system/TEST-04-JOURNAL.service.d/"
        mkdir -p "$dropin_dir"
        printf '[Service]\nStandardOutput=journal+console\nStandardError=journal+console' >"$dropin_dir/99-stdout.conf"
@@ -0,0 +1,56 @@
+#!/bin/bash
+# SPDX-License-Identifier: LGPL-2.1-or-later
+set -e
+
+case "$1" in
+    setup)
+        if [[ -f "$STATE_DIRECTORY/inprogress" ]]; then
+            exit 0
+        fi
+
+        if [[ -d /snapshot ]]; then
+            echo "Run systemctl soft-reboot first to make sure the test runs within a pristine rootfs" >&2
+            exit 1
+        fi
+
+        . /usr/lib/os-release
+
+        if test -n "$(shopt -s nullglob; echo /work/build/*.{rpm,deb,pkg.tar})"; then
+            case "$ID" in
+                arch)
+                    pacman --upgrade --needed --noconfirm /work/build/*.pkg.tar
+                    ;;
+                debian|ubuntu)
+                    apt-get install /work/build/*.deb
+                    ;;
+                opensuse*)
+                    zypper --non-interactive install --allow-unsigned-rpm /work/build/*.rpm
+                    ;;
+                centos|fedora)
+                    dnf upgrade --assumeyes --disablerepo="*" /work/build/*.rpm
+                    ;;
+                *)
+                    echo "Unknown distribution $ID" >&2
+                    exit 1
+            esac
+        fi
+
+        # TODO: Use a proper flat btrfs subvolume layout once we can create subvolumes without privileged in
+        # systemd-repart (see https://github.com/systemd/systemd/pull/33498). Until that's possible, we nest
+        # snapshots within each other.
+        if command -v btrfs >/dev/null && [[ "$(stat --file-system --format %T /)" == "btrfs" ]]; then
+            btrfs subvolume snapshot / /snapshot
+        fi
+
+        touch "$STATE_DIRECTORY/inprogress"
+        ;;
+    finalize)
+        # If we're rebooting, the test does a reboot as part of its execution and we shouldn't remove /inprogress.
+        if ! [[ "$(systemctl list-jobs)" =~ reboot.target|kexec.target|soft-reboot.target ]]; then
+            rm -f "$STATE_DIRECTORY/inprogress"
+        fi
+        ;;
+    *)
+        echo "Unknown verb $1" >&2
+        exit 1
+esac
@@ -62,20 +62,30 @@ def main():
        exit(77)

    keep_journal = os.getenv("TEST_SAVE_JOURNAL", "fail")
+    shell = bool(int(os.getenv("TEST_SHELL", "0")))
+
+    if shell and not sys.stderr.isatty():
+        print(f"--interactive must be passed to meson test to use TEST_SHELL=1", file=sys.stderr)
+        exit(1)

    name = args.name + (f"-{i}" if (i := os.getenv("MESON_TEST_ITERATION")) else "")

    dropin = textwrap.dedent(
        """\
-        [Unit]
-        SuccessAction=exit
-        SuccessActionExitStatus=123
-
        [Service]
        StandardOutput=journal+console
        """
    )

+    if not shell:
+        dropin += textwrap.dedent(
+            f"""
+            [Unit]
+            SuccessAction=exit
+            SuccessActionExitStatus=123
+            """
+        )
+
    if os.getenv("TEST_MATCH_SUBTEST"):
        dropin += textwrap.dedent(
            f"""
@@ -92,6 +102,7 @@ def main():
            """
        )

+    journal_file = None
    if not sys.stderr.isatty():
        dropin += textwrap.dedent(
            """
@@ -102,14 +113,13 @@ def main():

        journal_file = (args.meson_build_dir / (f"test/journal/{name}.journal")).absolute()
        journal_file.unlink(missing_ok=True)
-    else:
+    elif not shell:
        dropin += textwrap.dedent(
            """
            [Unit]
            Wants=multi-user.target
            """
        )
-        journal_file = None

    cmd = [
        args.mkosi,
@@ -140,7 +150,7 @@ def main():
        ' '.join([
            'systemd.hostname=H',
            f"SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/{args.name}.units:/usr/lib/systemd/tests/testdata/units:",
-            f"systemd.unit={args.unit}",
+            *([f"systemd.unit={args.unit}"] if not shell else []),
            'systemd.mask=systemd-networkd-wait-online.service',
            *(
                [
@@ -163,8 +173,8 @@ def main():
    if journal_file and (keep_journal == "0" or (result.returncode in (args.exit_code, 77) and keep_journal == "fail")):
        journal_file.unlink(missing_ok=True)

-    if result.returncode in (args.exit_code, 77):
-        exit(0 if result.returncode == args.exit_code else 77)
+    if shell or result.returncode in (args.exit_code, 77):
+        exit(0 if shell or result.returncode == args.exit_code else 77)

    if journal_file:
        ops = []
@@ -142,9 +142,11 @@ endif
 ############################################################

 if install_tests
-        install_data('run-unit-tests.py',
-                     install_mode : 'rwxr-xr-x',
-                     install_dir : testsdir)
+        foreach script : ['integration-test-setup.sh', 'run-unit-tests.py']
+                install_data(script,
+                             install_mode : 'rwxr-xr-x',
+                             install_dir : testsdir)
+        endforeach
 endif

 ############################################################
@@ -571,7 +571,7 @@ run_qemu() {
        ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"}
    )

-    if ! get_bool "$INTERACTIVE_DEBUG" && ! get_bool "$TEST_SKIP_SHUTDOWN"; then
+    if ! get_bool "$TEST_SHELL" && ! get_bool "$TEST_SKIP_SHUTDOWN"; then
        kernel_params+=(
            "panic=1"
            "softlockup_panic=1"
@@ -665,7 +665,7 @@ run_nspawn() {
        ${TEST_MATCH_TESTCASE:+"systemd.setenv=TEST_MATCH_TESTCASE=$TEST_MATCH_TESTCASE"}
    )

-    if get_bool "$INTERACTIVE_DEBUG"; then
+    if get_bool "$TEST_SHELL"; then
        nspawn_options+=("--console=interactive")
    elif ! get_bool "$TEST_SKIP_SHUTDOWN"; then
        kernel_params+=("systemd.wants=end.service")
@@ -1985,7 +1985,7 @@ install_debug_tools() {
    dinfo "Install debug tools"
    image_install -o "${DEBUGTOOLS[@]}"

-    if get_bool "$INTERACTIVE_DEBUG"; then
+    if get_bool "$TEST_SHELL"; then
        # Set default TERM from vt220 to linux, so at least basic key shortcuts work
        local getty_override="${initdir:?}/etc/systemd/system/serial-getty@.service.d"
        mkdir -p "$getty_override"
@@ -4,9 +4,12 @@ Description=%N
 Wants=basic.target network.target @wants@
 After=basic.target network.target @after@
 Before=getty-pre.target
+StateDirectory=%N

 [Service]
 ExecStartPre=rm -f /failed /testok
+ExecStartPre=/usr/lib/systemd/tests/integration-test-setup.sh setup
 ExecStart=@command@
+ExecStopPost=/usr/lib/systemd/tests/integration-test-setup.sh finalize
 Type=oneshot
 MemoryAccounting=@memory-accounting@
@@ -17,7 +17,11 @@ systemd-cat journalctl --list-boots
 run_subtests

 if [[ "$REBOOT_COUNT" -lt "$NUM_REBOOT" ]]; then
+    SYSTEMCTL_SKIP_AUTO_SOFT_REBOOT=1
+    export SYSTEMCTL_SKIP_AUTO_SOFT_REBOOT
    systemctl_final reboot
+    # Now block until the reboot killing spree kills us.
+    exec sleep infinity
 elif [[ "$REBOOT_COUNT" -gt "$NUM_REBOOT" ]]; then
    assert_not_reached
 fi
@@ -19,6 +19,21 @@ at_exit() {

 trap at_exit EXIT

+# Because this test tests soft-reboot, we have to get rid of the symlink we put at
+# /run/nextroot to allow rebooting into the previous snapshot if the test fails for
+# the duration of the test. However, let's make sure we put the symlink back in place
+# if the test fails.
+if [[ -L /run/nextroot ]]; then
+    at_error() {
+        mountpoint -q /run/nextroot && umount -R /run/nextroot
+        rm -rf /run/nextroot
+        ln -sf /snapshot /run/nextroot
+    }
+
+    trap at_error ERR
+    rm -f /run/nextroot
+fi
+
 systemd-analyze log-level debug

 export SYSTEMD_LOG_LEVEL=debug
				`@@ -1 +0,0 @@`
				`L /etc/default/locale - - - - ../locale.conf`