systemd/test/integration-tests/integration-test-wrapper.py

#!/usr/bin/env python3
# SPDX-License-Identifier: LGPL-2.1-or-later

"""Test wrapper command for driving integration tests."""

import argparse
import base64
import dataclasses
import datetime
import json
import os
import re
import shlex
import shutil
import signal
import subprocess
import sys
import tempfile
import textwrap
from pathlib import Path
from types import FrameType
from typing import Optional

EMERGENCY_EXIT_DROPIN = '''\
[Unit]
Wants=emergency-exit.service
'''


EMERGENCY_EXIT_SERVICE = '''\
[Unit]
DefaultDependencies=no
Conflicts=shutdown.target
Conflicts=rescue.service
Before=shutdown.target
Before=rescue.service
FailureAction=exit

[Service]
ExecStart=false
'''


@dataclasses.dataclass(frozen=True)
class Summary:
    distribution: str
    release: str
    architecture: str
    builddir: Path
    buildsubdir: Path
    environment: dict[str, str]

    @classmethod
    def get(cls, args: argparse.Namespace) -> 'Summary':
        j = json.loads(
            subprocess.run(
                [
                    args.mkosi,
                    '--directory', os.fspath(args.mkosi_dir),
                    '--json',
                    'summary',
                ],
                stdout=subprocess.PIPE,
                text=True,
            ).stdout
        )  # fmt: skip

        return Summary(
            distribution=j['Images'][-1]['Distribution'],
            release=j['Images'][-1]['Release'],
            architecture=j['Images'][-1]['Architecture'],
            builddir=Path(j['Images'][-1]['BuildDirectory']),
            buildsubdir=Path(j['Images'][-1]['BuildSubdirectory']),
            environment=j['Images'][-1]['Environment'],
        )


def process_coredumps(args: argparse.Namespace, journal_file: Path) -> bool:
    # Collect executable paths of all coredumps and filter out the expected ones.

    if args.coredump_exclude_regex:
        exclude_regex = re.compile(args.coredump_exclude_regex)
    else:
        exclude_regex = None

    result = subprocess.run(
        [
            'coredumpctl',
            '--file', journal_file,
            '--json=short',
        ],
        stdout=subprocess.PIPE,
        text=True,
    )  # fmt: skip

    # coredumpctl returns a non-zero exit status if there are no coredumps.
    if result.returncode != 0:
        return False

    coredumps = json.loads(result.stdout)

    coredumps = [
        coredump for coredump in coredumps if not exclude_regex or not exclude_regex.search(coredump['exe'])
    ]

    if not coredumps:
        return False

    subprocess.run(
        [
            'coredumpctl',
            '--file', journal_file,
            '--no-pager',
            'info',
            *(coredump['exe'] for coredump in coredumps),
        ],
        check=True,
    )  # fmt: skip

    return True


def process_sanitizer_report(args: argparse.Namespace, journal_file: Path) -> bool:
    # Collect sanitizer reports from the journal file.

    if args.sanitizer_exclude_regex:
        exclude_regex = re.compile(args.sanitizer_exclude_regex)
    else:
        exclude_regex = None

    total = 0
    fatal = 0
    asan = 0
    ubsan = 0
    msan = 0

    # Internal errors:
    # ==2554==LeakSanitizer has encountered a fatal error.
    # ==2554==HINT: For debugging, try setting environment variable LSAN_OPTIONS=verbosity=1:log_threads=1
    # ==2554==HINT: LeakSanitizer does not work under ptrace (strace, gdb, etc)
    fatal_begin = re.compile(r'==[0-9]+==.+?\w+Sanitizer has encountered a fatal error')
    fatal_end = re.compile(r'==[0-9]+==HINT:\s+\w+Sanitizer')

    # 'Standard' errors:
    #
    # TODO: there's currently a bug in LLVM 22 due to which certain systemd
    # units can throw the following warning:
    # [ 3366.747202] systemd-oomd[93]: ==93==WARNING: ptrace appears to be blocked (is seccomp enabled?).
    #                LeakSanitizer may hang.
    # [ 3366.747202] systemd-oomd[93]: ==93==Child exited with signal 15.
    #
    # which is then picked up by the following regex and causes the test to
    # fail. Let's, temporarily, exclude this warning from the regex to mitigate
    # this.
    #
    # See: https://github.com/llvm/llvm-project/issues/193714
    standard_begin = re.compile(r'([0-9]+: runtime error|==[0-9]+==(?!WARNING: ptrace).+?\w+Sanitizer)')
    standard_end = re.compile(r'SUMMARY:\s+(\w+)Sanitizer')

    # extract COMM
    find_comm = re.compile(r'^\[[.0-9 ]+?\]\s(.*?:)\s')

    with subprocess.Popen(
        [
            'journalctl',
            '--output', 'short-monotonic',
            '--no-hostname',
            '--quiet',
            '--priority', 'info',
            '--file', journal_file,
        ],
        stdout=subprocess.PIPE,
        text=True,
    ) as p:  # fmt: skip
        assert p.stdout

        is_fatal = False
        is_standard = False
        comm = None

        while True:
            line = p.stdout.readline()
            if not line and p.poll() is not None:
                break

            if not is_standard and fatal_begin.search(line):
                m = find_comm.search(line)
                if m:
                    if exclude_regex and exclude_regex.search(m.group(1)):
                        continue
                    comm = m.group(1)

                sys.stderr.write(line)

                is_fatal = True
                total += 1
                fatal += 1
                continue

            if is_fatal:
                if comm and comm not in line:
                    continue

                sys.stderr.write(line)

                if fatal_end.search(line):
                    print(file=sys.stderr)
                    is_fatal = False
                    comm = None
                continue

            if standard_begin.search(line):
                m = find_comm.search(line)
                if m:
                    if exclude_regex and exclude_regex.search(m.group(1)):
                        continue
                    comm = m.group(1)

                sys.stderr.write(line)

                is_standard = True
                total += 1
                continue

            if is_standard:
                if comm and comm not in line:
                    continue

                sys.stderr.write(line)

                kind = standard_end.search(line)
                if kind:
                    print(file=sys.stderr)
                    is_standard = False
                    comm = None

                    t = kind.group(1)
                    if t == 'Address':
                        asan += 1
                    elif t == 'UndefinedBehavior':
                        ubsan += 1
                    elif t == 'Memory':
                        msan += 1

    if total > 0:
        print(
            f'Found {total} sanitizer issues ({fatal} internal, {asan} asan, {ubsan} ubsan, {msan} msan).',
            file=sys.stderr,
        )
    else:
        print('No sanitizer issues found.', file=sys.stderr)

    return total > 0


def process_coverage(args: argparse.Namespace, summary: Summary, name: str, journal_file: Path) -> None:
    coverage = subprocess.run(
        [
            'journalctl',
            '--file', journal_file,
            '--field=COVERAGE_TAR',
        ],
        stdout=subprocess.PIPE,
        text=True,
        check=True,
    ).stdout  # fmt: skip

    (args.meson_build_dir / 'test/coverage').mkdir(exist_ok=True)

    initial = args.meson_build_dir / 'test/coverage/initial.coverage-info'
    output = args.meson_build_dir / f'test/coverage/{name}.coverage-info'

    for b64 in coverage.splitlines():
        tarball = base64.b64decode(b64)

        with tempfile.TemporaryDirectory(prefix='coverage-') as tmp:
            subprocess.run(
                [
                    'tar',
                    '--extract',
                    '--file', '-',
                    '--directory', tmp,
                    '--keep-directory-symlink',
                    '--no-overwrite-dir',
                    '--zstd',
                ],
                input=tarball,
                check=True,
            )  # fmt: skip

            for p in Path(tmp).iterdir():
                if not p.name.startswith('#'):
                    continue

                dst = Path(tmp) / p.name.replace('#', '/').lstrip('/')
                dst.parent.mkdir(parents=True, exist_ok=True)
                p.rename(dst)

            subprocess.run(
                [
                    'find',
                    tmp,
                    '-name', '*.gcda',
                    '-size', '0',
                    '-delete',
                ],
                input=tarball,
                check=True,
            )  # fmt: skip

            subprocess.run(
                [
                    'rsync',
                    '--archive',
                    '--prune-empty-dirs',
                    '--include=*/',
                    '--include=*.gcno',
                    '--exclude=*',
                    f'{os.fspath(summary.builddir / summary.buildsubdir)}/',
                    os.fspath(Path(tmp) / 'work/build'),
                ],
                check=True,
            )

            subprocess.run(
                [
                    'lcov',
                    *(
                        [
                            '--gcov-tool', 'llvm-cov',
                            '--gcov-tool', 'gcov',
                        ]
                        if summary.environment.get('LLVM', '0') == '1'
                        else []
                    ),
                    '--directory', tmp,
                    '--base-directory', 'src/',
                    '--capture',
                    '--exclude', '*.gperf',
                    '--output-file', f'{output}.new',
                    '--ignore-errors', 'inconsistent,inconsistent,source,negative',
                    '--substitute', 's#src/src#src#g',
                    '--no-external',
                    '--quiet',
                ],
                check=True,
                cwd=os.fspath(args.mkosi_dir),
            )  # fmt: skip

            subprocess.run(
                [
                    'lcov',
                    '--ignore-errors', 'inconsistent,inconsistent,format,corrupt,empty',
                    '--add-tracefile', output if output.exists() else initial,
                    '--add-tracefile', f'{output}.new',
                    '--output-file', output,
                    '--quiet',
                ],
                check=True,
                cwd=os.fspath(args.mkosi_dir),
            )  # fmt: skip

            Path(f'{output}.new').unlink()

            print(f'Wrote coverage report for {name} to {output}', file=sys.stderr)


def in_userns() -> bool:
    # The initial user namespace has a well-known inode number (USER_NS_INIT_INO, linux/nsfs.h).
    # Without kernel support for user namespaces, ours can only be the initial one.
    try:
        return os.stat('/proc/self/ns/user').st_ino != 0xEFFFFFFD
    except OSError:
        return False


def coco_host_type() -> Optional[str]:
    try:
        if Path('/sys/module/kvm_intel/parameters/tdx').read_text().strip() == 'Y':
            return 'tdx'
    except OSError:
        pass
    try:
        if (
            Path('/sys/module/kvm_amd/parameters/sev_snp').read_text().strip() == 'Y'
            and Path('/dev/sev').exists()
        ):
            return 'sev-snp'
    except OSError:
        pass
    return None


def coco_nspawn_args(coco_type: str) -> list[str]:
    # systemd-nspawn args (appended after '--' in boot mode) that expose the devices vmspawn needs to
    # launch a confidential guest from inside the boot-mode container.
    binds = ['--bind=/dev/kvm', '--bind=/dev/vhost-vsock']
    if coco_type == 'sev-snp':
        binds += ['--bind=/dev/sev']
    # TDX guests obtain quotes via the host's Quote Generation Service; forward its socket if present.
    if coco_type == 'tdx' and Path('/run/tdx-qgs/qgs.socket').exists():
        binds += ['--bind=/run/tdx-qgs/qgs.socket']
    return binds


def statfs(path: Path) -> str:
    return subprocess.run(
        ['stat', '--file-system', os.fspath(path), '--format=%T'],
        stdout=subprocess.PIPE,
        text=True,
        check=True,
    ).stdout.strip()


INTERRUPTED = False


def onsignal(signal: int, frame: Optional[FrameType]) -> None:
    global INTERRUPTED
    if INTERRUPTED:
        return

    INTERRUPTED = True
    raise KeyboardInterrupt()


def main() -> None:
    signal.signal(signal.SIGINT, onsignal)
    signal.signal(signal.SIGTERM, onsignal)
    signal.signal(signal.SIGHUP, onsignal)

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--mkosi', default=None)
    parser.add_argument('--meson-source-dir', required=True, type=Path)
    parser.add_argument('--meson-build-dir', required=True, type=Path)
    parser.add_argument('--name', required=True)
    parser.add_argument('--unit', required=True)
    parser.add_argument('--storage', required=True)
    parser.add_argument('--firmware', required=True)
    parser.add_argument('--slow', action=argparse.BooleanOptionalAction)
    parser.add_argument('--vm', action=argparse.BooleanOptionalAction)
    parser.add_argument('--exit-code', required=True, type=int)
    parser.add_argument('--coredump-exclude-regex', required=True)
    parser.add_argument('--sanitizer-exclude-regex', required=True)
    parser.add_argument('--rtc', action=argparse.BooleanOptionalAction)
    parser.add_argument('--tpm', action=argparse.BooleanOptionalAction)
    parser.add_argument('--vm-images', action=argparse.BooleanOptionalAction)
    parser.add_argument('--coco', default=None, choices=('tdx', 'sev-snp', 'any'))
    parser.add_argument('--skip', action=argparse.BooleanOptionalAction)
    parser.add_argument('--suppress-sync', action=argparse.BooleanOptionalAction, default=False)
    parser.add_argument('mkosi_args', nargs='*')
    args = parser.parse_args()

    if not args.mkosi:
        args.mkosi = shutil.which('mkosi')
        if not args.mkosi:
            print('Could not find mkosi which is required to run the integration tests', file=sys.stderr)
            sys.exit(1)

    # The meson source directory can either be the top-level repository directory or the
    # test/integration-tests/standalone subdirectory in the repository directory. The mkosi configuration
    # will always be a parent directory of one of these directories and at most 4 levels upwards, so don't
    # look further than that.
    dirs = [args.meson_source_dir] + list(args.meson_source_dir.parents)
    for p in dirs[: min(len(dirs), 4)]:
        if (p / 'mkosi/mkosi.conf').exists():
            setattr(args, 'mkosi_dir', p)
            break
    else:
        print(
            f'Directory with mkosi config not found in any parent directories of {args.meson_source_dir}',
            file=sys.stderr,
        )
        exit(1)

    if args.slow and not bool(int(os.getenv('SYSTEMD_SLOW_TESTS', '0'))):
        print(
            f'SYSTEMD_SLOW_TESTS=1 not found in environment, skipping {args.name}',
            file=sys.stderr,
        )
        exit(77)

    if args.vm and bool(int(os.getenv('TEST_NO_QEMU', '0'))):
        print(f'TEST_NO_QEMU=1, skipping {args.name}', file=sys.stderr)
        exit(77)

    if args.skip:
        print(f'meson requirements for test {args.name} were not fulfilled, skipping', file=sys.stderr)
        exit(77)

    for s in os.getenv('TEST_SKIP', '').split():
        if s in args.name:
            print(f'Skipping {args.name} due to TEST_SKIP', file=sys.stderr)
            exit(77)

    keep_journal = os.getenv('TEST_SAVE_JOURNAL', 'fail')
    shell = bool(int(os.getenv('TEST_SHELL', '0')))
    summary = Summary.get(args)

    # Keep list in sync with TEST-06-SELINUX.sh
    if args.name == 'TEST-06-SELINUX' and summary.distribution not in ('centos', 'fedora', 'opensuse'):
        print('Skipping TEST-06-SELINUX, only enabled for CentOS/Fedora/openSUSE', file=sys.stderr)
        exit(77)

    if shell and not sys.stdin.isatty():
        print(
            '--interactive must be passed to meson test to use TEST_SHELL=1',
            file=sys.stderr,
        )
        exit(1)

    name = args.name + (f'-{i}' if (i := os.getenv('MESON_TEST_ITERATION')) else '')

    dropin = textwrap.dedent(
        '''\
        [Service]
        StandardOutput=journal+console
        '''
    )

    if not shell:
        dropin += textwrap.dedent(
            '''
            [Unit]
            SuccessAction=exit
            SuccessActionExitStatus=123
            '''
        )

    if os.getenv('TEST_MATCH_SUBTEST'):
        dropin += textwrap.dedent(
            f'''
            [Service]
            Environment=TEST_MATCH_SUBTEST={os.environ['TEST_MATCH_SUBTEST']}
            '''
        )

    if os.getenv('TEST_MATCH_TESTCASE'):
        dropin += textwrap.dedent(
            f'''
            [Service]
            Environment=TEST_MATCH_TESTCASE={os.environ['TEST_MATCH_TESTCASE']}
            '''
        )

    if os.getenv('TEST_RUN_DFUZZER'):
        dropin += textwrap.dedent(
            f'''
            [Service]
            Environment=TEST_RUN_DFUZZER={os.environ['TEST_RUN_DFUZZER']}
            '''
        )

    if os.getenv('TEST_JOURNAL_USE_TMP', '0') == '1':
        if statfs(Path('/tmp')) != 'tmpfs' and statfs(Path('/dev/shm')) == 'tmpfs':
            tmp = Path('/dev/shm')
        else:
            tmp = Path('/tmp')

        journal_file = tmp / f'systemd-integration-tests/journal/{name}.journal'
    else:
        journal_file = (args.meson_build_dir / f'test/journal/{name}.journal').absolute()

    journal_file.unlink(missing_ok=True)

    if not sys.stdin.isatty():
        dropin += textwrap.dedent(
            '''
            [Unit]
            FailureAction=exit
            '''
        )
    elif not shell:
        dropin += textwrap.dedent(
            '''
            [Unit]
            Wants=multi-user.target getty-pre.target
            Before=getty-pre.target

            [Service]
            StandardInput=tty
            StandardOutput=inherit
            StandardError=inherit
            TTYReset=yes
            TTYVHangup=yes
            IgnoreSIGPIPE=no
            # bash ignores SIGTERM
            KillSignal=SIGHUP
            '''
        )

    if sys.stdin.isatty():
        dropin += textwrap.dedent(
            '''
            [Service]
            ExecStartPre=/usr/lib/systemd/tests/testdata/integration-test-setup.sh setup
            ExecStopPost=/usr/lib/systemd/tests/testdata/integration-test-setup.sh finalize
            StateDirectory=%N
            '''
        )

    if args.rtc:
        if sys.version_info >= (3, 12):
            now = datetime.datetime.now(datetime.UTC)
        else:
            now = datetime.datetime.utcnow()

        rtc = datetime.datetime.strftime(now, r'%Y-%m-%dT%H:%M:%S')
    else:
        rtc = None

    # Whenever possible, boot without an initrd. This requires the target distribution kernel to have the
    # necessary modules (virtio-blk, ext4) builtin.
    if args.firmware == 'linux-noinitrd' and (summary.distribution, summary.release) not in (
        ('fedora', 'rawhide'),
        ('arch', 'rolling'),
    ):
        firmware = 'linux'
    else:
        firmware = args.firmware

    vm = args.vm or os.getuid() != 0 or os.getenv('TEST_PREFER_QEMU', '0') == '1'

    # Tests that launch nested VMs need the mkosi-built images, which are build outputs rather than
    # installed test artifacts. Bind the output directory read-only into the boot-mode container at
    # /work/vm-images on request, instead of mounting the whole build tree via RuntimeBuildSources.
    vm_images_args: list[str] = []
    if args.vm_images and not vm:
        output_dir = args.meson_build_dir / 'mkosi.output'
        if output_dir.exists():
            vm_images_args = [f'--bind-ro={os.fspath(output_dir)}:/work/vm-images']
        else:
            print(
                f'--vm-images requested but {output_dir} does not exist (images not built?); '
                f'/work/vm-images will be unavailable and tests depending on it will skip',
                file=sys.stderr,
            )

    # Coco needs to launch a real (L1) confidential guest, which requires the boot-mode nspawn on
    # a coco-capable bare-metal host. Nested QEMU VM (vm mode) won't work.
    coco_boot_args: list[str] = []
    if args.coco:
        if vm:
            print(
                f'{args.name} needs bare-metal boot mode (root, no nesting) for confidential VMs, skipping',
                file=sys.stderr,
            )
            exit(77)
        host = coco_host_type()
        if host is None or (args.coco != 'any' and host != args.coco):
            print(
                f'Host does not provide coco={args.coco} (detected: {host}), skipping {args.name}',
                file=sys.stderr,
            )
            exit(77)
        if not Path('/dev/vhost-vsock').exists():
            print(
                f'{args.name} needs /dev/vhost-vsock (load the vhost_vsock module) for the guest '
                f'notify channel, skipping',
                file=sys.stderr,
            )
            exit(77)
        coco_boot_args = coco_nspawn_args(host)
        # /sys/module/*/parameters isn't reliably readable inside the nspawn container, so hand the
        # resolved coco type to the test via the environment instead of making it re-detect.
        dropin += textwrap.dedent(
            f'''
            [Service]
            Environment=COCO_TYPE={host}
            '''
        )

    # Extra mkosi options for this invocation only. Each has to use the '--option=value' syntax.
    test_mkosi_args = shlex.split(os.getenv('TEST_MKOSI_ARGS', ''))
    for arg in test_mkosi_args:
        if not re.fullmatch('--[^=]+=.*', arg):
            sys.exit(f"TEST_MKOSI_ARGS must contain only '--option=value' mkosi options, got: {arg}")

    cmd = [
        args.mkosi,
        '--directory', os.fspath(args.mkosi_dir),
        '--machine', name,
        '--ephemeral=yes',
        *(['--forward-journal', journal_file] if journal_file else []),
        *(
            [
                '--credential', f'systemd.extra-unit.emergency-exit.service={shlex.quote(EMERGENCY_EXIT_SERVICE)}',
                '--credential', f'systemd.unit-dropin.emergency.target={shlex.quote(EMERGENCY_EXIT_DROPIN)}',
            ]
            if not sys.stdin.isatty()
            else []
        ),
        '--credential', f'systemd.unit-dropin.{args.unit}={shlex.quote(dropin)}',
        '--runtime-network=none',
        *([f'--qemu-args=-rtc base={rtc}'] if rtc else []),
        *args.mkosi_args,
        '--firmware', firmware,
        *(['--kvm', 'no'] if int(os.getenv('TEST_NO_KVM', '0')) else []),
        '--tpm', 'yes' if args.tpm else 'no',
        '--kernel-command-line-extra',
        ' '.join(
            [
                'systemd.hostname=H',
                f'SYSTEMD_UNIT_PATH=/usr/lib/systemd/tests/testdata/{args.name}.units:/usr/lib/systemd/tests/testdata/units:',
                *([f'systemd.unit={args.unit}'] if not shell else []),
                'systemd.mask=systemd-networkd-wait-online.service',
                *(
                    [
                        'systemd.mask=serial-getty@.service',
                        'systemd.show_status=error',
                        'systemd.crash_shell=0',
                        'systemd.crash_action=poweroff',
                        'loglevel=6',
                    ]
                    if not sys.stdin.isatty()
                    else []
                ),
            ]
        ),
        '--credential', f"journal.storage={'persistent' if sys.stdin.isatty() else args.storage}",
        *(['--runtime-build-sources=no', '--register=no'] if not sys.stdin.isatty() else []),
        *test_mkosi_args,
        'vm' if vm else 'boot',
        *(
            [
                '--',
                '--capability=CAP_BPF',
                f'--suppress-sync={"yes" if args.suppress_sync else "no"}',
                # In a non-initial user namespace the kernel only allows mounting a fresh proc or
                # sysfs if the mount namespace already contains a fully visible mount of the same
                # filesystem type (mount_too_revealing()), which nspawn's pivoted root doesn't.
                # Satisfy the check with read-only binds, but only when actually confined, as
                # they expose the outer proc and sysfs to the container. Bind non-recursively:
                # carried-along submounts (e.g. lxcfs overmounts of container runtimes) would
                # spoil full visibility wherever they end up locked, and --bind-ro wouldn't make
                # them read-only either.
                *(
                    ['--bind-ro=/proc:/work/proc:norbind', '--bind-ro=/sys:/work/sys:norbind']
                    if in_userns()
                    else []
                ),
                *vm_images_args,
                *coco_boot_args,
            ]
            if not vm
            else []
        ),
    ]  # fmt: skip

    try:
        result = subprocess.run(cmd)

        # On Debian/Ubuntu we get a lot of random QEMU crashes. Retry once, and then skip if it fails again.
        if args.vm and result.returncode == 247 and args.exit_code != 247:
            if journal_file:
                journal_file.unlink(missing_ok=True)
            print(
                f'Test {args.name} failed due to QEMU crash (error 247), retrying...',
                file=sys.stderr,
            )
            result = subprocess.run(cmd)
            if args.vm and result.returncode == 247 and args.exit_code != 247:
                print(
                    f'Test {args.name} failed due to QEMU crash (error 247), ignoring',
                    file=sys.stderr,
                )
                exit(77)
            print(
                f'Test {args.name} worked on re-run after QEMU crash (error 247)',
                file=sys.stderr,
            )
    except KeyboardInterrupt:
        result = subprocess.CompletedProcess(args=cmd, returncode=-signal.SIGINT)

    coredumps = process_coredumps(args, journal_file)

    sanitizer = False
    if summary.environment.get('SANITIZERS'):
        sanitizer = process_sanitizer_report(args, journal_file)

    if (
        summary.environment.get('COVERAGE', '0') == '1'
        and result.returncode in (args.exit_code, 77)
        and not coredumps
        and not sanitizer
    ):
        process_coverage(args, summary, name, journal_file)

    if keep_journal == '0' or (
        keep_journal == 'fail'
        and result.returncode in (args.exit_code, 77)
        and not coredumps
        and not sanitizer
    ):
        journal_file.unlink(missing_ok=True)
    elif os.getenv('TEST_JOURNAL_USE_TMP', '0') == '1' and journal_file.exists():
        dst = args.meson_build_dir / f'test/journal/{name}.journal'
        dst.parent.mkdir(parents=True, exist_ok=True)
        journal_file = Path(shutil.move(journal_file, dst))

    if shell or (result.returncode in (args.exit_code, 77) and not coredumps and not sanitizer):
        exit_code = 0 if shell or result.returncode == args.exit_code else 77
        exit_str = 'succeeded' if exit_code == 0 else 'skipped'
    else:
        # 0 also means we failed so translate that to a non-zero exit code to mark the test as failed.
        exit_code = result.returncode or 1
        exit_str = 'failed'

    if journal_file.exists():
        ops = []

        if os.getenv('GITHUB_ACTIONS'):
            id = os.environ['GITHUB_RUN_ID']
            wf = os.environ['GITHUB_WORKFLOW']
            iter = os.environ['GITHUB_RUN_ATTEMPT']
            runner = os.environ['TEST_RUNNER']
            artifact = (
                f'ci-{wf}-{id}-{iter}-{summary.distribution}-{summary.release}-{runner}-failed-test-journals'
            )
            ops += [f'gh run download {id} --name {artifact} -D ci/{artifact}']
            journal_file = Path(f'ci/{artifact}/test/journal/{name}.journal')

        ops += [f'journalctl --file {journal_file} --no-hostname -o short-monotonic -u {args.unit} -p info']

        print(
            f'Test {exit_str}, relevant logs can be viewed with: \n\n{(" && ".join(ops))}\n', file=sys.stderr
        )

    exit(exit_code)


if __name__ == '__main__':
    main()