mkosi: Use build image prepare scripts for tools tree as well (#36861)

Instead of listing dependencies manually for the default tools tree, let's reuse the prepare scripts from the build image. To make this work, the sync script has to be configured for the tools tree as well so that it's invoked both when building the tools tree and for the regular image, otherwise, when doing the first build in a fresh checkout, the sync script won't have executed yet as sync scripts for the regular images are executed after building the default tools tree.
2025-03-28 13:31:28 +00:00
parent 848b5ae0d1 1b49fb9aaa
commit 6561d02ee1
35 changed files with 141 additions and 111 deletions
@@ -25,7 +25,7 @@ jobs:

    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      - uses: systemd/mkosi@6e64559161c1a5ed88ad1eacdd77f9a45cbbf6a6
+      - uses: systemd/mkosi@03d1a1b4cdc474c55166a2f48be244748011206d

      # Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space
      # immediately, we remove the files in the background. However, we first move them to a different location
@@ -120,7 +120,7 @@ jobs:

    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      - uses: systemd/mkosi@6e64559161c1a5ed88ad1eacdd77f9a45cbbf6a6
+      - uses: systemd/mkosi@03d1a1b4cdc474c55166a2f48be244748011206d

      # Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space
      # immediately, we remove the files in the background. However, we first move them to a different location
@@ -191,10 +191,14 @@ jobs:

      - name: Configure meson
        run: |
+          # /usr/sbin/bpftool is completely broken inside containers on Ubuntu which makes meson blow up so
+          # disable the bpf-framework stuff to avoid the issue.
+          # TODO: Drop when we move off Ubuntu Noble as this will be fixed in the next Ubuntu LTS release.
          sudo mkosi sandbox -- \
            meson setup \
            --buildtype=debugoptimized \
            -Dintegration-tests=true \
+            -Dbpf-framework=disabled \
            build

      - name: Build image
@@ -19,7 +19,7 @@ actions:
  post-upstream-clone:
    # Use the Fedora Rawhide specfile
    - git clone https://src.fedoraproject.org/rpms/systemd .packit_rpm
-    - bash -c 'git -C .packit_rpm checkout "$(grep GIT_COMMIT= mkosi.images/build/mkosi.conf.d/centos-fedora/mkosi.conf | cut -d= -f2)"'
+    - bash -c 'git -C .packit_rpm checkout "$(grep GIT_COMMIT= mkosi.conf.d/10-centos-fedora/mkosi.conf | cut -d= -f2)"'
    - bash -c 'echo "%bcond upstream 1" > .packit_rpm/systemd.spec.new'
    - bash -c 'echo "%define meson_extra_configure_options --werror" >> .packit_rpm/systemd.spec.new'
    - bash -c 'cat .packit_rpm/systemd.spec >> .packit_rpm/systemd.spec.new'
@@ -39,7 +39,7 @@ chance that your distribution's packaged version of mkosi will be too old.
 Then, you can build, run and test systemd executables as follows:

 ```sh
-$ mkosi -f sandbox -- meson setup build
+$ mkosi -f sandbox -- meson setup -Dbpf-framework=disabled build # bpftool detection inside mkosi sandbox is broken on Ubuntu Noble and older
 $ mkosi -f sandbox -- meson compile -C build
 $ mkosi -f sandbox -- build/systemctl --version
 $ mkosi -f sandbox -- meson test -C build          # Run the unit tests
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later

 [Config]
-MinimumVersion=25
+MinimumVersion=25~devel
 Dependencies=
        exitrd
        initrd
@@ -11,7 +11,6 @@ Dependencies=

 PassEnvironment=
        NO_BUILD
-        NO_SYNC
        WIPE
        SANITIZERS
        CFLAGS
@@ -24,6 +23,11 @@ PassEnvironment=
        ASAN_OPTIONS
        COVERAGE
        VCS_TAG
+        GIT_URL
+        GIT_SUBDIR
+        GIT_BRANCH
+        GIT_COMMIT
+        PKG_SUBDIR

 [Output]
 RepartDirectories=mkosi.repart
@@ -1,11 +1,8 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later

 [Build]
+ToolsTreeSyncScripts=%D/mkosi.sync
 ToolsTreePackages=
-        gcc
        gdb
-        gperf
        llvm
-        meson
-        pkgconf
-        rsync
+        meson # Also needed in the NO_BUILD case so we list it explicitly.
@@ -4,19 +4,11 @@
 ToolsTreeDistribution=arch

 [Build]
+ToolsTreePrepareScripts=%D/mkosi.images/build/mkosi.conf.d/arch/mkosi.prepare
 ToolsTreePackages=
-        cryptsetup
+        base-devel
        github-cli
        lcov
-        libcap
-        libmicrohttpd
-        libxslt
        mypy
-        perl-json-xs
-        python-jinja
-        python-lxml
-        python-pytest
        ruff
        shellcheck
-        tpm2-tss
-        util-linux-libs
@@ -5,18 +5,7 @@ ToolsTreeDistribution=|fedora
 ToolsTreeDistribution=|centos

 [Build]
+ToolsTreePrepareScripts=%D/mkosi.images/build/mkosi.conf.d/centos-fedora/mkosi.prepare
 ToolsTreePackages=
-        pkgconfig(blkid)
-        pkgconfig(libcap)
-        pkgconfig(libcryptsetup)
-        pkgconfig(libcurl)
-        pkgconfig(fdisk)
-        pkgconfig(libmicrohttpd)
-        pkgconfig(mount)
-        tpm2-tss-devel
-        python3-jinja2
-        python3-lxml
        python3-mypy
-        python3-pytest
-        libxslt
-        docbook-style-xsl
+        rpm-build
@@ -5,20 +5,9 @@ ToolsTreeDistribution=|debian
 ToolsTreeDistribution=|ubuntu

 [Build]
+ToolsTreePrepareScripts=%D/mkosi.images/build/mkosi.conf.d/debian-ubuntu/mkosi.prepare
 ToolsTreePackages=
        gh
        lcov
-        libblkid-dev
-        libcap-dev
-        libcryptsetup-dev
-        libcurl4-openssl-dev
-        libfdisk-dev
-        libmicrohttpd-dev
-        libmount-dev
-        libtss2-dev
        mypy
-        python3-jinja2
-        python3-lxml
-        python3-pytest
        shellcheck
-        xsltproc
@@ -4,23 +4,11 @@
 ToolsTreeDistribution=opensuse

 [Build]
+ToolsTreePrepareScripts=%D/mkosi.images/build/mkosi.conf.d/opensuse/mkosi.prepare
 ToolsTreePackages=
-        lcov
-        libz1
        gh
+        lcov
        mypy
-        pkgconfig(blkid)
-        pkgconfig(libcap)
-        pkgconfig(libcryptsetup)
-        pkgconfig(libcurl)
-        pkgconfig(fdisk)
-        pkgconfig(libmicrohttpd)
-        pkgconfig(mount)
-        pkgconfig(libopenssl)
        python3-ruff
-        tpm2-0-tss-devel
-        python3-jinja2
-        python3-lxml
-        python3-pytest
-        xsltproc
+        rpm-build
        ShellCheck
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+[Match]
+ToolsTreeDistribution=ubuntu
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+# The ports Ubuntu archive is for non i386/amd64 repositories
+
+[Match]
+Architecture=!x86-64
+Architecture=!x86
+ToolsTreeRelease=noble
+
+[Build]
+ToolsTreeSandboxTrees=%D/mkosi.conf.d/10-ubuntu/noble-backports-ports.sources:/etc/apt/sources.list.d/noble-backports-ports.sources
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+# The main Ubuntu archive is only for i386/amd64 repositories
+
+[Match]
+Architecture=|x86-64
+Architecture=|x86
+ToolsTreeRelease=noble
+
+[Build]
+ToolsTreeSandboxTrees=%D/mkosi.conf.d/10-ubuntu/noble-backports.sources:/etc/apt/sources.list.d/noble-backports.sources
@@ -3,6 +3,13 @@
 [Match]
 Distribution=arch

+[Build]
+Environment=
+        GIT_URL=https://gitlab.archlinux.org/archlinux/packaging/packages/systemd.git
+        GIT_BRANCH=main
+        GIT_COMMIT=1b4994657df1c7495607f0c2f20db73550b94f34
+        PKG_SUBDIR=arch
+
 [Content]
 VolatilePackages=
        systemd
@@ -4,6 +4,13 @@
 Distribution=|centos
 Distribution=|fedora

+[Build]
+Environment=
+        GIT_URL=https://src.fedoraproject.org/rpms/systemd.git
+        GIT_BRANCH=rawhide
+        GIT_COMMIT=617952132de4ed882b9b00ceeed077f843130f91
+        PKG_SUBDIR=fedora
+
 [Content]
 VolatilePackages=
        systemd
@@ -1,3 +0,0 @@
-Package: *
-Pin: release l=mkosi
-Pin-Priority: 1100
@@ -5,7 +5,12 @@ Distribution=|debian
 Distribution=|ubuntu

 [Build]
-SandboxTrees=mkosi-pinning.pref:/etc/apt/preferences.d/mkosi-pinning.pref
+Environment=
+        GIT_URL=https://salsa.debian.org/systemd-team/systemd.git
+        GIT_SUBDIR=debian
+        GIT_BRANCH=debian/master
+        GIT_COMMIT=d8c7f8f7f461b1edc3bf5040509e697ea574c990
+        PKG_SUBDIR=debian

 [Content]
 VolatilePackages=
@@ -9,6 +9,12 @@ Repositories=non-oss

 [Build]
 SandboxTrees=macros.db_backend:/etc/rpm/macros.db_backend
+Environment=
+        GIT_URL=https://github.com/bmwiedemann/openSUSE
+        GIT_SUBDIR=packages/s/systemd
+        GIT_BRANCH=master
+        GIT_COMMIT=9002a773f0ab185908dc8fb6c831c1cd9fdf11f0
+        PKG_SUBDIR=opensuse

 [Content]
 VolatilePackages=
@@ -40,6 +40,15 @@ disable iscsi.service
 disable iscsid.socket
 disable iscsiuio.socket

+# Enabled by default on Fedora, but don't make sense for our use case.
+disable lvm2-lvmpolld.socket
+disable lvm2-monitor.service
+disable dracut-shutdown.service
+disable dm-event.socket
+disable fstrim.timer
+disable raid-check.timer
+disable systemd-tmpfiles-clean.timer
+
 # mkosi relabels the image itself so no need to do it on boot.
 disable selinux-autorelabel-mark.service

@@ -0,0 +1,5 @@
+#!/bin/bash
+# SPDX-License-Identifier: CC-0
+set -e
+
+touch -r "$BUILDROOT/usr" "$BUILDROOT/etc/.updated" "$BUILDROOT/var/.updated"
@@ -3,13 +3,6 @@
 [Match]
 Distribution=arch

-[Build]
-Environment=
-        GIT_URL=https://gitlab.archlinux.org/archlinux/packaging/packages/systemd.git
-        GIT_BRANCH=main
-        GIT_COMMIT=1b4994657df1c7495607f0c2f20db73550b94f34
-        PKG_SUBDIR=arch
-
 [Content]
 Packages=
        base
@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 set -e

-if [[ "$1" == "build" ]]; then
+if [[ "$1" == "build" ]] || ((NO_BUILD)); then
    exit 0
 fi

@@ -4,13 +4,6 @@
 Distribution=|centos
 Distribution=|fedora

-[Build]
-Environment=
-        GIT_URL=https://src.fedoraproject.org/rpms/systemd.git
-        GIT_BRANCH=rawhide
-        GIT_COMMIT=617952132de4ed882b9b00ceeed077f843130f91
-        PKG_SUBDIR=fedora
-
 [Content]
 Packages=
        clang-devel
@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 set -e

-if [[ "$1" == "build" ]]; then
+if [[ "$1" == "build" ]] || ((NO_BUILD)); then
    exit 0
 fi

@@ -4,14 +4,6 @@
 Distribution=|debian
 Distribution=|ubuntu

-[Build]
-Environment=
-        GIT_URL=https://salsa.debian.org/systemd-team/systemd.git
-        GIT_SUBDIR=debian
-        GIT_BRANCH=debian/master
-        GIT_COMMIT=d8c7f8f7f461b1edc3bf5040509e697ea574c990
-        PKG_SUBDIR=debian
-
 [Content]
 Packages=
        apt
@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 set -e

-if [[ "$1" == "build" ]]; then
+if [[ "$1" == "build" ]] || ((NO_BUILD)); then
    exit 0
 fi

@@ -3,14 +3,6 @@
 [Match]
 Distribution=opensuse

-[Build]
-Environment=
-        GIT_URL=https://github.com/bmwiedemann/openSUSE
-        GIT_SUBDIR=packages/s/systemd
-        GIT_BRANCH=master
-        GIT_COMMIT=9002a773f0ab185908dc8fb6c831c1cd9fdf11f0
-        PKG_SUBDIR=opensuse
-
 [Content]
 Packages=
        clang
@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 set -e

-if [[ "$1" == "build" ]]; then
+if [[ "$1" == "build" ]] || ((NO_BUILD)); then
    exit 0
 fi

@@ -1,5 +1,8 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later

+id = find_program('id', required : true)
+uid = run_command(id, '-u', check : true).stdout().strip().to_int()
+
 integration_tests += [
        integration_test_template + {
                'name' : fs.name(meson.current_source_dir()),
@@ -11,5 +14,6 @@ integration_tests += [
                # Make sure we don't mount anything with virtiofs as otherwise fixfiles will try to relabel
                # it.
                'mkosi-args' : integration_test_template['mkosi-args'] + ['--runtime-build-sources=no'],
+                'enabled' : uid == 0,
        },
 ]
@@ -8,5 +8,6 @@ integration_tests += [
                ],
                'exit-code' : 124,
                'vm' : true,
+                'firmware' : 'linux',
        },
 ]
@@ -22,5 +22,6 @@ integration_tests += [
                        '--runtime-size=11G',
                ],
                'vm' : true,
+                'firmware' : 'linux',
        },
 ]
@@ -350,6 +350,15 @@ def process_coverage(args: argparse.Namespace, summary: Summary, name: str, jour
            print(f'Wrote coverage report for {name} to {output}', file=sys.stderr)


+def statfs(path: Path) -> str:
+    return subprocess.run(
+        ['stat', '--file-system', os.fspath(path), '--format=%T'],
+        stdout=subprocess.PIPE,
+        text=True,
+        check=True,
+    ).stdout.strip()
+
+
 def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--mkosi', required=True)
@@ -365,6 +374,7 @@ def main() -> None:
    parser.add_argument('--coredump-exclude-regex', required=True)
    parser.add_argument('--sanitizer-exclude-regex', required=True)
    parser.add_argument('--rtc', action=argparse.BooleanOptionalAction)
+    parser.add_argument('--skip', action=argparse.BooleanOptionalAction)
    parser.add_argument('mkosi_args', nargs='*')
    args = parser.parse_args()

@@ -386,6 +396,10 @@ def main() -> None:
        print(f'TEST_NO_QEMU=1, skipping {args.name}', file=sys.stderr)
        exit(77)

+    if args.skip:
+        print(f'meson requirements for test {args.name} were not fulfilled, skipping', file=sys.stderr)
+        exit(77)
+
    for s in os.getenv('TEST_SKIP', '').split():
        if s in args.name:
            print(f'Skipping {args.name} due to TEST_SKIP', file=sys.stderr)
@@ -437,7 +451,12 @@ def main() -> None:
        )

    if os.getenv('TEST_JOURNAL_USE_TMP', '0') == '1':
-        journal_file = Path(f'/tmp/systemd-integration-tests/journal/{name}.journal')
+        if statfs(Path('/tmp')) != 'tmpfs' and statfs(Path('/dev/shm')) == 'tmpfs':
+            tmp = Path('/dev/shm')
+        else:
+            tmp = Path('/tmp')
+
+        journal_file = tmp / f'systemd-integration-tests/journal/{name}.journal'
    else:
        journal_file = (args.meson_build_dir / f'test/journal/{name}.journal').absolute()

@@ -489,6 +508,22 @@ def main() -> None:
    else:
        rtc = None

+    # mkosi will use the UEFI secure boot firmware by default on UEFI platforms. However, this breaks on
+    # Github Actions in combination with KVM because of a HyperV bug so make sure we use the non secure
+    # boot firmware on Github Actions.
+    # TODO: Drop after the HyperV bug that breaks secure boot KVM guests is solved
+    if args.firmware == 'auto' and os.getenv('GITHUB_ACTIONS'):
+        firmware = 'uefi'
+    # Whenever possible, boot without an initrd. This requires the target distribution kernel to have the
+    # necessary modules (virtio-blk, ext4) builtin.
+    elif args.firmware == 'linux-noinitrd' and (summary.distribution, summary.release) not in (
+        ('fedora', 'rawhide'),
+        ('arch', 'rolling'),
+    ):
+        firmware = 'linux'
+    else:
+        firmware = args.firmware
+
    cmd = [
        args.mkosi,
        '--directory', os.fspath(args.meson_source_dir),
@@ -508,11 +543,7 @@ def main() -> None:
        '--runtime-scratch=no',
        *([f'--qemu-args=-rtc base={rtc}'] if rtc else []),
        *args.mkosi_args,
-        # mkosi will use the UEFI secure boot firmware by default on UEFI platforms. However, this breaks on
-        # Github Actions in combination with KVM because of a HyperV bug so make sure we use the non secure
-        # boot firmware on Github Actions.
-        # TODO: Drop after the HyperV bug that breaks secure boot KVM guests is solved
-        '--firmware', 'uefi' if args.firmware == 'auto' and os.getenv("GITHUB_ACTIONS") else args.firmware,
+        '--firmware', firmware,
        *(['--kvm', 'no'] if int(os.getenv('TEST_NO_KVM', '0')) else []),
        '--kernel-command-line-extra',
        ' '.join(
@@ -7,7 +7,7 @@ integration_test_template = {
        'timeout' : 1800,
        'storage' : 'volatile',
        'priority' : 0,
-        'firmware' : 'linux',
+        'firmware' : 'linux-noinitrd',
        'enabled' : true,
        'configuration' : {
                'memory-accounting' : 'no',
@@ -125,6 +125,10 @@ foreach integration_test : integration_tests
                integration_test_args += ['--rtc']
        endif

+        if not integration_test['enabled']
+                integration_test_args += ['--skip']
+        endif
+
        if not mkosi.found()
                continue
        endif
@@ -153,10 +157,6 @@ foreach integration_test : integration_tests
                integration_test_env += {'SYSTEMD_INTEGRATION_TESTS': '1'}
        endif

-        if not integration_test['enabled']
-                continue
-        endif
-
        # We don't explicitly depend on the "mkosi" target because that means the image is rebuilt on every
        # "ninja -C build". Instead, the mkosi target has to be rebuilt manually before running the
        # integration tests with mkosi.
@@ -126,7 +126,7 @@ def update_distro(args, distro: str, config: dict):
    print(f"+ {shlex.join(cmd)}")
    changes = subprocess.check_output(cmd, text=True).strip()

-    conf_dir = Path('mkosi.images/build/mkosi.conf.d')
+    conf_dir = Path('mkosi.conf.d')
    files = conf_dir.glob('*/*.conf')
    for file in files:
        s = file.read_text()