From c008b703701e676a88e16de4c867b9fc4818070d Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 3 Jan 2025 23:25:23 +0100 Subject: [PATCH 1/3] mkosi: Update to latest --- .github/workflows/coverage.yml | 2 +- .github/workflows/mkosi.yml | 2 +- test/TEST-64-UDEV-STORAGE/btrfs_basic.configure | 2 +- test/TEST-64-UDEV-STORAGE/iscsi_lvm.configure | 2 +- test/TEST-64-UDEV-STORAGE/long_sysfs_path.configure | 2 +- test/TEST-64-UDEV-STORAGE/lvm_basic.configure | 2 +- test/TEST-64-UDEV-STORAGE/mdadm_basic.configure | 2 +- test/TEST-64-UDEV-STORAGE/mdadm_lvm.configure | 2 +- test/TEST-64-UDEV-STORAGE/multipath_basic_failover.configure | 2 +- test/TEST-64-UDEV-STORAGE/nvme_basic.configure | 2 +- test/TEST-64-UDEV-STORAGE/nvme_subsystem.configure | 2 +- test/TEST-64-UDEV-STORAGE/simultaneous_events.configure | 2 +- test/TEST-64-UDEV-STORAGE/virtio_scsi_basic.configure | 2 +- .../virtio_scsi_identically_named_partitions.configure | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 4d90641d1a..587a1c0c5b 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -24,7 +24,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - - uses: systemd/mkosi@ba07d53000b6c560ad0b9f07550aca93c0284e88 + - uses: systemd/mkosi@7051715d8541d4d077a218f7445976225ec74d78 # Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space # immediately, we remove the files in the background. However, we first move them to a different location diff --git a/.github/workflows/mkosi.yml b/.github/workflows/mkosi.yml index 4d128822fb..873ed589b6 100644 --- a/.github/workflows/mkosi.yml +++ b/.github/workflows/mkosi.yml @@ -113,7 +113,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - - uses: systemd/mkosi@ba07d53000b6c560ad0b9f07550aca93c0284e88 + - uses: systemd/mkosi@7051715d8541d4d077a218f7445976225ec74d78 # Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space # immediately, we remove the files in the background. However, we first move them to a different location diff --git a/test/TEST-64-UDEV-STORAGE/btrfs_basic.configure b/test/TEST-64-UDEV-STORAGE/btrfs_basic.configure index 8ef58a9fdd..85ec983fc3 100755 --- a/test/TEST-64-UDEV-STORAGE/btrfs_basic.configure +++ b/test/TEST-64-UDEV-STORAGE/btrfs_basic.configure @@ -11,7 +11,7 @@ config["QemuArgs"] += ["-device", "virtio-scsi-pci,id=scsi0"] for i in range(4): id = f"drivebtrfsbasic{i}" - config["QemuDrives"] += [ + config["Drives"] += [ { "Id": id, "Size": "350M" if i == 0 else "128M", diff --git a/test/TEST-64-UDEV-STORAGE/iscsi_lvm.configure b/test/TEST-64-UDEV-STORAGE/iscsi_lvm.configure index ca23e33858..1d7a0c5b9a 100755 --- a/test/TEST-64-UDEV-STORAGE/iscsi_lvm.configure +++ b/test/TEST-64-UDEV-STORAGE/iscsi_lvm.configure @@ -11,7 +11,7 @@ config["QemuArgs"] += ["-device", "virtio-scsi-pci,id=scsi0"] for i in range(4): id = f"driveiscsibasic{i}" - config["QemuDrives"] += [ + config["Drives"] += [ { "Id": id, "Size": "150M" if i == 0 else "70M", diff --git a/test/TEST-64-UDEV-STORAGE/long_sysfs_path.configure b/test/TEST-64-UDEV-STORAGE/long_sysfs_path.configure index 02e2ac4daa..a920171add 100755 --- a/test/TEST-64-UDEV-STORAGE/long_sysfs_path.configure +++ b/test/TEST-64-UDEV-STORAGE/long_sysfs_path.configure @@ -7,7 +7,7 @@ import sys config = json.load(sys.stdin) -config["QemuDrives"] += [ +config["Drives"] += [ { "Id": "drive0", "Size": "64M", diff --git a/test/TEST-64-UDEV-STORAGE/lvm_basic.configure b/test/TEST-64-UDEV-STORAGE/lvm_basic.configure index 9387b8377c..c4074becb4 100755 --- a/test/TEST-64-UDEV-STORAGE/lvm_basic.configure +++ b/test/TEST-64-UDEV-STORAGE/lvm_basic.configure @@ -11,7 +11,7 @@ config["QemuArgs"] += ["-device", "virtio-scsi-pci,id=scsi0"] for i in range(4): id = f"drivelvmbasic{i}" - config["QemuDrives"] += [ + config["Drives"] += [ { "Id": id, "Size": "32M", diff --git a/test/TEST-64-UDEV-STORAGE/mdadm_basic.configure b/test/TEST-64-UDEV-STORAGE/mdadm_basic.configure index 3f00afa955..d722e081e2 100755 --- a/test/TEST-64-UDEV-STORAGE/mdadm_basic.configure +++ b/test/TEST-64-UDEV-STORAGE/mdadm_basic.configure @@ -11,7 +11,7 @@ config["QemuArgs"] += ["-device", "virtio-scsi-pci,id=scsi0"] for i in range(5): id = f"drivemdadmbasic{i}" - config["QemuDrives"] += [ + config["Drives"] += [ { "Id": id, "Size": "64M", diff --git a/test/TEST-64-UDEV-STORAGE/mdadm_lvm.configure b/test/TEST-64-UDEV-STORAGE/mdadm_lvm.configure index b7661d9e81..d669895925 100755 --- a/test/TEST-64-UDEV-STORAGE/mdadm_lvm.configure +++ b/test/TEST-64-UDEV-STORAGE/mdadm_lvm.configure @@ -11,7 +11,7 @@ config["QemuArgs"] += ["-device", "virtio-scsi-pci,id=scsi0"] for i in range(5): id = f"drivemdadmlvm{i}" - config["QemuDrives"] += [ + config["Drives"] += [ { "Id": id, "Size": "64M", diff --git a/test/TEST-64-UDEV-STORAGE/multipath_basic_failover.configure b/test/TEST-64-UDEV-STORAGE/multipath_basic_failover.configure index 5f323b8a8b..a59927c3b7 100755 --- a/test/TEST-64-UDEV-STORAGE/multipath_basic_failover.configure +++ b/test/TEST-64-UDEV-STORAGE/multipath_basic_failover.configure @@ -18,7 +18,7 @@ for ndisk in range(16): for nback in range(4): id = f"drive{ndisk}x{nback}" - config["QemuDrives"] += [ + config["Drives"] += [ { "Id": id, "Size": size, diff --git a/test/TEST-64-UDEV-STORAGE/nvme_basic.configure b/test/TEST-64-UDEV-STORAGE/nvme_basic.configure index b740c096c4..948003e4de 100755 --- a/test/TEST-64-UDEV-STORAGE/nvme_basic.configure +++ b/test/TEST-64-UDEV-STORAGE/nvme_basic.configure @@ -18,7 +18,7 @@ if 'name "nvme"' not in result.stdout: def add_drive(i: int, serial: str) -> None: global config id = f"nvme{i}" - config["QemuDrives"] += [ + config["Drives"] += [ { "Id": id, "Size": "1M", diff --git a/test/TEST-64-UDEV-STORAGE/nvme_subsystem.configure b/test/TEST-64-UDEV-STORAGE/nvme_subsystem.configure index 3d7900773c..029d4360ba 100755 --- a/test/TEST-64-UDEV-STORAGE/nvme_subsystem.configure +++ b/test/TEST-64-UDEV-STORAGE/nvme_subsystem.configure @@ -16,7 +16,7 @@ if 'name "nvme"' not in result.stdout: exit(77) for id in ("nvme0", "nvme1"): - config["QemuDrives"] += [ + config["Drives"] += [ { "Id": id, "Size": "1M", diff --git a/test/TEST-64-UDEV-STORAGE/simultaneous_events.configure b/test/TEST-64-UDEV-STORAGE/simultaneous_events.configure index a0edb01cc1..668f3e2aa0 100755 --- a/test/TEST-64-UDEV-STORAGE/simultaneous_events.configure +++ b/test/TEST-64-UDEV-STORAGE/simultaneous_events.configure @@ -9,7 +9,7 @@ config = json.load(sys.stdin) for i in range(10): id = f"drivesimultaneousevents{i}" - config["QemuDrives"] += [ + config["Drives"] += [ { "Id": id, "Size": "128M", diff --git a/test/TEST-64-UDEV-STORAGE/virtio_scsi_basic.configure b/test/TEST-64-UDEV-STORAGE/virtio_scsi_basic.configure index ab8d530a3e..91e1fe14e5 100755 --- a/test/TEST-64-UDEV-STORAGE/virtio_scsi_basic.configure +++ b/test/TEST-64-UDEV-STORAGE/virtio_scsi_basic.configure @@ -12,7 +12,7 @@ for i in range(4): for i in range(128): id = f"drive{i}" - config["QemuDrives"] += [ + config["Drives"] += [ { "Id": id, "Size": "1M", diff --git a/test/TEST-64-UDEV-STORAGE/virtio_scsi_identically_named_partitions.configure b/test/TEST-64-UDEV-STORAGE/virtio_scsi_identically_named_partitions.configure index edc5138ce8..dce6b1503f 100755 --- a/test/TEST-64-UDEV-STORAGE/virtio_scsi_identically_named_partitions.configure +++ b/test/TEST-64-UDEV-STORAGE/virtio_scsi_identically_named_partitions.configure @@ -21,7 +21,7 @@ config["QemuArgs"] += ["-device", "virtio-scsi-pci,id=scsi0,num_queues=4"] for i in range(0, num_disk): id = f"drive{i}" - config["QemuDrives"] += [ + config["Drives"] += [ { "Id": id, "Size": "40M", From 8a9b989cba75810406b0c881bcabe03c27fa47e5 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Sun, 5 Jan 2025 23:19:44 +0100 Subject: [PATCH 2/3] mkosi: Make path exclude more generic The systemd rpms we try to install in packit have /usr/bin/bash and /usr/bin/python3 as dependencies which breaks dnf5 because mkosi doesn't download filelists metadata and https://bugzilla.redhat.com/show_bug.cgi?id=2263771 is still not fixed almost a year after being reported. --- mkosi.conf.d/10-centos-fedora/mkosi.prepare | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mkosi.conf.d/10-centos-fedora/mkosi.prepare b/mkosi.conf.d/10-centos-fedora/mkosi.prepare index 9aca0096d5..1e5bd21995 100755 --- a/mkosi.conf.d/10-centos-fedora/mkosi.prepare +++ b/mkosi.conf.d/10-centos-fedora/mkosi.prepare @@ -12,8 +12,11 @@ for DEPS in --requires --recommends --suggests; do # We need --latest-limit=1 to only consider the newest version of the packages. # --latest-limit=1 is per . so we have to pass --arch= explicitly to make sure i686 packages # are not considered on x86-64. + # Because mkosi does not download the filelists repository metadata, we have to exclude all path based + # dependencies, (even those within /usr and /bin because of + # https://bugzilla.redhat.com/show_bug.cgi?id=2263771). dnf repoquery --arch="$DISTRIBUTION_ARCHITECTURE,noarch" --latest-limit=1 --quiet "$DEPS" "${PACKAGES[@]}" | - grep --extended-regexp --invert-match --regexp "$(IFS=\| ; echo "${PACKAGES[*]}")" --regexp /bin/sh --regexp grubby --regexp sdubby --regexp libcurl-minimal | + grep --extended-regexp --invert-match --regexp "$(IFS=\| ; echo "${PACKAGES[*]}")" --regexp /bin --regexp /usr --regexp grubby --regexp sdubby --regexp libcurl-minimal | sort --unique | xargs --delimiter '\n' --no-run-if-empty mkosi-install done From b0b88f18b3237704d1f6d5b3dcb533367816ce23 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 20 Dec 2024 15:34:13 +0100 Subject: [PATCH 3/3] packit: Move fmf metadata into upstream This moves https://src.fedoraproject.org/tests/systemd into upstream so it's one less moving piece that we need to remember. While we're at it, do various cleanups as well: - Remove non-packit logic - Set NO_BUILD in mkosi.local.conf as it's not picked up from the process environment - Don't set unused environment variables - Remove workdir logic as we run in an ephemeral VM already - Drop --verbose from meson test to reduce noise from passed tests - Use mkosi sandbox and drop duplicated deps from metadata file - Stop running the unit tests as they already run as part of the rpm build and as part of the integration tests. - Various style fixes --- .packit.yml | 3 +- test/fmf/.fmf/version | 1 + test/fmf/integration-tests/main.fmf | 10 ++++ test/fmf/integration-tests/test.sh | 88 +++++++++++++++++++++++++++++ test/fmf/plans/ci.fmf | 12 ++++ 5 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 test/fmf/.fmf/version create mode 100644 test/fmf/integration-tests/main.fmf create mode 100755 test/fmf/integration-tests/test.sh create mode 100644 test/fmf/plans/ci.fmf diff --git a/.packit.yml b/.packit.yml index 2dcc9e86ca..eef6e48fcf 100644 --- a/.packit.yml +++ b/.packit.yml @@ -57,8 +57,7 @@ jobs: - job: tests trigger: pull_request - fmf_url: https://src.fedoraproject.org/tests/systemd - fmf_ref: main + fmf_path: test/fmf tmt_plan: ci targets: - fedora-rawhide-x86_64 diff --git a/test/fmf/.fmf/version b/test/fmf/.fmf/version new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/test/fmf/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/test/fmf/integration-tests/main.fmf b/test/fmf/integration-tests/main.fmf new file mode 100644 index 0000000000..a1686c09c3 --- /dev/null +++ b/test/fmf/integration-tests/main.fmf @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +summary: Run the upstream integration test suite +test: ./test.sh +duration: 2h +require: + - coreutils + - distribution-gpg-keys + - dnf + - git-core diff --git a/test/fmf/integration-tests/test.sh b/test/fmf/integration-tests/test.sh new file mode 100755 index 0000000000..51cbab6ec9 --- /dev/null +++ b/test/fmf/integration-tests/test.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# SPDX-License-Identifier: LGPL-2.1-or-later + +set -eux +set -o pipefail + +# Switch SELinux to permissive, since the tests don't set proper contexts +setenforce 0 + +# Prepare systemd source tree +git clone "$PACKIT_TARGET_URL" systemd +pushd systemd +# If we're running in a pull request job, merge the remote branch into the current main +if [[ -n "${PACKIT_SOURCE_URL:-}" ]]; then + git remote add pr "${PACKIT_SOURCE_URL:?}" + git fetch pr "${PACKIT_SOURCE_BRANCH:?}" + git merge "pr/$PACKIT_SOURCE_BRANCH" +fi +git log --oneline -5 +popd + +# Now prepare mkosi, possibly at the same version required by the systemd repo +git clone https://github.com/systemd/mkosi +# If we have it, pin the mkosi version to the same one used by Github Actions, to ensure consistency +if [ -f .github/workflows/mkosi.yml ]; then + mkosi_hash="$(grep systemd/mkosi@ .github/workflows/mkosi.yml | sed "s|.*systemd/mkosi@||g")" + git -C mkosi checkout "$mkosi_hash" +fi + +export PATH="$PWD/mkosi/bin:$PATH" + +pushd systemd + +# shellcheck source=/dev/null +. /etc/os-release || . /usr/lib/os-release + +tee mkosi.local.conf <> /etc/yum.repos.d/copr_build* + +# Disable mkosi's own repository logic +touch /etc/yum.repos.d/mkosi.repo + +# TODO: drop once BTRFS regression is fixed in kernel 6.13 +sed -i "s/Format=btrfs/Format=ext4/" mkosi.repart/10-root.conf + +# If we don't have KVM, skip running in qemu, as it's too slow. But try to load the module first. +modprobe kvm || true +if [[ ! -e /dev/kvm ]]; then + export TEST_NO_QEMU=1 +fi + +# Create missing mountpoint for mkosi sandbox. +mkdir -p /etc/pacman.d/gnupg + +mkosi summary +mkosi -f sandbox true +mkosi -f sandbox meson setup --buildtype=debugoptimized -Dintegration-tests=true build +mkosi genkey +mkosi -f sandbox meson compile -C build mkosi +mkosi -f sandbox \ + meson test \ + -C build \ + --no-rebuild \ + --suite integration-tests \ + --print-errorlogs \ + --no-stdsplit \ + --num-processes "$(($(nproc) - 1))" + +popd diff --git a/test/fmf/plans/ci.fmf b/test/fmf/plans/ci.fmf new file mode 100644 index 0000000000..3e2a16bc89 --- /dev/null +++ b/test/fmf/plans/ci.fmf @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +summary: systemd Fedora test suite +discover: + how: fmf +prepare: + - name: systemd + how: install + exclude: + - systemd-standalone-.* +execute: + how: tmt