diff --git a/man/systemd-repart.xml b/man/systemd-repart.xml index 096eb0c53d..75104c2abc 100644 --- a/man/systemd-repart.xml +++ b/man/systemd-repart.xml @@ -803,6 +803,36 @@ + + Environment + + + + $SOURCE_DATE_EPOCH + + Takes a UNIX timestamp in seconds. Clamps access and modification times written + into the image. See the + reproducible builds + specification. + + + + + + Byte-for-byte reproducible images require mode. Populating a file + system through the kernel writes metadata that cannot be corrected from userspace afterwards. + + Format=vfat images have additional noise: the + directory entry of the volume label is written with a wallclock timestamp by + mkfs.fat8 + with dosfstools versions up to and including 4.2. A later release or a backport of the + upstream fix + sets it to $SOURCE_DATE_EPOCH. + + Offline and online mode also write vfat short names differently, so images + built in both ways never match each other. + + Exit status diff --git a/src/repart/repart.c b/src/repart/repart.c index dfd05f56ab..388f25c398 100644 --- a/src/repart/repart.c +++ b/src/repart/repart.c @@ -6807,6 +6807,17 @@ static int file_is_denylisted(const char *source, Hashmap *denylist) { return 0; } +/* Adding an entry to a directory bumps its mtime. Clamp it back to ts. */ +static int clamp_directory_mtime(int dir_fd, usec_t ts) { + assert(dir_fd >= 0); + + if (ts == USEC_INFINITY) + return 0; + + /* chase() hands out O_PATH descriptors. */ + return futimens_opath(dir_fd, (const struct timespec[2]) { TIMESPEC_OMIT, *TIMESPEC_STORE(ts) }); +} + static int do_copy_files(Context *context, Partition *p, const char *root) { _cleanup_hashmap_free_ Hashmap *subvolumes = NULL; int r; @@ -6830,6 +6841,7 @@ static int do_copy_files(Context *context, Partition *p, const char *root) { } bool copy_ownership = fstype_can_ownership(p->format); + usec_t ts = parse_source_date_epoch(); /* copy_tree_at() automatically copies the permissions of source directories to target directories if * it created them. However, the root directory is created by us, so we have to manually take care @@ -6854,7 +6866,7 @@ static int do_copy_files(Context *context, Partition *p, const char *root) { (void) copy_xattr(sfd, NULL, rfd, NULL, COPY_ALL_XATTRS); if (copy_ownership) (void) copy_access(sfd, rfd); - (void) copy_times(sfd, rfd, 0); + (void) copy_times_full(sfd, rfd, /* flags= */ 0, ts); break; } @@ -6866,7 +6878,6 @@ static int do_copy_files(Context *context, Partition *p, const char *root) { _cleanup_hashmap_free_ Hashmap *denylist = NULL; _cleanup_hashmap_free_ Hashmap *subvolumes_by_source_inode = NULL; _cleanup_close_ int sfd = -EBADF, pfd = -EBADF, tfd = -EBADF; - usec_t ts = parse_source_date_epoch(); r = make_copy_files_denylist(context, p, line->source, line->target, &denylist); if (r < 0) @@ -6915,19 +6926,35 @@ static int do_copy_files(Context *context, Partition *p, const char *root) { if (pfd < 0) return log_error_errno(pfd, "Failed to open parent directory of target: %m"); - r = copy_tree_at( + r = copy_tree_at_full( sfd, ".", pfd, fn, uid, gid, line->flags, - denylist, subvolumes_by_source_inode); + ts, + denylist, subvolumes_by_source_inode, + /* progress_path= */ NULL, + /* progress_bytes= */ NULL, + /* userdata= */ NULL); + if (r >= 0) { + /* Creating the target bumped the parent directory's mtime, + * just like in the regular file case below. */ + r = clamp_directory_mtime(pfd, ts); + if (r < 0) + return log_error_errno( + r, "Failed to set timestamp of '%s': %m", dn); + } } else - r = copy_tree_at( + r = copy_tree_at_full( sfd, ".", tfd, ".", uid, gid, line->flags, - denylist, subvolumes_by_source_inode); + ts, + denylist, subvolumes_by_source_inode, + /* progress_path= */ NULL, + /* progress_bytes= */ NULL, + /* userdata= */ NULL); if (r < 0) return log_error_errno(r, "Failed to copy '%s%s' to '%s%s': %m", strempty(arg_copy_source), line->source, strempty(root), line->target); @@ -6974,15 +7001,12 @@ static int do_copy_files(Context *context, Partition *p, const char *root) { (void) copy_xattr(sfd, NULL, tfd, NULL, COPY_ALL_XATTRS); if (copy_ownership) (void) copy_access(sfd, tfd); - (void) copy_times(sfd, tfd, 0); + (void) copy_times_full(sfd, tfd, /* flags= */ 0, ts); - if (ts != USEC_INFINITY) { - struct timespec tspec; - timespec_store(&tspec, ts); - - if (futimens(pfd, (const struct timespec[2]) { TIMESPEC_OMIT, tspec }) < 0) - return log_error_errno(errno, "Failed to set timestamp of '%s': %m", dn); - } + /* Creating the file bumped the parent directory's mtime. */ + r = clamp_directory_mtime(pfd, ts); + if (r < 0) + return log_error_errno(r, "Failed to set timestamp of '%s': %m", dn); } } @@ -7017,6 +7041,7 @@ static int do_make_directories(Partition *p, const char *root) { } static int do_make_symlinks(Partition *p, const char *root) { + usec_t ts = parse_source_date_epoch(); int r; assert(p); @@ -7032,6 +7057,18 @@ static int do_make_symlinks(Partition *p, const char *root) { if (symlinkat(*target, parent_fd, f) < 0) return log_error_errno(errno, "Failed to create symlink at %s to %s: %m", *path, *target); + + /* The new symlink carries the wall clock, and creating it bumped the directory we put + * it into. We are the last population step, so nothing overwrites this again. */ + if (ts != USEC_INFINITY && + utimensat(parent_fd, f, + (const struct timespec[2]) { *TIMESPEC_STORE(ts), *TIMESPEC_STORE(ts) }, + AT_SYMLINK_NOFOLLOW) < 0) + return log_error_errno(errno, "Failed to set timestamp of symlink '%s': %m", *path); + + r = clamp_directory_mtime(parent_fd, ts); + if (r < 0) + return log_error_errno(r, "Failed to set timestamp of '%s': %m", *path); } return 0; @@ -7218,6 +7255,21 @@ static int partition_populate_directory(Context *context, Partition *p, char **r if (r < 0) return log_error_errno(errno, "Failed to create temporary directory: %m"); + /* Nothing below touches its atime (adding an entry to a directory only moves mtime), so stamp here. */ + usec_t ts = parse_source_date_epoch(); + if (ts != USEC_INFINITY) { + _cleanup_close_ int rfd = -EBADF; + struct timespec tspec; + + rfd = open(root, O_DIRECTORY|O_CLOEXEC|O_NOFOLLOW); + if (rfd < 0) + return log_error_errno(errno, "Failed to open temporary directory: %m"); + + timespec_store(&tspec, ts); + if (futimens(rfd, (const struct timespec[2]) { tspec, tspec }) < 0) + return log_error_errno(errno, "Failed to set timestamp of temporary directory: %m"); + } + r = do_copy_files(context, p, root); if (r < 0) return r; diff --git a/test/units/TEST-58-REPART.sh b/test/units/TEST-58-REPART.sh index d2b0e95970..c862c07d2e 100755 --- a/test/units/TEST-58-REPART.sh +++ b/test/units/TEST-58-REPART.sh @@ -2166,7 +2166,7 @@ EOF } testcase_make_symlinks() { - local defs imgs output + local defs imgs output epoch if systemd-detect-virt --quiet --container; then echo "Skipping MakeSymlinks= test in container." @@ -2191,7 +2191,13 @@ MakeSymlinks=/dir/foo-%a:/bar-%a MakeSymlinks=/dir/bar-%a:../bar-%a EOF - systemd-repart --offline="$OFFLINE" \ + # Build with an epoch, so that this also covers do_make_symlinks()' time stamping. It has to + # stamp the symlink itself and repair the directory it lands in, because creating an entry there + # bumps that directory back to the wall clock. + epoch=1700000000 + + env SOURCE_DATE_EPOCH="$epoch" \ + systemd-repart --offline="$OFFLINE" \ --definitions="$defs" \ --empty=create \ --size=1G \ @@ -2205,6 +2211,12 @@ EOF assert_eq "$(readlink "$imgs/mnt/dir/foo")" "/bar" assert_eq "$(readlink "$imgs/mnt/dir/foo-${architecture}")" "/bar-${architecture}" assert_eq "$(readlink "$imgs/mnt/dir/bar-${architecture}")" "../bar-${architecture}" + + # the symlink's own mtime (no -L) + assert_eq "$(stat -c %Y "$imgs/mnt/dir/foo")" "$epoch" + # a MakeDirectories= dir, stamped with the epoch and then bumped by the symlinks below it. + assert_eq "$(stat -c %Y "$imgs/mnt/dir")" "$epoch" + systemd-dissect -U "$imgs/mnt" } @@ -2756,6 +2768,123 @@ EOF cmp "$imgs/test1.img" "$imgs/test2.img" } +testcase_vfat_reproducibility() { + local defs imgs img dot_efi dot_linux + + if ! command -v mdir >/dev/null; then + echo "Skipping vfat reproducibility test, mtools is not installed." + return 0 + fi + + defs="$(mktemp --directory "/tmp/test-repart.defs.XXXXXXXXXX")" + imgs="$(mktemp --directory "/var/tmp/test-repart.imgs.XXXXXXXXXX")" + # shellcheck disable=SC2064 + trap "rm -rf '$defs' '$imgs'" RETURN + + tee "$defs/esp.conf" <"$imgs/tree/EFI/Linux/old.efi" + echo new >"$imgs/tree/EFI/Linux/new.efi" + touch --date=@1600000000 "$imgs/tree/EFI/Linux/old.efi" + touch --date=@1750000000 "$imgs/tree/EFI/Linux/new.efi" + + # The timestamps as mdir(1) reports them. + local -r time_epoch="2023-11-14 22:13" # $SOURCE_DATE_EPOCH below, i.e. @1700000000 + local -r time_old="2020-09-13 12:26" # old.efi's mtime, before the epoch + local -r time_new="2025-06-15 15:06" # new.efi's mtime, after the epoch + + # Build $1 from the tree and set $img to it, in the "file@@offset" form mdir(1) wants. The + # remaining arguments are passed to env(1), to control $SOURCE_DATE_EPOCH. + build_image() { + local name="$1" output offset + shift + + output=$(env "$@" \ + systemd-repart \ + --offline="$OFFLINE" \ + --definitions="$defs" \ + --empty=create \ + --size=auto \ + --seed="$seed" \ + --dry-run=no \ + --root="$imgs/tree" \ + --json=pretty \ + "$imgs/$name") + + offset=$(jq -r '.[0].offset' <<<"$output") + img="$imgs/$name@@$offset" + + # dump listing for debugging + fat_entry -/ ::/ + } + + # Print $img's mdir(1) entry for the given path. mdir lists a directory's contents; a trailing `*` + # gets the directory's own entry instead. Insensitive path matching, thus works with mtools' 8.3 + # lower case and kernel vfat's upper case names. + fat_entry() { + env MTOOLS_SKIP_CHECK=1 TZ=UTC mdir -i "$img" "$@" + } + + # Same, for the "." entry of directory $1, which cannot be addressed directly. + fat_dot_entry() { + fat_entry "$1" | grep -E '^\. +' + } + + build_image epoch.img SOURCE_DATE_EPOCH=1700000000 + + # The directories and the file newer than the epoch are clamped down to it, ... + assert_in "$time_epoch" "$(fat_entry '::/EFI*')" + assert_in "$time_epoch" "$(fat_entry '::/EFI/Linux*')" + assert_in "$time_epoch" "$(fat_entry '::/EFI/Linux/new.efi')" + # ... while the older one keeps its own mtime. + assert_in "$time_old" "$(fat_entry '::/EFI/Linux/old.efi')" + + # "." and ".." behave differently + dot_efi=$(fat_dot_entry '::/EFI') + dot_linux=$(fat_dot_entry '::/EFI/Linux') + if [[ "$OFFLINE" == "yes" ]]; then + # mmd creates them and respects SOURCE_DATE_EPOCH + assert_in "$time_epoch" "$dot_efi" + assert_in "$time_epoch" "$dot_linux" + else + # The kernel's vfat driver creates them with wallclock mtime. Our utimensat() + # afterwards only rewrites the directory's entry in its *parent*, never this pair. + # Out of reach from userspace, remains unreproducible. Just log them. + echo "'.' entries carry the wall clock, as expected online: $dot_efi / $dot_linux" + fi + + # Without an epoch, both files keep their own mtime. + build_image wallclock.img -u SOURCE_DATE_EPOCH + + assert_in "$time_new" "$(fat_entry '::/EFI/Linux/new.efi')" + assert_in "$time_old" "$(fat_entry '::/EFI/Linux/old.efi')" + + # A value we cannot parse is dropped rather than passed on + build_image bogus.img SOURCE_DATE_EPOCH=99999999999999999999 + + assert_in "$time_new" "$(fat_entry '::/EFI/Linux/new.efi')" + assert_in "$time_old" "$(fat_entry '::/EFI/Linux/old.efi')" + + # SOURCE_DATE_EPOCH in hex - we do parse that, but mtools doesn't + build_image hex.img SOURCE_DATE_EPOCH=0x6553F100 + + assert_in "$time_epoch" "$(fat_entry '::/EFI*')" + + # Note the images still are not reproducible byte for byte: the volume label entry carries + # mkfs.fat's wall clock, in *both* modes, and that needs a dosfstools with $SOURCE_DATE_EPOCH + # support (https://github.com/dosfstools/dosfstools/commit/8da7bc93315c, release > 4.2). Once + # that lands, offline mode can move to a `cmp` like in testcase_ext_reproducibility; online + # mode cannot, because of the "." and ".." entries above. +} + testcase_luks2_keyhash() { local defs imgs output root