Merge pull request #16933 from poettering/copy-hardlinks

copy: optionally recreate hardlinks when copying file trees

src/basic/btrfs-util.c

@@ -1654,7 +1654,10 @@ int btrfs_subvol_snapshot_fd_full(
                 } else if (r < 0)
                         return r;
 
-                r = copy_directory_fd_full(old_fd, new_path, COPY_MERGE|COPY_REFLINK|COPY_SAME_MOUNT|(FLAGS_SET(flags, BTRFS_SNAPSHOT_SIGINT) ? COPY_SIGINT : 0), progress_path, progress_bytes, userdata);
+                r = copy_directory_fd_full(
+                                old_fd, new_path,
+                                COPY_MERGE|COPY_REFLINK|COPY_SAME_MOUNT|COPY_HARDLINKS|(FLAGS_SET(flags, BTRFS_SNAPSHOT_SIGINT) ? COPY_SIGINT : 0),
+                                progress_path, progress_bytes, userdata);
                 if (r < 0)
                         goto fallback_fail;
 

src/basic/copy.c

@@ -22,8 +22,10 @@
 #include "missing_syscall.h"
 #include "mountpoint-util.h"
 #include "nulstr-util.h"
+#include "rm-rf.h"
 #include "selinux-util.h"
 #include "stat-util.h"
+#include "stdio-util.h"
 #include "string-util.h"
 #include "strv.h"
 #include "time-util.h"
@@ -394,6 +396,188 @@ static int fd_copy_symlink(
         return 0;
 }
 
+/* Encapsulates the database we store potential hardlink targets in */
+typedef struct HardlinkContext {
+        int dir_fd;    /* An fd to the directory we use as lookup table. Never AT_FDCWD. Lazily created, when
+                        * we add the first entry. */
+
+        /* These two fields are used to create the hardlink repository directory above — via
+         * mkdirat(parent_fd, subdir) — and are kept so that we can automatically remove the directory again
+         * when we are done. */
+        int parent_fd; /* Possibly AT_FDCWD */
+        char *subdir;
+} HardlinkContext;
+
+static int hardlink_context_setup(
+                HardlinkContext *c,
+                int dt,
+                const char *to,
+                CopyFlags copy_flags) {
+
+        _cleanup_close_ int dt_copy = -1;
+        int r;
+
+        assert(c);
+        assert(c->dir_fd < 0 && c->dir_fd != AT_FDCWD);
+        assert(c->parent_fd < 0);
+        assert(!c->subdir);
+
+        /* If hardlink recreation is requested we have to maintain a database of inodes that are potential
+         * hardlink sources. Given that generally disk sizes have to be assumed to be larger than what fits
+         * into physical RAM we cannot maintain that database in dynamic memory alone. Here we opt to
+         * maintain it on disk, to simplify things: inside the destination directory we'll maintain a
+         * temporary directory consisting of hardlinks of every inode we copied that might be subject of
+         * hardlinks. We can then use that as hardlink source later on. Yes, this means additional disk IO
+         * but thankfully Linux is optimized for this kind of thing. If this ever becomes a performance
+         * bottleneck we can certainly place an in-memory hash table in front of this, but for the beginning,
+         * let's keep things simple, and just use the disk as lookup table for inodes.
+         *
+         * Note that this should have zero performace impact as long as .n_link of all files copied remains
+         * <= 0, because in that case we will not actually allocate the hardlink inode lookup table directory
+         * on disk (we do so lazily, when the first candidate with .n_link > 1 is seen). This means, in the
+         * common case where hardlinks are not used at all or only for few files the fact that we store the
+         * table on disk shouldn't matter perfomance-wise. */
+
+        if (!FLAGS_SET(copy_flags, COPY_HARDLINKS))
+                return 0;
+
+        if (dt == AT_FDCWD)
+                dt_copy = AT_FDCWD;
+        else if (dt < 0)
+                return -EBADF;
+        else {
+                dt_copy = fcntl(dt, F_DUPFD_CLOEXEC, 3);
+                if (dt_copy < 0)
+                        return -errno;
+        }
+
+        r = tempfn_random_child(to, "hardlink", &c->subdir);
+        if (r < 0)
+                return r;
+
+        c->parent_fd = TAKE_FD(dt_copy);
+
+        /* We don't actually create the directory we keep the table in here, that's done on-demand when the
+         * first entry is added, using hardlink_context_realize() below. */
+        return 1;
+}
+
+static int hardlink_context_realize(HardlinkContext *c) {
+        int r;
+
+        if (!c)
+                return 0;
+
+        if (c->dir_fd >= 0) /* Already realized */
+                return 1;
+
+        if (c->parent_fd < 0 && c->parent_fd != AT_FDCWD) /* Not configured */
+                return 0;
+
+        assert(c->subdir);
+
+        if (mkdirat(c->parent_fd, c->subdir, 0700) < 0)
+                return -errno;
+
+        c->dir_fd = openat(c->parent_fd, c->subdir, O_RDONLY|O_DIRECTORY|O_CLOEXEC);
+        if (c->dir_fd < 0) {
+                r = -errno;
+                unlinkat(c->parent_fd, c->subdir, AT_REMOVEDIR);
+                return r;
+        }
+
+        return 1;
+}
+
+static void hardlink_context_destroy(HardlinkContext *c) {
+        int r;
+
+        assert(c);
+
+        /* Automatically remove the hardlink lookup table directory again after we are done. This is used via
+         * _cleanup_() so that we really delete this, even on failure. */
+
+        if (c->dir_fd >= 0) {
+                r = rm_rf_children(TAKE_FD(c->dir_fd), REMOVE_PHYSICAL, NULL); /* consumes dir_fd in all cases, even on failure */
+                if (r < 0)
+                        log_debug_errno(r, "Failed to remove hardlink store (%s) contents, ignoring: %m", c->subdir);
+
+                assert(c->parent_fd >= 0 || c->parent_fd == AT_FDCWD);
+                assert(c->subdir);
+
+                if (unlinkat(c->parent_fd, c->subdir, AT_REMOVEDIR) < 0)
+                        log_debug_errno(errno, "Failed to remove hardlink store (%s) directory, ignoring: %m", c->subdir);
+        }
+
+        assert_cc(AT_FDCWD < 0);
+        c->parent_fd = safe_close(c->parent_fd);
+
+        c->subdir = mfree(c->subdir);
+}
+
+static int try_hardlink(
+                HardlinkContext *c,
+                const struct stat *st,
+                int dt,
+                const char *to) {
+
+        char dev_ino[DECIMAL_STR_MAX(dev_t)*2 + DECIMAL_STR_MAX(uint64_t) + 4];
+
+        assert(st);
+        assert(dt >= 0 || dt == AT_FDCWD);
+        assert(to);
+
+        if (!c) /* No temporary hardlink directory, don't bother */
+                return 0;
+
+        if (st->st_nlink <= 1) /* Source not hardlinked, don't bother */
+                return 0;
+
+        if (c->dir_fd < 0) /* not yet realized, hence empty */
+                return 0;
+
+        xsprintf(dev_ino, "%u:%u:%" PRIu64, major(st->st_dev), minor(st->st_dev), (uint64_t) st->st_ino);
+        if (linkat(c->dir_fd, dev_ino, dt, to, 0) < 0)  {
+                if (errno != ENOENT) /* doesn't exist in store yet */
+                        log_debug_errno(errno, "Failed to hardlink %s to %s, ignoring: %m", dev_ino, to);
+                return 0;
+        }
+
+        return 1;
+}
+
+static int memorize_hardlink(
+                HardlinkContext *c,
+                const struct stat *st,
+                int dt,
+                const char *to) {
+
+        char dev_ino[DECIMAL_STR_MAX(dev_t)*2 + DECIMAL_STR_MAX(uint64_t) + 4];
+        int r;
+
+        assert(st);
+        assert(dt >= 0 || dt == AT_FDCWD);
+        assert(to);
+
+        if (!c) /* No temporary hardlink directory, don't bother */
+                return 0;
+
+        if (st->st_nlink <= 1) /* Source not hardlinked, don't bother */
+                return 0;
+
+        r = hardlink_context_realize(c); /* Create the hardlink store lazily */
+        if (r < 0)
+                return r;
+
+        xsprintf(dev_ino, "%u:%u:%" PRIu64, major(st->st_dev), minor(st->st_dev), (uint64_t) st->st_ino);
+        if (linkat(dt, to, c->dir_fd, dev_ino, 0) < 0) {
+                log_debug_errno(errno, "Failed to hardlink %s to %s, ignoring: %m", to, dev_ino);
+                return 0;
+        }
+
+        return 1;
+}
+
 static int fd_copy_regular(
                 int df,
                 const char *from,
@@ -403,6 +587,7 @@ static int fd_copy_regular(
                 uid_t override_uid,
                 gid_t override_gid,
                 CopyFlags copy_flags,
+                HardlinkContext *hardlink_context,
                 copy_progress_bytes_t progress,
                 void *userdata) {
 
@@ -414,6 +599,12 @@ static int fd_copy_regular(
         assert(st);
         assert(to);
 
+        r = try_hardlink(hardlink_context, st, dt, to);
+        if (r < 0)
+                return r;
+        if (r > 0) /* worked! */
+                return 0;
+
         fdf = openat(df, from, O_RDONLY|O_CLOEXEC|O_NOCTTY|O_NOFOLLOW);
         if (fdf < 0)
                 return -errno;
@@ -456,6 +647,7 @@ static int fd_copy_regular(
                 (void) unlinkat(dt, to, 0);
         }
 
+        (void) memorize_hardlink(hardlink_context, st, dt, to);
         return r;
 }
 
@@ -467,13 +659,20 @@ static int fd_copy_fifo(
                 const char *to,
                 uid_t override_uid,
                 gid_t override_gid,
-                CopyFlags copy_flags) {
+                CopyFlags copy_flags,
+                HardlinkContext *hardlink_context) {
         int r;
 
         assert(from);
         assert(st);
         assert(to);
 
+        r = try_hardlink(hardlink_context, st, dt, to);
+        if (r < 0)
+                return r;
+        if (r > 0) /* worked! */
+                return 0;
+
         if (copy_flags & COPY_MAC_CREATE) {
                 r = mac_selinux_create_file_prepare_at(dt, to, S_IFIFO);
                 if (r < 0)
@@ -494,6 +693,7 @@ static int fd_copy_fifo(
         if (fchmodat(dt, to, st->st_mode & 07777, 0) < 0)
                 r = -errno;
 
+        (void) memorize_hardlink(hardlink_context, st, dt, to);
         return r;
 }
 
@@ -505,13 +705,20 @@ static int fd_copy_node(
                 const char *to,
                 uid_t override_uid,
                 gid_t override_gid,
-                CopyFlags copy_flags) {
+                CopyFlags copy_flags,
+                HardlinkContext *hardlink_context) {
         int r;
 
         assert(from);
         assert(st);
         assert(to);
 
+        r = try_hardlink(hardlink_context, st, dt, to);
+        if (r < 0)
+                return r;
+        if (r > 0) /* worked! */
+                return 0;
+
         if (copy_flags & COPY_MAC_CREATE) {
                 r = mac_selinux_create_file_prepare_at(dt, to, st->st_mode & S_IFMT);
                 if (r < 0)
@@ -532,6 +739,7 @@ static int fd_copy_node(
         if (fchmodat(dt, to, st->st_mode & 07777, 0) < 0)
                 r = -errno;
 
+        (void) memorize_hardlink(hardlink_context, st, dt, to);
         return r;
 }
 
@@ -546,11 +754,17 @@ static int fd_copy_directory(
                 uid_t override_uid,
                 gid_t override_gid,
                 CopyFlags copy_flags,
+                HardlinkContext *hardlink_context,
                 const char *display_path,
                 copy_progress_path_t progress_path,
                 copy_progress_bytes_t progress_bytes,
                 void *userdata) {
 
+        _cleanup_(hardlink_context_destroy) HardlinkContext our_hardlink_context = {
+                .dir_fd = -1,
+                .parent_fd = -1,
+        };
+
         _cleanup_close_ int fdf = -1, fdt = -1;
         _cleanup_closedir_ DIR *d = NULL;
         struct dirent *de;
@@ -570,6 +784,16 @@ static int fd_copy_directory(
         if (fdf < 0)
                 return -errno;
 
+        if (!hardlink_context) {
+                /* If recreating hardlinks is requested let's set up a context for that now. */
+                r = hardlink_context_setup(&our_hardlink_context, dt, to, copy_flags);
+                if (r < 0)
+                        return r;
+                if (r > 0) /* It's enabled and allocated, let's now use the same context for all recursive
+                            * invocations from here down */
+                        hardlink_context = &our_hardlink_context;
+        }
+
         d = take_fdopendir(&fdf);
         if (!d)
                 return -errno;
@@ -668,15 +892,15 @@ static int fd_copy_directory(
                                         continue;
                         }
 
-                        q = fd_copy_directory(dirfd(d), de->d_name, &buf, fdt, de->d_name, original_device, depth_left-1, override_uid, override_gid, copy_flags, child_display_path, progress_path, progress_bytes, userdata);
+                        q = fd_copy_directory(dirfd(d), de->d_name, &buf, fdt, de->d_name, original_device, depth_left-1, override_uid, override_gid, copy_flags, hardlink_context, child_display_path, progress_path, progress_bytes, userdata);
                 } else if (S_ISREG(buf.st_mode))
-                        q = fd_copy_regular(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags, progress_bytes, userdata);
+                        q = fd_copy_regular(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags, hardlink_context, progress_bytes, userdata);
                 else if (S_ISLNK(buf.st_mode))
                         q = fd_copy_symlink(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags);
                 else if (S_ISFIFO(buf.st_mode))
-                        q = fd_copy_fifo(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags);
+                        q = fd_copy_fifo(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags, hardlink_context);
                 else if (S_ISBLK(buf.st_mode) || S_ISCHR(buf.st_mode) || S_ISSOCK(buf.st_mode))
-                        q = fd_copy_node(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags);
+                        q = fd_copy_node(dirfd(d), de->d_name, &buf, fdt, de->d_name, override_uid, override_gid, copy_flags, hardlink_context);
                 else
                         q = -EOPNOTSUPP;
 
@@ -730,15 +954,15 @@ int copy_tree_at_full(
                 return -errno;
 
         if (S_ISREG(st.st_mode))
-                return fd_copy_regular(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags, progress_bytes, userdata);
+                return fd_copy_regular(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags, NULL, progress_bytes, userdata);
         else if (S_ISDIR(st.st_mode))
-                return fd_copy_directory(fdf, from, &st, fdt, to, st.st_dev, COPY_DEPTH_MAX, override_uid, override_gid, copy_flags, NULL, progress_path, progress_bytes, userdata);
+                return fd_copy_directory(fdf, from, &st, fdt, to, st.st_dev, COPY_DEPTH_MAX, override_uid, override_gid, copy_flags, NULL, NULL, progress_path, progress_bytes, userdata);
         else if (S_ISLNK(st.st_mode))
                 return fd_copy_symlink(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags);
         else if (S_ISFIFO(st.st_mode))
-                return fd_copy_fifo(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags);
+                return fd_copy_fifo(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags, NULL);
         else if (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode) || S_ISSOCK(st.st_mode))
-                return fd_copy_node(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags);
+                return fd_copy_node(fdf, from, &st, fdt, to, override_uid, override_gid, copy_flags, NULL);
         else
                 return -EOPNOTSUPP;
 }
@@ -762,7 +986,7 @@ int copy_directory_fd_full(
         if (!S_ISDIR(st.st_mode))
                 return -ENOTDIR;
 
-        return fd_copy_directory(dirfd, NULL, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags, NULL, progress_path, progress_bytes, userdata);
+        return fd_copy_directory(dirfd, NULL, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags, NULL, NULL, progress_path, progress_bytes, userdata);
 }
 
 int copy_directory_full(
@@ -784,7 +1008,7 @@ int copy_directory_full(
         if (!S_ISDIR(st.st_mode))
                 return -ENOTDIR;
 
-        return fd_copy_directory(AT_FDCWD, from, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags, NULL, progress_path, progress_bytes, userdata);
+        return fd_copy_directory(AT_FDCWD, from, &st, AT_FDCWD, to, st.st_dev, COPY_DEPTH_MAX, UID_INVALID, GID_INVALID, copy_flags, NULL, NULL, progress_path, progress_bytes, userdata);
 }
 
 int copy_file_fd_full(

src/basic/copy.h

@@ -17,6 +17,7 @@ typedef enum CopyFlags {
         COPY_CRTIME      = 1 << 5, /* Generate a user.crtime_usec xattr off the source crtime if there is one, on copying */
         COPY_SIGINT      = 1 << 6, /* Check for SIGINT regularly and return EINTR if seen (caller needs to block SIGINT) */
         COPY_MAC_CREATE  = 1 << 7, /* Create files with the correct MAC label (currently SELinux only) */
+        COPY_HARDLINKS   = 1 << 8, /* Try to reproduce hard links */
 } CopyFlags;
 
 typedef int (*copy_progress_bytes_t)(uint64_t n_bytes, void *userdata);