sysupdate: keep database of installed files/patterns, and use to GC them (#42646)

Transfer files might come and go, components might be enabled and
disabled. Patterns might change. Let's keep track of what we install, so
that we can automatically gc everything no longer owned by any enabled
transfer.

src/basic/recurse-dir.c

@@ -27,6 +27,35 @@ static bool ignore_dirent(const struct dirent *de, RecurseDirFlags flags) {
 
         /* Depending on flag either ignore everything starting with ".", or just "." itself and ".." */
 
+        if ((flags & _RECURSE_DIR_MUST_BE_MASK) != 0) {
+                RecurseDirFlags f;
+
+                switch (de->d_type) {
+
+                case DT_DIR:
+                        f = RECURSE_DIR_MUST_BE_DIRECTORY;
+                        break;
+
+                case DT_REG:
+                        f = RECURSE_DIR_MUST_BE_REGULAR;
+                        break;
+
+                case DT_LNK:
+                        f = RECURSE_DIR_MUST_BE_SYMLINK;
+                        break;
+
+                case DT_SOCK:
+                        f = RECURSE_DIR_MUST_BE_SOCKET;
+                        break;
+
+                default:
+                        return true;
+                }
+
+                if (!FLAGS_SET(flags, f))
+                        return true;
+        }
+
         return FLAGS_SET(flags, RECURSE_DIR_IGNORE_DOT) ?
                 de->d_name[0] == '.' :
                 dot_or_dot_dot(de->d_name);
@@ -39,6 +68,9 @@ int readdir_all(int dir_fd, RecurseDirFlags flags, DirectoryEntries **ret) {
 
         assert(dir_fd >= 0);
 
+        if ((flags & _RECURSE_DIR_MUST_BE_MASK) != 0) /* We need the type to validate it */
+                flags |= RECURSE_DIR_ENSURE_TYPE;
+
         /* Returns an array with pointers to "struct dirent" directory entries, optionally sorted.
          *
          * Start with space for up to 8 directory entries. We expect at least 2 ("." + ".."), hence hopefully
@@ -87,9 +119,6 @@ int readdir_all(int dir_fd, RecurseDirFlags flags, DirectoryEntries **ret) {
         de->n_entries = 0;
         struct dirent *entry;
         FOREACH_DIRENT_IN_BUFFER(entry, de->buffer, de->buffer_size) {
-                if (ignore_dirent(entry, flags))
-                        continue;
-
                 if (FLAGS_SET(flags, RECURSE_DIR_ENSURE_TYPE)) {
                         r = dirent_ensure_type(dir_fd, entry);
                         if (r == -ENOENT)
@@ -99,6 +128,9 @@ int readdir_all(int dir_fd, RecurseDirFlags flags, DirectoryEntries **ret) {
                                 return r;
                 }
 
+                if (ignore_dirent(entry, flags))
+                        continue;
+
                 de->n_entries++;
         }
 
@@ -117,14 +149,14 @@ int readdir_all(int dir_fd, RecurseDirFlags flags, DirectoryEntries **ret) {
 
         j = 0;
         FOREACH_DIRENT_IN_BUFFER(entry, de->buffer, de->buffer_size) {
-                if (ignore_dirent(entry, flags))
-                        continue;
-
                 /* If d_type == DT_UNKNOWN that means we failed to ensure the type in the earlier loop and
                  * didn't include the dentry in de->n_entries and as such should skip it here as well. */
                 if (FLAGS_SET(flags, RECURSE_DIR_ENSURE_TYPE) && entry->d_type == DT_UNKNOWN)
                         continue;
 
+                if (ignore_dirent(entry, flags))
+                        continue;
+
                 de->entries[j++] = entry;
         }
         assert(j == de->n_entries);
@@ -166,6 +198,9 @@ int recurse_dir(
         assert(dir_fd >= 0);
         assert(func);
 
+        /* We cannot descend into dirs if we are supposed to ignore them */
+        assert((flags & _RECURSE_DIR_MUST_BE_MASK) == 0 || FLAGS_SET(flags, RECURSE_DIR_MUST_BE_DIRECTORY));
+
         /* This is a lot like ftw()/nftw(), but a lot more modern, i.e. built around openat()/statx()/O_PATH,
          * and under the assumption that fds are not as 'expensive' as they used to be. */
 

src/basic/recurse-dir.h

@@ -7,14 +7,19 @@
 
 typedef enum RecurseDirFlags {
         /* Interpreted by readdir_all() */
-        RECURSE_DIR_SORT         = 1 << 0,  /* sort file directory entries before processing them */
-        RECURSE_DIR_IGNORE_DOT   = 1 << 1,  /* ignore all dot files ("." and ".." are always ignored) */
-        RECURSE_DIR_ENSURE_TYPE  = 1 << 2,  /* guarantees that 'd_type' field of 'de' is not DT_UNKNOWN */
+        RECURSE_DIR_SORT              = 1 << 0,  /* sort file directory entries before processing them */
+        RECURSE_DIR_IGNORE_DOT        = 1 << 1,  /* ignore all dot files ("." and ".." are always ignored) */
+        RECURSE_DIR_ENSURE_TYPE       = 1 << 2,  /* guarantees that 'd_type' field of 'de' is not DT_UNKNOWN */
+        RECURSE_DIR_MUST_BE_DIRECTORY = 1 << 3,  /* ignore all entries that aren't directories */
+        RECURSE_DIR_MUST_BE_REGULAR   = 1 << 4,  /* ignore all entries that aren't regular files */
+        RECURSE_DIR_MUST_BE_SYMLINK   = 1 << 5,  /* ignore all entries that aren't symlinks */
+        RECURSE_DIR_MUST_BE_SOCKET    = 1 << 6,  /* ignore all entries that aren't socket */
+        _RECURSE_DIR_MUST_BE_MASK     = RECURSE_DIR_MUST_BE_DIRECTORY|RECURSE_DIR_MUST_BE_REGULAR|RECURSE_DIR_MUST_BE_SYMLINK|RECURSE_DIR_MUST_BE_SOCKET,
 
         /* Interpreted by recurse_dir() */
-        RECURSE_DIR_SAME_MOUNT   = 1 << 3,  /* skips over subdirectories that are submounts */
-        RECURSE_DIR_INODE_FD     = 1 << 4,  /* passes an opened inode fd (O_DIRECTORY fd in case of dirs, O_PATH otherwise) */
-        RECURSE_DIR_TOPLEVEL     = 1 << 5,  /* call RECURSE_DIR_ENTER/RECURSE_DIR_LEAVE once for top-level dir, too, with dir_fd=-1 and NULL dirent */
+        RECURSE_DIR_SAME_MOUNT        = 1 << 7,  /* skips over subdirectories that are submounts */
+        RECURSE_DIR_INODE_FD          = 1 << 8,  /* passes an opened inode fd (O_DIRECTORY fd in case of dirs, O_PATH otherwise) */
+        RECURSE_DIR_TOPLEVEL          = 1 << 9,  /* call RECURSE_DIR_ENTER/RECURSE_DIR_LEAVE once for top-level dir, too, with dir_fd=-1 and NULL dirent */
 } RecurseDirFlags;
 
 typedef struct DirectoryEntries {

src/basic/sha256.c

@@ -11,6 +11,8 @@ int sha256_fd(int fd, uint64_t max_size, uint8_t ret[static SHA256_DIGEST_SIZE])
         struct sha256_ctx ctx;
         uint64_t total_size = 0;
 
+        assert(fd >= 0);
+
         sha256_init_ctx(&ctx);
 
         for (;;) {
@@ -53,3 +55,12 @@ int parse_sha256(const char *s, uint8_t ret[static SHA256_DIGEST_SIZE]) {
 bool sha256_is_valid(const char *s) {
         return s && in_charset(s, HEXDIGITS) && (strlen(s) == SHA256_DIGEST_SIZE * 2);
 }
+
+char* sha256_direct_hex(const void *buffer, size_t sz) {
+        assert(buffer || sz == 0);
+
+        if (sz == SIZE_MAX)
+                sz = strlen(buffer);
+
+        return hexmem(SHA256_DIRECT(buffer, sz), SHA256_DIGEST_SIZE);
+}

src/basic/sha256.h

@@ -11,3 +11,5 @@ int sha256_fd(int fd, uint64_t max_size, uint8_t ret[static SHA256_DIGEST_SIZE])
 int parse_sha256(const char *s, uint8_t ret[static SHA256_DIGEST_SIZE]);
 
 bool sha256_is_valid(const char *s) _pure_;
+
+char* sha256_direct_hex(const void *buffer, size_t sz);

src/basic/string-util.c

@@ -1145,6 +1145,9 @@ bool string_is_safe(const char *p, StringSafeFlags flags) {
         if (FLAGS_SET(flags, STRING_FILENAME) && !filename_is_valid(p))
                 return false;
 
+        if (FLAGS_SET(flags, STRING_FILENAME_PART) && !filename_part_is_valid(p))
+                return false;
+
         return true;
 }
 

src/basic/string-util.h

@@ -229,7 +229,8 @@ typedef enum StringSafeFlags {
         STRING_ALLOW_QUOTES        = 1 << 4, /* Allow quotes (" or ') */
         STRING_ALLOW_GLOBS         = 1 << 5, /* Allow globs (?, * or [) */
         STRING_FILENAME            = 1 << 6, /* Verify the string is valid as regular filename */
-        STRING_DISALLOW_WHITESPACE = 1 << 7, /* Refuse whitespace (space, tab, newline, …) */
+        STRING_FILENAME_PART       = 1 << 7, /* Verify the string is valid as part of a regular filename */
+        STRING_DISALLOW_WHITESPACE = 1 << 8, /* Refuse whitespace (space, tab, newline, …) */
 } StringSafeFlags;
 
 bool string_is_safe(const char *p, StringSafeFlags flags) _pure_;