sysupdate: keep database of installed files/patterns, and use to GC them (#42646)

Transfer files might come and go, components might be enabled and
disabled. Patterns might change. Let's keep track of what we install, so
that we can automatically gc everything no longer owned by any enabled
transfer.

man/systemd-sysupdate.xml

@@ -204,6 +204,31 @@
         <xi:include href="version-info.xml" xpointer="v251"/></listitem>
       </varlistentry>
 
+      <varlistentry>
+        <term><option>cleanup</option></term>
+
+        <listitem><para>Removes orphaned files that were previously installed by a transfer, but are no
+        longer owned by any currently defined transfer file. Whenever a resource is installed into the file
+        system, <command>systemd-sysupdate</command> records the target directory and the matching pattern in
+        an installation database below <filename>/var/lib/systemd/sysupdate/</filename>. This command
+        iterates through these records, determines which files they match, and deletes those that are no
+        longer covered by any of the patterns of the transfer files currently in place. This is useful to
+        garbage-collect files that used to be owned by a transfer file that has since been modified, disabled
+        or removed altogether (for example because a component is no longer being updated).</para>
+
+        <para>By default only the selected component is processed (i.e. the one selected via
+        <option>--component=</option>, or the default one if none were selected). Use
+        <option>--component-all</option> to process all components known to the installation database in a
+        single invocation.</para>
+
+        <para>This operation only removes files that were installed into the file system (i.e. resources of
+        type <literal>regular-file</literal>, <literal>directory</literal> and <literal>subvolume</literal>,
+        see <citerefentry><refentrytitle>sysupdate.d</refentrytitle><manvolnum>5</manvolnum></citerefentry>);
+        it does not touch partition-based resources.</para>
+
+        <xi:include href="version-info.xml" xpointer="v262"/></listitem>
+      </varlistentry>
+
       <xi:include href="standard-options.xml" xpointer="help" />
       <xi:include href="standard-options.xml" xpointer="version" />
     </variablelist>
@@ -246,6 +271,19 @@
         <xi:include href="version-info.xml" xpointer="v251"/></listitem>
       </varlistentry>
 
+      <varlistentry>
+        <term><option>--component-all</option></term>
+        <term><option>-A</option></term>
+
+        <listitem><para>Instead of operating on a single component, operate on all known components (as well as
+        the default, component-less installation). This is currently only supported for the
+        <command>cleanup</command> command; all other commands will fail if this switch is used.</para>
+
+        <para>This option may not be combined with <option>--component=</option>.</para>
+
+        <xi:include href="version-info.xml" xpointer="v262"/></listitem>
+      </varlistentry>
+
       <varlistentry>
         <term><option>--definitions=</option></term>
 

src/basic/recurse-dir.c

@@ -27,6 +27,35 @@ static bool ignore_dirent(const struct dirent *de, RecurseDirFlags flags) {
 
         /* Depending on flag either ignore everything starting with ".", or just "." itself and ".." */
 
+        if ((flags & _RECURSE_DIR_MUST_BE_MASK) != 0) {
+                RecurseDirFlags f;
+
+                switch (de->d_type) {
+
+                case DT_DIR:
+                        f = RECURSE_DIR_MUST_BE_DIRECTORY;
+                        break;
+
+                case DT_REG:
+                        f = RECURSE_DIR_MUST_BE_REGULAR;
+                        break;
+
+                case DT_LNK:
+                        f = RECURSE_DIR_MUST_BE_SYMLINK;
+                        break;
+
+                case DT_SOCK:
+                        f = RECURSE_DIR_MUST_BE_SOCKET;
+                        break;
+
+                default:
+                        return true;
+                }
+
+                if (!FLAGS_SET(flags, f))
+                        return true;
+        }
+
         return FLAGS_SET(flags, RECURSE_DIR_IGNORE_DOT) ?
                 de->d_name[0] == '.' :
                 dot_or_dot_dot(de->d_name);
@@ -39,6 +68,9 @@ int readdir_all(int dir_fd, RecurseDirFlags flags, DirectoryEntries **ret) {
 
         assert(dir_fd >= 0);
 
+        if ((flags & _RECURSE_DIR_MUST_BE_MASK) != 0) /* We need the type to validate it */
+                flags |= RECURSE_DIR_ENSURE_TYPE;
+
         /* Returns an array with pointers to "struct dirent" directory entries, optionally sorted.
          *
          * Start with space for up to 8 directory entries. We expect at least 2 ("." + ".."), hence hopefully
@@ -87,9 +119,6 @@ int readdir_all(int dir_fd, RecurseDirFlags flags, DirectoryEntries **ret) {
         de->n_entries = 0;
         struct dirent *entry;
         FOREACH_DIRENT_IN_BUFFER(entry, de->buffer, de->buffer_size) {
-                if (ignore_dirent(entry, flags))
-                        continue;
-
                 if (FLAGS_SET(flags, RECURSE_DIR_ENSURE_TYPE)) {
                         r = dirent_ensure_type(dir_fd, entry);
                         if (r == -ENOENT)
@@ -99,6 +128,9 @@ int readdir_all(int dir_fd, RecurseDirFlags flags, DirectoryEntries **ret) {
                                 return r;
                 }
 
+                if (ignore_dirent(entry, flags))
+                        continue;
+
                 de->n_entries++;
         }
 
@@ -117,14 +149,14 @@ int readdir_all(int dir_fd, RecurseDirFlags flags, DirectoryEntries **ret) {
 
         j = 0;
         FOREACH_DIRENT_IN_BUFFER(entry, de->buffer, de->buffer_size) {
-                if (ignore_dirent(entry, flags))
-                        continue;
-
                 /* If d_type == DT_UNKNOWN that means we failed to ensure the type in the earlier loop and
                  * didn't include the dentry in de->n_entries and as such should skip it here as well. */
                 if (FLAGS_SET(flags, RECURSE_DIR_ENSURE_TYPE) && entry->d_type == DT_UNKNOWN)
                         continue;
 
+                if (ignore_dirent(entry, flags))
+                        continue;
+
                 de->entries[j++] = entry;
         }
         assert(j == de->n_entries);
@@ -166,6 +198,9 @@ int recurse_dir(
         assert(dir_fd >= 0);
         assert(func);
 
+        /* We cannot descend into dirs if we are supposed to ignore them */
+        assert((flags & _RECURSE_DIR_MUST_BE_MASK) == 0 || FLAGS_SET(flags, RECURSE_DIR_MUST_BE_DIRECTORY));
+
         /* This is a lot like ftw()/nftw(), but a lot more modern, i.e. built around openat()/statx()/O_PATH,
          * and under the assumption that fds are not as 'expensive' as they used to be. */
 

src/basic/recurse-dir.h

@@ -7,14 +7,19 @@
 
 typedef enum RecurseDirFlags {
         /* Interpreted by readdir_all() */
-        RECURSE_DIR_SORT         = 1 << 0,  /* sort file directory entries before processing them */
-        RECURSE_DIR_IGNORE_DOT   = 1 << 1,  /* ignore all dot files ("." and ".." are always ignored) */
-        RECURSE_DIR_ENSURE_TYPE  = 1 << 2,  /* guarantees that 'd_type' field of 'de' is not DT_UNKNOWN */
+        RECURSE_DIR_SORT              = 1 << 0,  /* sort file directory entries before processing them */
+        RECURSE_DIR_IGNORE_DOT        = 1 << 1,  /* ignore all dot files ("." and ".." are always ignored) */
+        RECURSE_DIR_ENSURE_TYPE       = 1 << 2,  /* guarantees that 'd_type' field of 'de' is not DT_UNKNOWN */
+        RECURSE_DIR_MUST_BE_DIRECTORY = 1 << 3,  /* ignore all entries that aren't directories */
+        RECURSE_DIR_MUST_BE_REGULAR   = 1 << 4,  /* ignore all entries that aren't regular files */
+        RECURSE_DIR_MUST_BE_SYMLINK   = 1 << 5,  /* ignore all entries that aren't symlinks */
+        RECURSE_DIR_MUST_BE_SOCKET    = 1 << 6,  /* ignore all entries that aren't socket */
+        _RECURSE_DIR_MUST_BE_MASK     = RECURSE_DIR_MUST_BE_DIRECTORY|RECURSE_DIR_MUST_BE_REGULAR|RECURSE_DIR_MUST_BE_SYMLINK|RECURSE_DIR_MUST_BE_SOCKET,
 
         /* Interpreted by recurse_dir() */
-        RECURSE_DIR_SAME_MOUNT   = 1 << 3,  /* skips over subdirectories that are submounts */
-        RECURSE_DIR_INODE_FD     = 1 << 4,  /* passes an opened inode fd (O_DIRECTORY fd in case of dirs, O_PATH otherwise) */
-        RECURSE_DIR_TOPLEVEL     = 1 << 5,  /* call RECURSE_DIR_ENTER/RECURSE_DIR_LEAVE once for top-level dir, too, with dir_fd=-1 and NULL dirent */
+        RECURSE_DIR_SAME_MOUNT        = 1 << 7,  /* skips over subdirectories that are submounts */
+        RECURSE_DIR_INODE_FD          = 1 << 8,  /* passes an opened inode fd (O_DIRECTORY fd in case of dirs, O_PATH otherwise) */
+        RECURSE_DIR_TOPLEVEL          = 1 << 9,  /* call RECURSE_DIR_ENTER/RECURSE_DIR_LEAVE once for top-level dir, too, with dir_fd=-1 and NULL dirent */
 } RecurseDirFlags;
 
 typedef struct DirectoryEntries {

src/basic/sha256.c

@@ -11,6 +11,8 @@ int sha256_fd(int fd, uint64_t max_size, uint8_t ret[static SHA256_DIGEST_SIZE])
         struct sha256_ctx ctx;
         uint64_t total_size = 0;
 
+        assert(fd >= 0);
+
         sha256_init_ctx(&ctx);
 
         for (;;) {
@@ -53,3 +55,12 @@ int parse_sha256(const char *s, uint8_t ret[static SHA256_DIGEST_SIZE]) {
 bool sha256_is_valid(const char *s) {
         return s && in_charset(s, HEXDIGITS) && (strlen(s) == SHA256_DIGEST_SIZE * 2);
 }
+
+char* sha256_direct_hex(const void *buffer, size_t sz) {
+        assert(buffer || sz == 0);
+
+        if (sz == SIZE_MAX)
+                sz = strlen(buffer);
+
+        return hexmem(SHA256_DIRECT(buffer, sz), SHA256_DIGEST_SIZE);
+}

src/basic/sha256.h

@@ -11,3 +11,5 @@ int sha256_fd(int fd, uint64_t max_size, uint8_t ret[static SHA256_DIGEST_SIZE])
 int parse_sha256(const char *s, uint8_t ret[static SHA256_DIGEST_SIZE]);
 
 bool sha256_is_valid(const char *s) _pure_;
+
+char* sha256_direct_hex(const void *buffer, size_t sz);

src/basic/string-util.c

@@ -1145,6 +1145,9 @@ bool string_is_safe(const char *p, StringSafeFlags flags) {
         if (FLAGS_SET(flags, STRING_FILENAME) && !filename_is_valid(p))
                 return false;
 
+        if (FLAGS_SET(flags, STRING_FILENAME_PART) && !filename_part_is_valid(p))
+                return false;
+
         return true;
 }
 

src/basic/string-util.h

@@ -229,7 +229,8 @@ typedef enum StringSafeFlags {
         STRING_ALLOW_QUOTES        = 1 << 4, /* Allow quotes (" or ') */
         STRING_ALLOW_GLOBS         = 1 << 5, /* Allow globs (?, * or [) */
         STRING_FILENAME            = 1 << 6, /* Verify the string is valid as regular filename */
-        STRING_DISALLOW_WHITESPACE = 1 << 7, /* Refuse whitespace (space, tab, newline, …) */
+        STRING_FILENAME_PART       = 1 << 7, /* Verify the string is valid as part of a regular filename */
+        STRING_DISALLOW_WHITESPACE = 1 << 8, /* Refuse whitespace (space, tab, newline, …) */
 } StringSafeFlags;
 
 bool string_is_safe(const char *p, StringSafeFlags flags) _pure_;

src/debug-generator/debug-generator.c

@@ -279,7 +279,7 @@ static int process_unit_credentials(const char *credentials_dir) {
 
         assert(credentials_dir);
 
-        r = readdir_all_at(AT_FDCWD, credentials_dir, RECURSE_DIR_SORT|RECURSE_DIR_IGNORE_DOT|RECURSE_DIR_ENSURE_TYPE, &des);
+        r = readdir_all_at(AT_FDCWD, credentials_dir, RECURSE_DIR_SORT|RECURSE_DIR_IGNORE_DOT|RECURSE_DIR_MUST_BE_REGULAR, &des);
         if (r < 0)
                 return log_error_errno(r, "Failed to enumerate credentials from credentials directory '%s': %m", credentials_dir);
 
@@ -287,9 +287,6 @@ static int process_unit_credentials(const char *credentials_dir) {
                 struct dirent *de = *i;
                 const char *unit, *dropin;
 
-                if (de->d_type != DT_REG)
-                        continue;
-
                 unit = startswith(de->d_name, "systemd.extra-unit.");
                 dropin = startswith(de->d_name, "systemd.unit-dropin.");
 

src/kernel-install/kernel-install.c

@@ -11,7 +11,6 @@
 #include "build.h"
 #include "chase.h"
 #include "conf-files.h"
-#include "dirent-util.h"
 #include "dissect-image.h"
 #include "env-file.h"
 #include "env-util.h"
@@ -1247,21 +1246,11 @@ static int verb_add_all(int argc, char *argv[], uintptr_t _data, void *userdata)
                 return log_error_errno(fd, "Failed to open %s/usr/lib/modules/: %m", strempty(arg_root));
 
         _cleanup_free_ DirectoryEntries *de = NULL;
-        r = readdir_all(fd, RECURSE_DIR_SORT|RECURSE_DIR_IGNORE_DOT, &de);
+        r = readdir_all(fd, RECURSE_DIR_SORT|RECURSE_DIR_IGNORE_DOT|RECURSE_DIR_MUST_BE_DIRECTORY, &de);
         if (r < 0)
                 return log_error_errno(r, "Failed to numerate /usr/lib/modules/ contents: %m");
 
         FOREACH_ARRAY(d, de->entries, de->n_entries) {
-                r = dirent_ensure_type(fd, *d);
-                if (r < 0) {
-                        if (r != -ENOENT) /* don't log if just gone by now */
-                                log_debug_errno(r, "Failed to check if '%s/usr/lib/modules/%s' is a directory, ignoring: %m", strempty(arg_root), (*d)->d_name);
-                        continue;
-                }
-
-                if ((*d)->d_type != DT_DIR)
-                        continue;
-
                 _cleanup_free_ char *fn = path_join((*d)->d_name, "vmlinuz");
                 if (!fn)
                         return log_oom();
@@ -1483,7 +1472,7 @@ static int verb_list(int argc, char *argv[], uintptr_t _data, void *userdata) {
                 return log_error_errno(fd, "Failed to open %s/usr/lib/modules/: %m", strempty(arg_root));
 
         _cleanup_free_ DirectoryEntries *de = NULL;
-        r = readdir_all(fd, RECURSE_DIR_SORT|RECURSE_DIR_IGNORE_DOT, &de);
+        r = readdir_all(fd, RECURSE_DIR_SORT|RECURSE_DIR_IGNORE_DOT|RECURSE_DIR_MUST_BE_DIRECTORY, &de);
         if (r < 0)
                 return log_error_errno(r, "Failed to numerate /usr/lib/modules/ contents: %m");
 
@@ -1501,16 +1490,6 @@ static int verb_list(int argc, char *argv[], uintptr_t _data, void *userdata) {
                 if (!j)
                         return log_oom();
 
-                r = dirent_ensure_type(fd, *d);
-                if (r < 0) {
-                        if (r != -ENOENT) /* don't log if just gone by now */
-                                log_debug_errno(r, "Failed to check if '%s/%s' is a directory, ignoring: %m", strempty(arg_root), j);
-                        continue;
-                }
-
-                if ((*d)->d_type != DT_DIR)
-                        continue;
-
                 _cleanup_free_ char *fn = path_join((*d)->d_name, "vmlinuz");
                 if (!fn)
                         return log_oom();

src/libsystemd/sd-varlink/varlink-util.c

@@ -272,7 +272,7 @@ ssize_t varlink_execute_directory(
                 return log_debug_errno(errno, "Failed to open '%s': %m", path);
 
         _cleanup_free_ DirectoryEntries *dentries = NULL;
-        r = readdir_all(fd, RECURSE_DIR_IGNORE_DOT|RECURSE_DIR_ENSURE_TYPE, &dentries);
+        r = readdir_all(fd, RECURSE_DIR_IGNORE_DOT|RECURSE_DIR_MUST_BE_SOCKET|RECURSE_DIR_MUST_BE_SYMLINK, &dentries);
         if (r < 0)
                 return log_debug_errno(r, "Failed to enumerate '%s': %m", path);
 
@@ -282,9 +282,6 @@ ssize_t varlink_execute_directory(
         FOREACH_ARRAY(dp, dentries->entries, dentries->n_entries) {
                 struct dirent *de = *dp;
 
-                if (!IN_SET(de->d_type, DT_SOCK, DT_LNK))
-                        continue;
-
                 t++;
 
                 _cleanup_free_ char *j = path_join(path, de->d_name);

src/nsresourced/userns-registry.c

@@ -1219,7 +1219,7 @@ int userns_registry_per_uid(int dir_fd, uid_t owner) {
 
         _cleanup_free_ DirectoryEntries *de = NULL;
 
-        r = readdir_all_at(dir_fd, uid_fn, RECURSE_DIR_IGNORE_DOT|RECURSE_DIR_ENSURE_TYPE, &de);
+        r = readdir_all_at(dir_fd, uid_fn, RECURSE_DIR_IGNORE_DOT|RECURSE_DIR_MUST_BE_REGULAR, &de);
         if (r == -ENOENT)
                 return 0;
         if (r < 0)
@@ -1228,9 +1228,6 @@ int userns_registry_per_uid(int dir_fd, uid_t owner) {
         FOREACH_ARRAY(i, de->entries, de->n_entries) {
                 struct dirent *e = *i;
 
-                if (e->d_type != DT_REG)
-                        continue;
-
                 if (!startswith(e->d_name, "i") || !endswith(e->d_name, ".userns"))
                         continue;
 

src/shared/creds-util.c

@@ -1805,16 +1805,13 @@ int pick_up_credentials(const PickUpCredential *table, size_t n_table_entry) {
                 return log_error_errno(credential_dir_fd, "Failed to open credentials directory: %m");
 
         _cleanup_free_ DirectoryEntries *des = NULL;
-        r = readdir_all(credential_dir_fd, RECURSE_DIR_SORT|RECURSE_DIR_IGNORE_DOT|RECURSE_DIR_ENSURE_TYPE, &des);
+        r = readdir_all(credential_dir_fd, RECURSE_DIR_SORT|RECURSE_DIR_IGNORE_DOT|RECURSE_DIR_MUST_BE_REGULAR, &des);
         if (r < 0)
                 return log_error_errno(r, "Failed to enumerate credentials: %m");
 
         FOREACH_ARRAY(i, des->entries, des->n_entries) {
                 struct dirent *de = *i;
 
-                if (de->d_type != DT_REG)
-                        continue;
-
                 FOREACH_ARRAY(t, table, n_table_entry) {
                         r = pick_up_credential_one(credential_dir_fd, de->d_name, t);
                         if (r != 0) {

src/shared/pcrextend-util.c

@@ -358,7 +358,7 @@ int pcrextend_imds_userdata_word(const struct iovec *data, char **ret) {
         /* We include both a hash of the complete user data, and a truncated version of the data in the word
          * we measure. The former protects the actual data, the latter is useful for debugging. */
 
-        _cleanup_free_ char *hash = hexmem(SHA256_DIRECT(data->iov_base, data->iov_len), SHA256_DIGEST_SIZE);
+        _cleanup_free_ char *hash = sha256_direct_hex(data->iov_base, data->iov_len);
         if (!hash)
                 return log_oom();
 

src/shared/tar-util.c

@@ -14,7 +14,6 @@
 #include "chattr-util.h"
 #include "fd-util.h"
 #include "fs-util.h"
-#include "hexdecoct.h"
 #include "iovec-util.h"
 #include "libarchive-util.h"
 #include "mountpoint-util.h"
@@ -1184,7 +1183,7 @@ static int hardlink_lookup(
         else
                 assert(d->have_unique_mount_id == (r > 0));
 
-        m = hexmem(SHA256_DIRECT(handle->f_handle, handle->handle_bytes), SHA256_DIGEST_SIZE);
+        m = sha256_direct_hex(handle->f_handle, handle->handle_bytes);
         if (!m)
                 return log_oom();
 

src/storage/storagectl.c

@@ -348,16 +348,13 @@ static int verb_providers(int argc, char *argv[], uintptr_t data, void *userdata
                         return log_error_errno(errno, "Failed to open '%s': %m", socket_path);
         } else {
                 _cleanup_free_ DirectoryEntries *dentries = NULL;
-                r = readdir_all(fd, RECURSE_DIR_SORT|RECURSE_DIR_IGNORE_DOT|RECURSE_DIR_ENSURE_TYPE, &dentries);
+                r = readdir_all(fd, RECURSE_DIR_SORT|RECURSE_DIR_IGNORE_DOT|RECURSE_DIR_MUST_BE_SOCKET, &dentries);
                 if (r < 0)
                         return log_error_errno(r, "Failed to enumerate '%s': %m", socket_path);
 
                 FOREACH_ARRAY(dp, dentries->entries, dentries->n_entries) {
                         struct dirent *de = *dp;
 
-                        if (de->d_type != DT_SOCK)
-                                continue;
-
                         if (!storage_provider_name_is_valid(de->d_name))
                                 continue;
 

src/sysupdate/meson.build

@@ -2,6 +2,7 @@
 
 systemd_sysupdate_sources = files(
         'sysupdate-cache.c',
+        'sysupdate-cleanup.c',
         'sysupdate-feature.c',
         'sysupdate-instance.c',
         'sysupdate-partition.c',
@@ -45,6 +46,11 @@ executables += [
                 'objects' : ['systemd-sysupdate'],
                 'conditions' : ['ENABLE_SYSUPDATED'],
         },
+        test_template + {
+                'sources' : files('test-sysupdate-util.c'),
+                'objects' : ['systemd-sysupdate'],
+                'conditions' : ['ENABLE_SYSUPDATE'],
+        },
 ]
 
 if conf.get('ENABLE_SYSUPDATED') == 1

src/sysupdate/sysupdate-cleanup.c

@@ -0,0 +1,450 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "alloc-util.h"
+#include "chase.h"
+#include "errno-util.h"
+#include "fd-util.h"
+#include "fs-util.h"
+#include "log.h"
+#include "path-util.h"
+#include "recurse-dir.h"
+#include "rm-rf.h"
+#include "sha256.h"
+#include "string-util.h"
+#include "strv.h"
+#include "sysupdate.h"
+#include "sysupdate-cleanup.h"
+#include "sysupdate-pattern.h"
+#include "sysupdate-resource.h"
+#include "sysupdate-transfer.h"
+#include "sysupdate-util.h"
+
+/* This implements the "installdb", which is a simple database of directories + patterns that we ever
+ * installed something into, i.e. for any resource we ever considered "owned" by a transfer file. This is
+ * useful to automatically clean up "orphaned" files that used to be owned by a transfer file, but might no
+ * longer be in newer versions of those transfer files, or where the transfer files/components got
+ * removed/disabled altogether.
+ *
+ * This is ultimately just a per-component content-addressable database implemented via a special directory
+ * in /var/lib/systemd/sysupdate/ that carries symlinks to store the data. Whenever we drop a file into the
+ * system we create an entry in it. An entry symlink's filename is a SHA256 hash of the symlink's target. The
+ * target encodes the directory of the transfer file used, suffixed by the pattern used by the transfer
+ * file. If a transfer file lists multiple patterns, multiple entries are generated, one for each pattern.
+ *
+ * The on-disk layout hence looks roughly like this (for a component "foo" with a transfer file that has
+ * Path=/var/lib/machines/ and two patterns "image_@v.raw" and "image_@v.efi"):
+ *
+ *     /var/lib/systemd/sysupdate/
+ *     └── installdb.foo/
+ *         ├── 8cbee6aa38b98811598118ebbc0eb4c1b7e479e7bfa4312c0b36edc765d1733b → /var/lib/machines/./image_@v.raw
+ *         └── 042266dec8deae09c1e75f3d015734513b75a1daa38b4173c907b5345cf4ed41 → /var/lib/machines/./image_@v.efi
+ *
+ * (For the component-less case the directory is just called "installdb", without the ".foo" suffix.) The
+ * symlink names are the SHA256 hashes (in hex) of their respective targets, and the "/./" separates the
+ * directory part from the pattern part of the target.
+ *
+ * With this in place we have an always updated database of any file and pattern ever owned by any transfer
+ * file we operated on. When doing a clean-up run, we now iterate through all installdb directories (i.e
+ * every component ever installed), and all entries in them. We look for all files the entries match. We then
+ * check if the current set of transfer files also owns these files. If yes, we keep both those files and the
+ * installdb entry. If however no current transfer files own these files anymore, we first delete the files,
+ * and then the installdb entry, since it no longer matches any files. */
+
+static int context_installdb_acquire_fd(Context *c, bool make) {
+        assert(c);
+
+        if (c->installdb_fd >= 0)
+                return 0;
+
+        _cleanup_free_ char *j = NULL;
+        const char *p;
+        if (c->component) {
+                j = strjoin("/var/lib/systemd/sysupdate/installdb.", c->component);
+                if (!j)
+                        return log_oom();
+
+                p = j;
+        } else
+                p = "/var/lib/systemd/sysupdate/installdb";
+
+        ChaseFlags flags = CHASE_MUST_BE_DIRECTORY|CHASE_PREFIX_ROOT;
+
+        if (make)
+                flags |= CHASE_MKDIR_0755;
+
+        c->installdb_fd = chase_and_open(
+                        p,
+                        arg_root,
+                        flags,
+                        O_DIRECTORY|O_CLOEXEC|(make ? O_CREAT : 0),
+                        /* ret_path= */ NULL);
+        if (c->installdb_fd == -ENOENT && !make)
+                return 0;
+        if (c->installdb_fd < 0)
+                return log_error_errno(c->installdb_fd, "Failed to open install database '%s%s': %m", empty_or_root(arg_root) ? "" : arg_root, p);
+
+        return 1;
+}
+
+static int installdb_make_names(const char *path, const char *pattern, char **ret_key, char **ret_value) {
+        assert(path);
+        assert(pattern);
+
+        /* We'll generate a string from the location and the pattern that looks a lot like a path, but
+         * actually isn't, it's a path concatenated with a pattern. We separate both parts with /./. */
+        _cleanup_free_ char *s = strjoin(path, "/./", pattern);
+        if (!s)
+                return log_oom();
+
+        _cleanup_free_ char *h = sha256_direct_hex(s, SIZE_MAX);
+        if (!h)
+                return log_oom();
+
+        if (ret_key)
+                *ret_key = TAKE_PTR(h);
+
+        if (ret_value)
+                *ret_value = TAKE_PTR(s);
+
+        return 0;
+}
+
+int context_installdb_record(
+                Context *c,
+                const char *path,
+                char **patterns) {
+
+        int r;
+
+        assert(c);
+        assert(path);
+
+        /* Creates installdb entries for the specified pairs of directory and pattern. This is called
+         * whenever we install a new file. */
+
+        if (strv_isempty(patterns))
+                return 0;
+
+        /* The provided path comes with arg_root prefixed. Strip it here again */
+        const char *p = arg_root ? ASSERT_PTR(path_startswith(path, arg_root)) : path;
+
+        r = context_installdb_acquire_fd(c, /* make= */ true);
+        if (r < 0)
+                return r;
+
+        int ret = 0;
+        STRV_FOREACH(i, patterns) {
+                _cleanup_free_ char *key = NULL, *value = NULL;
+                r = installdb_make_names(p, *i, &key, &value);
+                if (r < 0)
+                        return r;
+
+                r = symlinkat_idempotent(value, c->installdb_fd, key, /* make_relative= */ false);
+                if (r < 0)
+                        RET_GATHER(ret, log_warning_errno(r, "Failed to add '%s' in '%s' entry to install database: %m", *i, path));
+        }
+
+        return ret;
+}
+
+static int context_is_path_currently_owned(
+                Context *c,
+                const char *path,
+                const char *relpath) {
+
+        int r;
+
+        assert(c);
+        assert(path);
+        assert(relpath);
+
+        /* Checks if the there's a transfer file for the directoy 'path', and then if any of its patterns
+         * match 'relpath' */
+
+        FOREACH_ARRAY(_t, c->transfers, c->n_transfers) {
+                Transfer *t = *_t;
+
+                if (!RESOURCE_IS_FILESYSTEM(t->target.type))
+                        continue;
+
+                if (!path_equal(t->target.path, path))
+                        continue;
+
+                /* OK, so we found a transfer that covers this directory. Now let's see if any of its patterns match */
+
+                r = pattern_match_many(t->target.patterns, relpath, /* ret= */ NULL);
+                if (r < 0) {
+                        _cleanup_free_ char *cl = strv_join(t->target.patterns, "', '");
+                        if (!cl)
+                                return log_oom();
+
+                        return log_error_errno(r, "Failed to match patterns '%s' against '%s': %m", cl, relpath);
+                }
+
+                if (IN_SET(r, PATTERN_MATCH_YES, PATTERN_MATCH_RETRY)) /* Yay, this path is pinned by this transfer file */
+                        return true;
+
+                assert(r == PATTERN_MATCH_NO);
+        }
+
+        return false; /* We found nothing! The path seems to be unowned. */
+}
+
+static int context_installdb_process_directory(
+                Context *c,
+                const char *path,            /* The configured Path= in the original transfer file */
+                const char *relpath,         /* For recursive path matches the path we encountered so far */
+                int dir_fd,
+                DirectoryEntries *de,
+                const char *pattern) {
+
+        int r;
+
+        assert(c);
+        assert(path);
+        assert(dir_fd >= 0);
+        assert(de);
+        assert(pattern);
+
+        int ret = 0;
+        bool keep_installdb = false;
+        FOREACH_ARRAY(_d, de->entries, de->n_entries) {
+                const struct dirent *d = *_d;
+
+                assert(IN_SET(d->d_type, DT_REG, DT_DIR)); /* caller must have filtered via readdir_all() RECURSE_DIR_MUST_BE_xyz flags already */
+
+                _cleanup_free_ char *j = NULL;
+                const char *p;
+                if (relpath) {
+                        j = path_join(relpath, d->d_name);
+                        if (!j)
+                                return log_oom();
+
+                        p = j;
+                } else
+                        p = d->d_name;
+
+                /* Let's see if this entry matches the pattern we recorded in the installdb? */
+                r = pattern_match(pattern, p, /* ret= */ NULL);
+                if (r < 0) {
+                        log_warning_errno(r, "Failed to match pattern '%s' against '%s', ignoring: %m", pattern, p);
+                        /* Can't match, do not clean up */
+                        continue;
+                }
+                if (r == PATTERN_MATCH_NO) /* No match, do not clean up */
+                        continue;
+                if (r == PATTERN_MATCH_RETRY) {
+                        /* Might match in a subdirectory */
+
+                        if (d->d_type != DT_DIR)
+                                continue;
+
+                        _cleanup_close_ int subdir_fd = RET_NERRNO(openat(dir_fd, d->d_name, O_DIRECTORY|O_CLOEXEC|O_NOFOLLOW));
+                        if (subdir_fd == -ENOENT)
+                                continue;
+                        if (subdir_fd < 0) {
+                                RET_GATHER(ret, log_warning_errno(subdir_fd, "Failed to open directory '%s', skipping: %m", p));
+                                continue;
+                        }
+
+                        _cleanup_free_ DirectoryEntries *subde = NULL;
+                        r = readdir_all(subdir_fd, RECURSE_DIR_ENSURE_TYPE|RECURSE_DIR_MUST_BE_DIRECTORY|RECURSE_DIR_MUST_BE_REGULAR, &subde);
+                        if (r < 0) {
+                                RET_GATHER(ret, log_error_errno(r, "Failed to enumerate resource path '%s': %m", p));
+                                continue;
+                        }
+
+                        r = context_installdb_process_directory(c, path, p, subdir_fd, subde, pattern);
+                        if (r < 0)
+                                RET_GATHER(ret, r);
+                        else
+                                keep_installdb = keep_installdb || r;
+                        continue;
+                }
+
+                assert(r == PATTERN_MATCH_YES);
+
+                /* Ah, we have a match, this is a candidate for cleanup. Let's see if any of the currently defined transfer files want to own it */
+
+                r = context_is_path_currently_owned(c, path, p);
+                if (r < 0)
+                        return r;
+                if (r > 0) {
+                        log_debug("Path '%s' is owned by current transfer files, keeping.", p);
+
+                        keep_installdb = true; /* We are keeping the file, let's also keep the installdb entry for it hence */
+                        continue;
+                }
+
+                /* OK, we found an orphaned inode that was owned by a previous invocation, but is no longer
+                 * owned by any of the current transfer files. Delete it. */
+
+                r = rm_rf_child(dir_fd, d->d_name, REMOVE_PHYSICAL|REMOVE_SUBVOLUME|REMOVE_CHMOD);
+                if (r < 0) {
+                        if (r != -ENOENT)
+                                RET_GATHER(ret, log_warning_errno(r, "Failed to remove '%s' which is no longer owned by any transfer files: %m", p));
+
+                        continue;
+                }
+
+                log_info("Successfully removed '%s' which is no longer owned by any transfer files.", p);
+        }
+
+        /* Report back if there's a reason to keep the installdb entry for this directory */
+        return ret < 0 ? ret : keep_installdb;
+}
+
+static int context_installdb_process_entry(
+                Context *c,
+                const char *key,
+                const char *value) {
+
+        int r;
+
+        assert(c);
+        assert(key);
+        assert(value);
+
+        _cleanup_free_ char *h = sha256_direct_hex(value, SIZE_MAX);
+        if (!h)
+                return log_oom();
+
+        if (!streq(key, h)) {
+                log_notice("Invalid hash of install database entry '%s' → '%s', expunging.", key, value);
+                return 0;
+        }
+
+        const char *s = strstr(value, "/./");
+        if (!s) {
+                log_notice("Malformed install database entry '%s' → '%s', expunging.", key, value);
+                return 0;
+        }
+
+        _cleanup_free_ char *path = strndup(value, s - value);
+        if (!path)
+                return log_oom();
+
+        if (!path_is_absolute(path) || !path_is_normalized(path)) {
+                log_notice("Install database path '%s' of entry '%s' → '%s' is invalid, expunging database entry.", path, key, value);
+                return 0;
+        }
+
+        const char *pattern = s + 3;
+
+        /* NB: We set CHASE_PROHIBIT_SYMLINKS because the path was normalized by the writer of the entry
+         * already, and if it isn't anymore, then something is fishy. */
+        _cleanup_close_ int dir_fd = chase_and_open(path, arg_root, CHASE_MUST_BE_DIRECTORY|CHASE_PREFIX_ROOT|CHASE_PROHIBIT_SYMLINKS, O_DIRECTORY|O_CLOEXEC, /* ret_path= */ NULL);
+        if (dir_fd == -ENOENT) {
+                log_debug("Install database path '%s' does not exist, expunging database entry.", path);
+                return 0;
+        }
+        if (dir_fd < 0)
+                return log_error_errno(dir_fd, "Failed to open resource path '%s': %m", path);
+
+        _cleanup_free_ DirectoryEntries *de = NULL;
+        r = readdir_all(dir_fd, RECURSE_DIR_ENSURE_TYPE|RECURSE_DIR_MUST_BE_DIRECTORY|RECURSE_DIR_MUST_BE_REGULAR, &de);
+        if (r < 0)
+                return log_error_errno(r, "Failed to enumerate resource path '%s': %m", path);
+
+        return context_installdb_process_directory(c, path, /* relpath= */ NULL, dir_fd, de, pattern);
+}
+
+int installdb_cleanup_component(const char *node, const char *component) {
+        int r;
+
+        _cleanup_(context_freep) Context* context = NULL;
+        r = context_make_offline(
+                        &context,
+                        node,
+                        component,
+                        /* read_definitions_flags= */ 0);
+        if (r < 0)
+                return r;
+
+        r = context_installdb_acquire_fd(context, /* make= */ false);
+        if (r < 0)
+                return r;
+        if (r == 0) {
+                log_debug("Not cleaning up component '%s', install database is empty.", strna(component));
+                return 0;
+        }
+
+        _cleanup_free_ DirectoryEntries *de = NULL;
+        r = readdir_all(context->installdb_fd, RECURSE_DIR_ENSURE_TYPE|RECURSE_DIR_MUST_BE_SYMLINK, &de);
+        if (r < 0)
+                return log_error_errno(r, "Failed to enumerate install database for component '%s': %m", strna(component));
+
+        int ret = 0;
+        FOREACH_ARRAY(_d, de->entries, de->n_entries) {
+                const struct dirent *d = *_d;
+
+                _cleanup_free_ char *v = NULL;
+                r = readlinkat_malloc(context->installdb_fd, d->d_name, &v);
+                if (r == -ENOENT)
+                        continue;
+                if (r < 0) {
+                        log_warning_errno(r, "Failed to read symlink '%s', ignoring: %m", d->d_name);
+                        continue;
+                }
+
+                r = context_installdb_process_entry(context, d->d_name, v);
+                if (r < 0)  {
+                        RET_GATHER(ret, r);
+                        continue;
+                }
+                if (r > 0) /* Still good, keep installdb entry */
+                        continue;
+
+                r = RET_NERRNO(unlinkat(context->installdb_fd, d->d_name, /* flags= */ 0));
+                if (r < 0 && r != -ENOENT)
+                        RET_GATHER(ret, log_warning_errno(r, "Failed to remove install database entry '%s': %m", d->d_name));
+        }
+
+        return ret;
+}
+
+int installdb_list_components(char ***ret) {
+        int r;
+
+        assert(ret);
+
+        _cleanup_close_ int dir_fd = chase_and_open(
+                        "/var/lib/systemd/sysupdate",
+                        arg_root,
+                        CHASE_MUST_BE_DIRECTORY|CHASE_PREFIX_ROOT,
+                        O_DIRECTORY|O_CLOEXEC,
+                        /* ret_path= */ NULL);
+        if (dir_fd == -ENOENT) {
+                *ret = NULL;
+                return 0;
+        }
+        if (dir_fd < 0)
+                return log_error_errno(dir_fd, "Failed to open '/var/lib/systemd/sysupdate/': %m");
+
+        _cleanup_free_ DirectoryEntries *de = NULL;
+        r = readdir_all(dir_fd, RECURSE_DIR_ENSURE_TYPE|RECURSE_DIR_MUST_BE_DIRECTORY, &de);
+        if (r < 0)
+                return log_error_errno(r, "Failed to enumerate installdb directory '/var/lib/systemd/sysupdate/': %m");
+
+        _cleanup_strv_free_ char **l = NULL;
+        FOREACH_ARRAY(_d, de->entries, de->n_entries) {
+                const struct dirent *d = *_d;
+
+                const char *e = startswith(d->d_name, "installdb.");
+                if (!e)
+                        continue;
+
+                if (!component_name_valid(e))
+                        continue;
+
+                if (strv_extend(&l, e) < 0)
+                        return log_oom();
+        }
+
+        strv_sort_uniq(l);
+        *ret = TAKE_PTR(l);
+        return 0;
+}

src/sysupdate/sysupdate-cleanup.h

@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include "sysupdate-forward.h"
+
+int context_installdb_record(Context *c, const char *path, char **patterns);
+
+int installdb_cleanup_component(const char *node, const char *component);
+int installdb_list_components(char ***ret);

src/sysupdate/sysupdate-forward.h

@@ -6,7 +6,9 @@
 #include "shared-forward.h" /* IWYU pragma: export */
 
 typedef struct Context Context;
-typedef struct PartitionInfo PartitionInfo;
-typedef struct Resource Resource;
 typedef struct Instance Instance;
 typedef struct InstanceMetadata InstanceMetadata;
+typedef struct PartitionInfo PartitionInfo;
+typedef struct Resource Resource;
+typedef struct Transfer Transfer;
+typedef struct UpdateSet UpdateSet;

src/sysupdate/sysupdate-pattern.c

@@ -458,7 +458,7 @@ int pattern_match_many(char **patterns, const char *s, InstanceMetadata *ret) {
                 r = pattern_match(*p, s, &found);
                 if (r < 0)
                         return r;
-                if (r > 0) {
+                if (r != PATTERN_MATCH_NO) {
                         if (ret) {
                                 *ret = found;
                                 found = (InstanceMetadata) INSTANCE_METADATA_NULL;

src/sysupdate/sysupdate-resource.c

@@ -158,6 +158,8 @@ static int resource_load_from_directory_recursive(
                         return log_oom();
 
                 r = pattern_match_many(rr->patterns, rel_joined_for_matching, &extracted_fields);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to match pattern: %m");
                 if (r == PATTERN_MATCH_RETRY) {
                         _cleanup_closedir_ DIR *subdir = NULL;
 
@@ -170,10 +172,7 @@ static int resource_load_from_directory_recursive(
                                 return r;
                         if (r == 0)
                                 continue;
-                }
-                else if (r < 0)
-                        return log_error_errno(r, "Failed to match pattern: %m");
-                else if (r == PATTERN_MATCH_NO)
+                } else if (r == PATTERN_MATCH_NO)
                         continue;
 
                 if (de->d_type == DT_DIR && m != S_IFDIR)

src/sysupdate/sysupdate-transfer.c

@@ -35,6 +35,7 @@
 #include "strv.h"
 #include "sync-util.h"
 #include "sysupdate.h"
+#include "sysupdate-cleanup.h"
 #include "sysupdate-feature.h"
 #include "sysupdate-instance.h"
 #include "sysupdate-pattern.h"
@@ -1672,6 +1673,8 @@ int transfer_install_instance(
                          resource_type_to_string(t->target.type));
 
                 t->temporary_pending_path = mfree(t->temporary_pending_path);
+
+                (void) context_installdb_record(t->context, t->target.path, t->target.patterns);
         }
 
         if (t->final_partition_label) {

src/sysupdate/sysupdate-util.c

@@ -4,8 +4,14 @@
 
 #include "bus-error.h"
 #include "bus-locator.h"
+#include "conf-files.h"
+#include "constants.h"
 #include "log.h"
 #include "login-util.h"
+#include "path-util.h"
+#include "set.h"
+#include "string-util.h"
+#include "strv.h"
 #include "sysupdate-util.h"
 
 int reboot_now(void) {
@@ -24,3 +30,72 @@ int reboot_now(void) {
 
         return 0;
 }
+
+bool component_name_valid(const char *c) {
+        /* See if the specified string enclosed in the directory prefix+suffix would be a valid file name */
+
+        if (!string_is_safe(c, STRING_FILENAME_PART))
+                return false;
+
+        /* Stack allocation is safe, since STRING_FILENAME_PART includes a length check */
+        const char *j = strjoina("sysupdate.", c, ".d");
+
+        return filename_is_valid(j);
+}
+
+int get_component_list(const char *root, char ***ret) {
+        int r;
+
+        assert(ret);
+
+        ConfFile **directories = NULL;
+        size_t n_directories = 0;
+        CLEANUP_ARRAY(directories, n_directories, conf_file_free_array);
+
+        r = conf_files_list_strv_full(
+                        ".d",
+                        root,
+                        CONF_FILES_DIRECTORY|CONF_FILES_WARN,
+                        (const char * const *) CONF_PATHS_STRV(""),
+                        &directories,
+                        &n_directories);
+        if (r < 0)
+                return r;
+
+        _cleanup_set_free_ Set *names = NULL;
+
+        FOREACH_ARRAY(i, directories, n_directories) {
+                ConfFile *e = *i;
+
+                const char *s = startswith(e->filename, "sysupdate.");
+                if (!s)
+                        continue;
+
+                const char *a = endswith(s, ".d");
+                if (!a)
+                        continue;
+
+                if (a == s)
+                        continue;
+
+                _cleanup_free_ char *n = strndup(s, a - s);
+                if (!n)
+                        return log_oom();
+
+                if (!component_name_valid(n))
+                        continue;
+
+                r = set_ensure_consume(&names, &string_hash_ops_free, TAKE_PTR(n));
+                if (r < 0 && r != -EEXIST)
+                        return r;
+        }
+
+        _cleanup_strv_free_ char **z = set_to_strv(&names);
+        if (!z)
+                return -ENOMEM;
+
+        strv_sort(z);
+
+        *ret = TAKE_PTR(z);
+        return 0;
+}

src/sysupdate/sysupdate-util.h

@@ -1,8 +1,12 @@
 /* SPDX-License-Identifier: LGPL-2.1-or-later */
-
 #pragma once
 
+#include <stdbool.h>
+
 int reboot_now(void);
 
-#define SD_SYSUPDATE_OFFLINE  (UINT64_C(1) << 0)
+#define SD_SYSUPDATE_OFFLINE   (UINT64_C(1) << 0)
 #define SD_SYSUPDATE_FLAGS_ALL (SD_SYSUPDATE_OFFLINE)
+
+bool component_name_valid(const char *c);
+int get_component_list(const char *root, char ***ret);

src/sysupdate/sysupdate.c

@@ -8,8 +8,12 @@
 #include "conf-files.h"
 #include "constants.h"
 #include "dissect-image.h"
+#include "errno-util.h"
+#include "fd-util.h"
 #include "format-table.h"
 #include "glyph-util.h"
+#include "hashmap.h"
+#include "help-util.h"
 #include "hexdecoct.h"
 #include "image-policy.h"
 #include "loop-util.h"
@@ -20,20 +24,18 @@
 #include "pager.h"
 #include "parse-argument.h"
 #include "parse-util.h"
-#include "path-util.h"
 #include "pretty-print.h"
-#include "set.h"
 #include "sort-util.h"
 #include "specifier.h"
 #include "string-util.h"
 #include "strv.h"
 #include "sysupdate.h"
+#include "sysupdate-cleanup.h"
 #include "sysupdate-feature.h"
 #include "sysupdate-instance.h"
 #include "sysupdate-transfer.h"
 #include "sysupdate-update-set.h"
 #include "sysupdate-util.h"
-#include "utf8.h"
 #include "verbs.h"
 
 static char *arg_definitions = NULL;
@@ -46,6 +48,7 @@ char *arg_root = NULL;
 static char *arg_image = NULL;
 static bool arg_reboot = false;
 static char *arg_component = NULL;
+static bool arg_component_all = false;
 static int arg_verify = -1;
 static ImagePolicy *arg_image_policy = NULL;
 static bool arg_offline = false;
@@ -64,27 +67,12 @@ const Specifier specifier_table[] = {
         {}
 };
 
-typedef struct Context {
-        Transfer **transfers;
-        size_t n_transfers;
-
-        Transfer **disabled_transfers;
-        size_t n_disabled_transfers;
-
-        Hashmap *features; /* Defined features, keyed by ID */
-
-        UpdateSet **update_sets;
-        size_t n_update_sets;
-
-        UpdateSet *newest_installed, *candidate;
-
-        Hashmap *web_cache; /* Cache for downloaded resources, keyed by URL */
-} Context;
-
-static Context* context_free(Context *c) {
+Context* context_free(Context *c) {
         if (!c)
                 return NULL;
 
+        free(c->component);
+
         FOREACH_ARRAY(tr, c->transfers, c->n_transfers)
                 transfer_free(*tr);
         free(c->transfers);
@@ -101,14 +89,21 @@ static Context* context_free(Context *c) {
 
         hashmap_free(c->web_cache);
 
+        safe_close(c->installdb_fd);
+
         return mfree(c);
 }
 
-DEFINE_TRIVIAL_CLEANUP_FUNC(Context*, context_free);
-
 static Context* context_new(void) {
-        /* For now, no fields to initialize non-zero */
-        return new0(Context, 1);
+        Context *c = new(Context, 1);
+        if (!c)
+                return NULL;
+
+        *c = (Context) {
+                .installdb_fd = -EBADF,
+        };
+
+        return c;
 }
 
 static DEFINE_POINTER_ARRAY_FREE_FUNC(Transfer*, transfer_free);
@@ -171,11 +166,6 @@ static int read_definitions(
         return 0;
 }
 
-typedef enum ReadDefinitionsFlags {
-        READ_DEFINITIONS_REQUIRES_ENABLED_TRANSFERS = 1 << 0,
-        READ_DEFINITIONS_REQUIRES_ANY_TRANSFERS     = 1 << 1,
-} ReadDefinitionsFlags;
-
 static int context_read_definitions(Context *c, const char* node, ReadDefinitionsFlags flags) {
         _cleanup_strv_free_ char **dirs = NULL;
         int r;
@@ -184,7 +174,7 @@ static int context_read_definitions(Context *c, const char* node, ReadDefinition
 
         if (arg_definitions)
                 dirs = strv_new(arg_definitions);
-        else if (arg_component) {
+        else if (c->component) {
                 char **l = CONF_PATHS_STRV("");
                 size_t i = 0;
 
@@ -195,7 +185,7 @@ static int context_read_definitions(Context *c, const char* node, ReadDefinition
                 STRV_FOREACH(dir, l) {
                         char *j;
 
-                        j = strjoin(*dir, "sysupdate.", arg_component, ".d");
+                        j = strjoin(*dir, "sysupdate.", c->component, ".d");
                         if (!j)
                                 return log_oom();
 
@@ -251,10 +241,10 @@ static int context_read_definitions(Context *c, const char* node, ReadDefinition
 
         if (FLAGS_SET(flags, READ_DEFINITIONS_REQUIRES_ANY_TRANSFERS) &&
             c->n_transfers + (FLAGS_SET(flags, READ_DEFINITIONS_REQUIRES_ENABLED_TRANSFERS) ? 0 : c->n_disabled_transfers) == 0) {
-                if (arg_component)
+                if (c->component)
                         return log_error_errno(SYNTHETIC_ERRNO(ENOENT),
                                                "No transfer definitions for component '%s' found.",
-                                               arg_component);
+                                               c->component);
 
                 return log_error_errno(SYNTHETIC_ERRNO(ENOENT),
                                        "No transfer definitions found.");
@@ -931,7 +921,11 @@ static int context_vacuum(
         return 0;
 }
 
-static int context_make_offline(Context **ret, const char *node, ReadDefinitionsFlags read_definitions_flags) {
+int context_make_offline(
+                Context **ret,
+                const char *node,
+                const char *component,
+                ReadDefinitionsFlags read_definitions_flags) {
         _cleanup_(context_freep) Context* context = NULL;
         int r;
 
@@ -944,6 +938,10 @@ static int context_make_offline(Context **ret, const char *node, ReadDefinitions
         if (!context)
                 return log_oom();
 
+        r = free_and_strdup_warn(&context->component, component);
+        if (r < 0)
+                return r;
+
         r = context_read_definitions(context, node, read_definitions_flags);
         if (r < 0)
                 return r;
@@ -956,7 +954,11 @@ static int context_make_offline(Context **ret, const char *node, ReadDefinitions
         return 0;
 }
 
-static int context_make_online(Context **ret, const char *node) {
+static int context_make_online(
+                Context **ret,
+                const char *node,
+                const char *component) {
+
         _cleanup_(context_freep) Context* context = NULL;
         int r;
 
@@ -965,8 +967,11 @@ static int context_make_online(Context **ret, const char *node) {
         /* Like context_make_offline(), but also communicates with the update source looking for new
          * versions (as long as --offline is not specified on the command line). */
 
-        r = context_make_offline(&context, node,
-                                 READ_DEFINITIONS_REQUIRES_ENABLED_TRANSFERS | READ_DEFINITIONS_REQUIRES_ANY_TRANSFERS);
+        r = context_make_offline(
+                        &context,
+                        node,
+                        component,
+                        READ_DEFINITIONS_REQUIRES_ENABLED_TRANSFERS|READ_DEFINITIONS_REQUIRES_ANY_TRANSFERS);
         if (r < 0)
                 return r;
 
@@ -1297,11 +1302,17 @@ static int verb_list(int argc, char *argv[], uintptr_t _data, void *userdata) {
         assert(argc <= 2);
         version = argc >= 2 ? argv[1] : NULL;
 
+        if (arg_component_all)
+                return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), "--component-all currently not supported for '%s'.", argv[0]);
+
         r = process_image(/* ro= */ true, &mounted_dir, &loop_device);
         if (r < 0)
                 return r;
 
-        r = context_make_online(&context, loop_device ? loop_device->node : NULL);
+        r = context_make_online(
+                        &context,
+                        loop_device ? loop_device->node : NULL,
+                        arg_component);
         if (r < 0)
                 return r;
 
@@ -1368,12 +1379,18 @@ static int verb_features(int argc, char *argv[], uintptr_t _data, void *userdata
         assert(argc <= 2);
         feature_id = argc >= 2 ? argv[1] : NULL;
 
+        if (arg_component_all)
+                return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), "--component-all currently not supported for '%s'.", argv[0]);
+
         r = process_image(/* ro= */ true, &mounted_dir, &loop_device);
         if (r < 0)
                 return r;
 
-        r = context_make_offline(&context, loop_device ? loop_device->node : NULL,
-                                 READ_DEFINITIONS_REQUIRES_ANY_TRANSFERS);
+        r = context_make_offline(
+                        &context,
+                        loop_device ? loop_device->node : NULL,
+                        arg_component,
+                        READ_DEFINITIONS_REQUIRES_ANY_TRANSFERS);
         if (r < 0)
                 return r;
 
@@ -1501,11 +1518,17 @@ static int verb_check_new(int argc, char *argv[], uintptr_t _data, void *userdat
 
         assert(argc <= 1);
 
+        if (arg_component_all)
+                return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), "--component-all currently not supported for '%s'.", argv[0]);
+
         r = process_image(/* ro= */ true, &mounted_dir, &loop_device);
         if (r < 0)
                 return r;
 
-        r = context_make_online(&context, loop_device ? loop_device->node : NULL);
+        r = context_make_online(
+                        &context,
+                        loop_device ? loop_device->node : NULL,
+                        arg_component);
         if (r < 0)
                 return r;
 
@@ -1551,6 +1574,9 @@ static int verb_update_impl(int argc, char **argv, UpdateActionFlags action_flag
         assert(argc <= 2);
         version = argc >= 2 ? argv[1] : NULL;
 
+        if (arg_component_all)
+                return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), "--component-all currently not supported for '%s'.", argv[0]);
+
         if (arg_instances_max < 2)
                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
                                       "The --instances-max argument must be >= 2 while updating");
@@ -1569,7 +1595,10 @@ static int verb_update_impl(int argc, char **argv, UpdateActionFlags action_flag
         if (r < 0)
                 return r;
 
-        r = context_make_online(&context, loop_device ? loop_device->node : NULL);
+        r = context_make_online(
+                        &context,
+                        loop_device ? loop_device->node : NULL,
+                        arg_component);
         if (r < 0)
                 return r;
 
@@ -1633,6 +1662,9 @@ static int verb_vacuum(int argc, char *argv[], uintptr_t _data, void *userdata)
 
         assert(argc <= 1);
 
+        if (arg_component_all)
+                return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), "--component-all currently not supported for '%s'.", argv[0]);
+
         if (arg_instances_max < 1)
                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
                                       "The --instances-max argument must be >= 1 while vacuuming");
@@ -1641,8 +1673,11 @@ static int verb_vacuum(int argc, char *argv[], uintptr_t _data, void *userdata)
         if (r < 0)
                 return r;
 
-        r = context_make_offline(&context, loop_device ? loop_device->node : NULL,
-                                 READ_DEFINITIONS_REQUIRES_ANY_TRANSFERS);
+        r = context_make_offline(
+                        &context,
+                        loop_device ? loop_device->node : NULL,
+                        arg_component,
+                        READ_DEFINITIONS_REQUIRES_ANY_TRANSFERS);
         if (r < 0)
                 return r;
 
@@ -1664,12 +1699,15 @@ static int verb_pending_or_reboot(int argc, char *argv[], uintptr_t _data, void
                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
                                        "The --root=/--image= switches may not be combined with the '%s' operation.", argv[0]);
 
-        if (arg_component)
+        if (arg_component || arg_component_all)
                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
-                                       "The --component= switch may not be combined with the '%s' operation, which only applies to the booted OS version.", argv[0]);
+                                       "The --component= and --component-all switches may not be combined with the '%s' operation, which only applies to the booted OS version.", argv[0]);
 
-        r = context_make_offline(&context, /* node= */ NULL,
-                                 READ_DEFINITIONS_REQUIRES_ENABLED_TRANSFERS | READ_DEFINITIONS_REQUIRES_ANY_TRANSFERS);
+        r = context_make_offline(
+                        &context,
+                        /* node= */ NULL,
+                        arg_component,
+                        READ_DEFINITIONS_REQUIRES_ENABLED_TRANSFERS|READ_DEFINITIONS_REQUIRES_ANY_TRANSFERS);
         if (r < 0)
                 return r;
 
@@ -1711,105 +1749,47 @@ static int verb_pending_or_reboot(int argc, char *argv[], uintptr_t _data, void
         return EXIT_SUCCESS;
 }
 
-static int component_name_valid(const char *c) {
-        _cleanup_free_ char *j = NULL;
-
-        /* See if the specified string enclosed in the directory prefix+suffix would be a valid file name */
-
-        if (isempty(c))
-                return false;
-
-        if (string_has_cc(c, NULL))
-                return false;
-
-        if (!utf8_is_valid(c))
-                return false;
-
-        j = strjoin("sysupdate.", c, ".d");
-        if (!j)
-                return -ENOMEM;
-
-        return filename_is_valid(j);
-}
-
 VERB_NOARG(verb_components, "components",
            "Show list of components");
 static int verb_components(int argc, char *argv[], uintptr_t _data, void *userdata) {
         _cleanup_(context_freep) Context* context = NULL;
         _cleanup_(loop_device_unrefp) LoopDevice *loop_device = NULL;
         _cleanup_(umount_and_rmdir_and_freep) char *mounted_dir = NULL;
-        _cleanup_set_free_ Set *names = NULL;
         bool has_default_component = false;
         int r;
 
         assert(argc <= 1);
 
+        if (arg_component_all)
+                return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), "--component-all currently not supported for '%s'.", argv[0]);
+
         r = process_image(/* ro= */ false, &mounted_dir, &loop_device);
         if (r < 0)
                 return r;
 
-        r = context_make_offline(&context, loop_device ? loop_device->node : NULL, 0);
+        r = context_make_offline(
+                        &context,
+                        loop_device ? loop_device->node : NULL,
+                        arg_component,
+                        /* read_definitions_flags= */ 0);
         if (r < 0)
                 return r;
 
-        ConfFile **directories = NULL;
-        size_t n_directories = 0;
-
-        CLEANUP_ARRAY(directories, n_directories, conf_file_free_array);
-
-        r = conf_files_list_strv_full(".d", arg_root, CONF_FILES_DIRECTORY|CONF_FILES_WARN,
-                                      (const char * const *) CONF_PATHS_STRV(""), &directories, &n_directories);
+        _cleanup_strv_free_ char **z = NULL;
+        r = get_component_list(arg_root, &z);
         if (r < 0)
-                return log_error_errno(r, "Failed to enumerate directories: %m");
-
-        FOREACH_ARRAY(i, directories, n_directories) {
-                ConfFile *e = *i;
-
-                if (streq(e->filename, "sysupdate.d")) {
-                        continue;
-                }
-
-                const char *s = startswith(e->filename, "sysupdate.");
-                if (!s)
-                        continue;
-
-                const char *a = endswith(s, ".d");
-                if (!a)
-                        continue;
-
-                _cleanup_free_ char *n = strndup(s, a - s);
-                if (!n)
-                        return log_oom();
-
-                r = component_name_valid(n);
-                if (r < 0)
-                        return log_error_errno(r, "Unable to validate component name '%s': %m", n);
-                if (r == 0)
-                        continue;
-
-                r = set_ensure_put(&names, &string_hash_ops_free, n);
-                if (r < 0 && r != -EEXIST)
-                        return log_error_errno(r, "Failed to add component '%s' to set: %m", n);
-                TAKE_PTR(n);
-        }
+                return log_error_errno(r, "Failed to enumerate components: %m");
 
         /* Does the system have at least one transfer file in /etc/sysupdate.d, which can be considered a
          * TARGET_HOST? See target_get_argument() in sysupdated.c */
         has_default_component = (!arg_definitions &&
-                                 !arg_component &&
+                                 !context->component &&
                                  !arg_root &&
                                  !arg_image &&
                                  context->n_transfers > 0);
 
-        /* We use simple free() rather than strv_free() here, since set_free() will free the strings for us */
-        _cleanup_free_ char **z = set_get_strv(names);
-        if (!z)
-                return log_oom();
-
-        strv_sort(z);
-
         if (!sd_json_format_enabled(arg_json_format_flags)) {
-                if (!has_default_component && set_isempty(names)) {
+                if (!has_default_component && strv_isempty(z)) {
                         log_info("No components defined.");
                         return 0;
                 }
@@ -1836,14 +1816,39 @@ static int verb_components(int argc, char *argv[], uintptr_t _data, void *userda
         return 0;
 }
 
-static int help(void) {
-        _cleanup_free_ char *link = NULL;
-        _cleanup_(table_unrefp) Table *common_options = NULL, *options = NULL, *verbs = NULL;
+VERB_NOARG(verb_cleanup, "cleanup", "Clean up orphaned files");
+static int verb_cleanup(int argc, char *argv[], uintptr_t _data, void *userdata) {
         int r;
 
-        r = terminal_urlify_man("systemd-sysupdate", "8", &link);
+        assert(argc <= 1);
+
+        _cleanup_(loop_device_unrefp) LoopDevice *loop_device = NULL;
+        _cleanup_(umount_and_rmdir_and_freep) char *mounted_dir = NULL;
+        r = process_image(/* ro= */ false, &mounted_dir, &loop_device);
         if (r < 0)
-                return log_oom();
+                return r;
+
+        const char *node = loop_device ? loop_device->node : NULL;
+
+        int ret = 0;
+        RET_GATHER(ret, installdb_cleanup_component(node, arg_component));
+
+        if (arg_component_all) {
+                _cleanup_strv_free_ char **z = NULL;
+                r = installdb_list_components(&z);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to enumerate components: %m");
+
+                STRV_FOREACH(i, z)
+                        RET_GATHER(ret, installdb_cleanup_component(node, *i));
+        }
+
+        return ret;
+}
+
+static int help(void) {
+        _cleanup_(table_unrefp) Table *common_options = NULL, *options = NULL, *verbs = NULL;
+        int r;
 
         r = verbs_get_help_table(&verbs);
         if (r < 0)
@@ -1859,13 +1864,10 @@ static int help(void) {
 
         (void) table_sync_column_widths(0, verbs, common_options, options);
 
-        printf("%s [OPTIONS...] [VERSION]\n"
-               "\n%sUpdate OS images.%s\n"
-               "\n%sCommands:%s\n",
-               program_invocation_short_name,
-               ansi_highlight(), ansi_normal(),
-               ansi_underline(), ansi_normal());
+        help_cmdline("[OPTIONS…] [VERSION]");
+        help_abstract("Update OS images.");
 
+        help_section("Commands");
         r = table_print_or_warn(verbs);
         if (r < 0)
                 return r;
@@ -1874,12 +1876,12 @@ static int help(void) {
         if (r < 0)
                 return r;
 
-        printf("\n%sOptions:%s\n", ansi_underline(), ansi_normal());
+        help_section("Options");
         r = table_print_or_warn(options);
         if (r < 0)
                 return r;
 
-        printf("\nSee the %s for details.\n", link);
+        help_man_page_reference("systemd-sysupdate", "8");
         return 0;
 }
 
@@ -1909,19 +1911,24 @@ static int parse_argv(int argc, char *argv[], char ***remaining_args) {
                        "Select component to update"):
                         if (isempty(opts.arg)) {
                                 arg_component = mfree(arg_component);
+                                arg_component_all = false;
                                 break;
                         }
 
-                        r = component_name_valid(opts.arg);
-                        if (r < 0)
-                                return log_error_errno(r, "Failed to determine if component name is valid: %m");
-                        if (r == 0)
+                        if (!component_name_valid(opts.arg))
                                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Component name invalid: %s", opts.arg);
 
                         r = free_and_strdup_warn(&arg_component, opts.arg);
                         if (r < 0)
                                 return r;
 
+                        arg_component_all = false;
+                        break;
+
+                OPTION('A', "component-all", NULL, "Process all components"):
+
+                        arg_component = mfree(arg_component);
+                        arg_component_all = true;
                         break;
 
                 OPTION_LONG("definitions", "DIR",

src/sysupdate/sysupdate.h

@@ -4,6 +4,37 @@
 #include "specifier.h"
 #include "sysupdate-forward.h"
 
+typedef struct Context {
+        char *component;
+
+        Transfer **transfers;
+        size_t n_transfers;
+
+        Transfer **disabled_transfers;
+        size_t n_disabled_transfers;
+
+        Hashmap *features; /* Defined features, keyed by ID */
+
+        UpdateSet **update_sets;
+        size_t n_update_sets;
+
+        UpdateSet *newest_installed, *candidate;
+
+        Hashmap *web_cache; /* Cache for downloaded resources, keyed by URL */
+
+        int installdb_fd;
+} Context;
+
+Context* context_free(Context *c);
+DEFINE_TRIVIAL_CLEANUP_FUNC(Context*, context_free);
+
+typedef enum ReadDefinitionsFlags {
+        READ_DEFINITIONS_REQUIRES_ENABLED_TRANSFERS = 1 << 0,
+        READ_DEFINITIONS_REQUIRES_ANY_TRANSFERS     = 1 << 1,
+} ReadDefinitionsFlags;
+
+int context_make_offline(Context **ret, const char *node, const char *component, ReadDefinitionsFlags read_definitions_flags);
+
 extern bool arg_sync;
 extern uint64_t arg_instances_max;
 extern char *arg_root;

src/sysupdate/test-sysupdate-util.c

@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include "sysupdate-util.h"
+#include "tests.h"
+
+TEST(component_name_valid) {
+        /* Valid component names: anything that turns "sysupdate.<name>.d" into a valid filename. */
+        ASSERT_TRUE(component_name_valid("foo"));
+        ASSERT_TRUE(component_name_valid("foo-bar"));
+        ASSERT_TRUE(component_name_valid("foo.bar"));
+        ASSERT_TRUE(component_name_valid("foo_bar_baz"));
+        ASSERT_TRUE(component_name_valid("0815"));
+        ASSERT_TRUE(component_name_valid("über"));            /* valid UTF-8 is fine */
+
+        /* Invalid: empty, slashes, control characters, invalid UTF-8. */
+        ASSERT_FALSE(component_name_valid(""));
+        ASSERT_FALSE(component_name_valid("foo/bar"));
+        ASSERT_FALSE(component_name_valid("/foo"));
+        ASSERT_FALSE(component_name_valid("foo/"));
+        ASSERT_FALSE(component_name_valid("foo\tbar"));
+        ASSERT_FALSE(component_name_valid("foo\nbar"));
+        ASSERT_FALSE(component_name_valid("foo\x7f"));
+        ASSERT_FALSE(component_name_valid("\xff"));           /* not valid UTF-8 */
+}
+
+DEFINE_TEST_MAIN(LOG_INFO);

src/test/test-recurse-dir.c

@@ -2,14 +2,17 @@
 
 #include <ftw.h>
 #include <linux/magic.h>
+#include <sys/stat.h>
 
 #include "fd-util.h"
 #include "log.h"
 #include "recurse-dir.h"
+#include "rm-rf.h"
 #include "stat-util.h"
 #include "strv.h"
 #include "tests.h"
 #include "time-util.h"
+#include "tmpfile-util.h"
 
 static char **list_nftw = NULL;
 
@@ -119,6 +122,71 @@ static int recurse_dir_callback(
         return RECURSE_DIR_CONTINUE;
 }
 
+static void assert_entries(DirectoryEntries *de, char **expected) {
+        ASSERT_NOT_NULL(de);
+
+        /* Verifies that the directory entries enumerated in 'de' are exactly the ones listed in
+         * 'expected' (order is irrelevant). */
+
+        ASSERT_EQ(de->n_entries, strv_length(expected));
+
+        FOREACH_ARRAY(i, de->entries, de->n_entries)
+                ASSERT_TRUE(strv_contains(expected, (*i)->d_name));
+
+        STRV_FOREACH(e, expected) {
+                bool found = false;
+
+                FOREACH_ARRAY(i, de->entries, de->n_entries)
+                        if (streq((*i)->d_name, *e)) {
+                                found = true;
+                                break;
+                        }
+
+                ASSERT_TRUE(found);
+        }
+}
+
+static void check_readdir_all(int tfd, RecurseDirFlags flags, char **expected) {
+        _cleanup_free_ DirectoryEntries *de = NULL;
+        _cleanup_close_ int fd = -EBADF;
+
+        /* readdir_all() consumes the directory fd offset, hence reopen a fresh fd for each enumeration. */
+        ASSERT_OK(fd = fd_reopen(tfd, O_DIRECTORY|O_CLOEXEC));
+        ASSERT_OK(readdir_all(fd, flags, &de));
+        assert_entries(de, expected);
+}
+
+static void test_must_be_flags(void) {
+        _cleanup_(rm_rf_physical_and_freep) char *t = NULL;
+        _cleanup_close_ int tfd = -EBADF, reg_fd = -EBADF;
+
+        log_info("/* %s */", __func__);
+
+        /* Populate a temporary directory with one entry of each interesting type and verify that the
+         * RECURSE_DIR_MUST_BE_* flags select exactly the right subset. */
+
+        ASSERT_OK(tfd = mkdtemp_open(NULL, O_DIRECTORY|O_CLOEXEC, &t));
+
+        ASSERT_OK_ERRNO(mkdirat(tfd, "dir", 0777));
+        ASSERT_OK_ERRNO(reg_fd = openat(tfd, "reg", O_CREAT|O_EXCL|O_WRONLY|O_CLOEXEC, 0666));
+        reg_fd = safe_close(reg_fd);
+        ASSERT_OK_ERRNO(symlinkat("reg", tfd, "lnk"));
+        ASSERT_OK_ERRNO(mknodat(tfd, "sock", S_IFSOCK|0666, 0));
+
+        /* Without any MUST_BE flag we get all four entries. */
+        check_readdir_all(tfd, 0, STRV_MAKE("dir", "reg", "lnk", "sock"));
+
+        /* A single MUST_BE flag selects exactly the entries of the matching type. */
+        check_readdir_all(tfd, RECURSE_DIR_MUST_BE_DIRECTORY, STRV_MAKE("dir"));
+        check_readdir_all(tfd, RECURSE_DIR_MUST_BE_REGULAR, STRV_MAKE("reg"));
+        check_readdir_all(tfd, RECURSE_DIR_MUST_BE_SYMLINK, STRV_MAKE("lnk"));
+        check_readdir_all(tfd, RECURSE_DIR_MUST_BE_SOCKET, STRV_MAKE("sock"));
+
+        /* The flags may be combined, in which case we get the union of the matching entries. */
+        check_readdir_all(tfd, RECURSE_DIR_MUST_BE_REGULAR|RECURSE_DIR_MUST_BE_SYMLINK, STRV_MAKE("reg", "lnk"));
+        check_readdir_all(tfd, RECURSE_DIR_MUST_BE_DIRECTORY|RECURSE_DIR_MUST_BE_SOCKET, STRV_MAKE("dir", "sock"));
+}
+
 int main(int argc, char *argv[]) {
         _cleanup_strv_free_ char **list_recurse_dir = NULL;
         const char *p;
@@ -128,6 +196,8 @@ int main(int argc, char *argv[]) {
         log_show_color(true);
         test_setup_logging(LOG_INFO);
 
+        test_must_be_flags();
+
         if (argc > 1)
                 p = argv[1];
         else

src/test/test-string-util.c

@@ -1639,6 +1639,20 @@ TEST(string_is_safe) {
         ASSERT_TRUE(string_is_safe("a\nb", STRING_ALLOW_NEWLINES));
         ASSERT_FALSE(string_is_safe("a\nb", STRING_ALLOW_NEWLINES | STRING_DISALLOW_WHITESPACE));
 
+        /* STRING_FILENAME_PART: like STRING_FILENAME, but "." and ".." are accepted; '/' still rejected. */
+        ASSERT_TRUE(string_is_safe("hello", STRING_FILENAME_PART));
+        ASSERT_TRUE(string_is_safe("hello.txt", STRING_FILENAME_PART));
+        ASSERT_TRUE(string_is_safe("...", STRING_FILENAME_PART));
+        ASSERT_TRUE(string_is_safe(".hidden", STRING_FILENAME_PART));
+        ASSERT_TRUE(string_is_safe(".", STRING_FILENAME_PART));             /* accepted, unlike STRING_FILENAME */
+        ASSERT_TRUE(string_is_safe("..", STRING_FILENAME_PART));            /* accepted, unlike STRING_FILENAME */
+        ASSERT_FALSE(string_is_safe("", STRING_FILENAME_PART));             /* empty still rejected by default */
+        ASSERT_TRUE(string_is_safe("", STRING_FILENAME_PART | STRING_ALLOW_EMPTY)); /* ... unless explicitly allowed */
+        ASSERT_FALSE(string_is_safe("/", STRING_FILENAME_PART));
+        ASSERT_FALSE(string_is_safe("/foo", STRING_FILENAME_PART));
+        ASSERT_FALSE(string_is_safe("foo/bar", STRING_FILENAME_PART));
+        ASSERT_FALSE(string_is_safe("foo/", STRING_FILENAME_PART));
+
         /* Pairwise combinations. */
         ASSERT_TRUE(string_is_safe("", STRING_ALLOW_EMPTY | STRING_ASCII));
         ASSERT_FALSE(string_is_safe("über", STRING_ALLOW_EMPTY | STRING_ASCII));

test/units/TEST-72-SYSUPDATE.sh

@@ -643,4 +643,219 @@ set -e
 [[ $rc -ne 134 ]]
 grep -F "Manifest hash at line 1 decoded to 31 bytes" "$WORKDIR/malformed-manifest/check-new.log" >/dev/null
 
+# Check the "cleanup" verb and the underlying install database. A successful
+# update must record an install database entry for every transfer that installs
+# into the file system, and those entries must cover all installed resources
+# (regular files as well as directories). Once the transfer file owning some
+# resources is removed, "systemd-sysupdate cleanup" must delete the now-orphaned
+# resources (and their install database entries), while leaving resources that
+# are still owned by a transfer file untouched.
+INSTALLDB="/var/lib/systemd/sysupdate/installdb"
+CLEANUP="$WORKDIR/cleanup"
+rm -rf "$CONFIGDIR" "$INSTALLDB" "$CLEANUP"
+mkdir -p "$CONFIGDIR" "$CLEANUP/source" "$CLEANUP/target"
+
+# The "alpha" transfer installs plain regular files, while the "beta" transfer
+# installs whole directories (each populated with a couple of files), to exercise
+# the recursive removal of orphaned directory resources during cleanup.
+cleanup_new_version() {
+    local version="${1:?}"
+    echo "$RANDOM" >"$CLEANUP/source/alpha-$version.bin"
+    rm -rf "$CLEANUP/source/beta-$version"
+    mkdir -p "$CLEANUP/source/beta-$version"
+    echo "$RANDOM" >"$CLEANUP/source/beta-$version/one.txt"
+    echo "$RANDOM" >"$CLEANUP/source/beta-$version/two.txt"
+    (cd "$CLEANUP/source" && sha256sum alpha-* >SHA256SUMS)
+}
+
+# Number of install database entries (symlinks) currently recorded.
+installdb_count() {
+    if [[ -d "$INSTALLDB" ]]; then
+        find "$INSTALLDB" -mindepth 1 -maxdepth 1 -type l | wc -l
+    else
+        echo 0
+    fi
+}
+
+# Assert that every resource (file or directory) currently installed in the
+# target directory is covered by at least one install database entry (i.e. its
+# name matches a recorded pattern that points at the target directory).
+assert_installdb_covers_target() {
+    local f base link tgt path pattern glob covered
+    for f in "$CLEANUP/target"/*; do
+        [[ -e "$f" ]] || continue
+        base="$(basename "$f")"
+        covered=0
+        while read -r link; do
+            tgt="$(readlink "$link")"
+            # Entries are stored as "<path>/./<pattern>".
+            path="${tgt%%/./*}"
+            pattern="${tgt#*/./}"
+            # Translate the sysupdate pattern into a shell glob (only @v is used here).
+            glob="${pattern//@v/*}"
+            # shellcheck disable=SC2053
+            if [[ "$path" == "$CLEANUP/target" && "$base" == $glob ]]; then
+                covered=1
+                break
+            fi
+        done < <(find "$INSTALLDB" -mindepth 1 -maxdepth 1 -type l)
+        [[ "$covered" -eq 1 ]] || { echo "Installed resource '$f' not covered by install database" >&2; exit 1; }
+    done
+}
+
+# Verify the installed beta-<version> directory matches its source.
+verify_beta_synced() {
+    local version="${1:?}"
+    test -d "$CLEANUP/target/beta-$version"
+    cmp "$CLEANUP/source/beta-$version/one.txt" "$CLEANUP/target/beta-$version/one.txt"
+    cmp "$CLEANUP/source/beta-$version/two.txt" "$CLEANUP/target/beta-$version/two.txt"
+}
+
+cat >"$CONFIGDIR/01-alpha.transfer" <<EOF
+[Source]
+Type=regular-file
+Path=$CLEANUP/source
+MatchPattern=alpha-@v.bin
+
+[Target]
+Type=regular-file
+Path=$CLEANUP/target
+MatchPattern=alpha-@v.bin
+InstancesMax=2
+EOF
+
+cat >"$CONFIGDIR/02-beta.transfer" <<EOF
+[Source]
+Type=directory
+Path=$CLEANUP/source
+MatchPattern=beta-@v
+
+[Target]
+Type=directory
+Path=$CLEANUP/target
+MatchPattern=beta-@v
+InstancesMax=2
+EOF
+
+# Install two versions; with InstancesMax=2 both are kept for each transfer.
+cleanup_new_version v1
+"$SYSUPDATE" --verify=no update
+cleanup_new_version v2
+"$SYSUPDATE" --verify=no update
+
+# All four resources must be installed, and the directory resources must have
+# been synced over completely.
+test -f "$CLEANUP/target/alpha-v1.bin"
+test -f "$CLEANUP/target/alpha-v2.bin"
+verify_beta_synced v1
+verify_beta_synced v2
+
+# The update must have recorded one install database entry per transfer pattern,
+# and those entries must cover every installed resource.
+[[ "$(installdb_count)" -eq 2 ]]
+assert_installdb_covers_target
+
+# Running cleanup while all transfer files are still in place must be a no-op:
+# nothing is orphaned, so nothing must be removed.
+"$SYSUPDATE" cleanup
+test -f "$CLEANUP/target/alpha-v1.bin"
+test -f "$CLEANUP/target/alpha-v2.bin"
+verify_beta_synced v1
+verify_beta_synced v2
+[[ "$(installdb_count)" -eq 2 ]]
+assert_installdb_covers_target
+
+# Remove the transfer file owning the "beta" directories and clean up. The beta
+# directories (with all their contents) and their install database entry must be
+# removed, while the alpha files must be kept since their transfer file is still
+# in place.
+rm "$CONFIGDIR/02-beta.transfer"
+"$SYSUPDATE" cleanup
+test -f "$CLEANUP/target/alpha-v1.bin"
+test -f "$CLEANUP/target/alpha-v2.bin"
+test ! -e "$CLEANUP/target/beta-v1"
+test ! -e "$CLEANUP/target/beta-v2"
+[[ "$(installdb_count)" -eq 1 ]]
+assert_installdb_covers_target
+
+# Now remove the remaining transfer file and clean up again. The alpha files and
+# the last install database entry must be removed too.
+rm "$CONFIGDIR/01-alpha.transfer"
+"$SYSUPDATE" cleanup
+test ! -f "$CLEANUP/target/alpha-v1.bin"
+test ! -f "$CLEANUP/target/alpha-v2.bin"
+[[ "$(installdb_count)" -eq 0 ]]
+
+rm -rf "$CONFIGDIR" "$INSTALLDB" "$CLEANUP"
+
+# Briefly check the "--component-all" switch of the "cleanup" verb. Each component
+# keeps its own install database (installdb.<component>), and "cleanup
+# --component-all" must clean up orphaned resources across *all* of them in one
+# go. Set up two components, install a resource into each, then drop both transfer
+# files and run a single "cleanup --component-all" — it must remove both
+# components' resources (and their install database entries).
+COMPALL="$WORKDIR/component-all"
+rm -rf "$COMPALL" /run/sysupdate.comp-a.d /run/sysupdate.comp-b.d \
+    /var/lib/systemd/sysupdate/installdb.comp-a /var/lib/systemd/sysupdate/installdb.comp-b
+mkdir -p "$COMPALL/source" "$COMPALL/target-a" "$COMPALL/target-b" \
+    /run/sysupdate.comp-a.d /run/sysupdate.comp-b.d
+
+echo "$RANDOM" >"$COMPALL/source/comp-a-v1.bin"
+echo "$RANDOM" >"$COMPALL/source/comp-b-v1.bin"
+(cd "$COMPALL/source" && sha256sum comp-* >SHA256SUMS)
+
+cat >/run/sysupdate.comp-a.d/01-comp-a.transfer <<EOF
+[Source]
+Type=regular-file
+Path=$COMPALL/source
+MatchPattern=comp-a-@v.bin
+
+[Target]
+Type=regular-file
+Path=$COMPALL/target-a
+MatchPattern=comp-a-@v.bin
+InstancesMax=1
+EOF
+
+cat >/run/sysupdate.comp-b.d/01-comp-b.transfer <<EOF
+[Source]
+Type=regular-file
+Path=$COMPALL/source
+MatchPattern=comp-b-@v.bin
+
+[Target]
+Type=regular-file
+Path=$COMPALL/target-b
+MatchPattern=comp-b-@v.bin
+InstancesMax=1
+EOF
+
+"$SYSUPDATE" --component=comp-a --verify=no update
+"$SYSUPDATE" --component=comp-b --verify=no update
+test -f "$COMPALL/target-a/comp-a-v1.bin"
+test -f "$COMPALL/target-b/comp-b-v1.bin"
+test -d /var/lib/systemd/sysupdate/installdb.comp-a
+test -d /var/lib/systemd/sysupdate/installdb.comp-b
+
+# --component-all is only supported for the "cleanup" verb, refuse it elsewhere.
+(! "$SYSUPDATE" --component-all --verify=no update)
+
+# With the transfer files still in place "cleanup --component-all" is a no-op:
+# nothing is orphaned.
+"$SYSUPDATE" --component-all cleanup
+test -f "$COMPALL/target-a/comp-a-v1.bin"
+test -f "$COMPALL/target-b/comp-b-v1.bin"
+
+# Drop both transfer files and clean up all components at once. Both resources
+# (and their install database entries) must now be gone.
+rm /run/sysupdate.comp-a.d/01-comp-a.transfer /run/sysupdate.comp-b.d/01-comp-b.transfer
+"$SYSUPDATE" --component-all cleanup
+test ! -e "$COMPALL/target-a/comp-a-v1.bin"
+test ! -e "$COMPALL/target-b/comp-b-v1.bin"
+[[ "$(find /var/lib/systemd/sysupdate/installdb.comp-a -type l | wc -l)" -eq 0 ]]
+[[ "$(find /var/lib/systemd/sysupdate/installdb.comp-b -type l | wc -l)" -eq 0 ]]
+
+rm -rf "$COMPALL" /run/sysupdate.comp-a.d /run/sysupdate.comp-b.d \
+    /var/lib/systemd/sysupdate/installdb.comp-a /var/lib/systemd/sysupdate/installdb.comp-b
+
 touch /testok