From b21ec07b540b7d1f2b83a97f373536dcecaf95f1 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 9 Nov 2022 11:31:15 +0100 Subject: [PATCH 1/4] recurse-dir: optionally, call callback when entering/leaving toplevel dir, too So far recurse_dir() will call the callback whenever we enter a directory, and then pass the struct dirent for that directory, and an fd for the directory the dirent is part of (i.e. the parent of the directory we call things for). For the top-level dir the function is invoked for we will not call the callback however, because we have no dirent for that, and not fd for the directory the top-level dir is part of. Let's add a flag to call it anyway, and in that case pass a NULL dirent and -1 as directory fd. This is useful when we want to treat the top-level dir the same as any dir further down. This is done opt-in since the callback must be ablet to handle a NULL dirent and a -1 directory fd. --- src/basic/recurse-dir.c | 36 +++++++++++++++++++++++++++++++++++- src/basic/recurse-dir.h | 1 + 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/basic/recurse-dir.c b/src/basic/recurse-dir.c index d16ca98f67..908e833501 100644 --- a/src/basic/recurse-dir.c +++ b/src/basic/recurse-dir.c @@ -126,6 +126,7 @@ int recurse_dir( void *userdata) { _cleanup_free_ DirectoryEntries *de = NULL; + STRUCT_STATX_DEFINE(root_sx); int r; assert(dir_fd >= 0); @@ -139,6 +140,26 @@ int recurse_dir( if (n_depth_max == UINT_MAX) /* special marker for "default" */ n_depth_max = DEFAULT_RECURSION_MAX; + if (FLAGS_SET(flags, RECURSE_DIR_TOPLEVEL)) { + if (statx_mask != 0) { + r = statx_fallback(dir_fd, "", AT_EMPTY_PATH, statx_mask, &root_sx); + if (r < 0) + return r; + } + + r = func(RECURSE_DIR_ENTER, + path, + -1, /* we have no parent fd */ + dir_fd, + NULL, /* we have no dirent */ + statx_mask != 0 ? &root_sx : NULL, + userdata); + if (IN_SET(r, RECURSE_DIR_LEAVE_DIRECTORY, RECURSE_DIR_SKIP_ENTRY)) + return 0; + if (r != RECURSE_DIR_CONTINUE) + return r; + } + r = readdir_all(dir_fd, flags, &de); if (r < 0) return r; @@ -397,7 +418,7 @@ int recurse_dir( p, statx_mask, n_depth_max - 1, - flags, + flags &~ RECURSE_DIR_TOPLEVEL, /* we already called the callback for this entry */ func, userdata); if (r != 0) @@ -427,6 +448,19 @@ int recurse_dir( return r; } + if (FLAGS_SET(flags, RECURSE_DIR_TOPLEVEL)) { + + r = func(RECURSE_DIR_LEAVE, + path, + -1, + dir_fd, + NULL, + statx_mask != 0 ? &root_sx : NULL, + userdata); + if (!IN_SET(r, RECURSE_DIR_LEAVE_DIRECTORY, RECURSE_DIR_SKIP_ENTRY, RECURSE_DIR_CONTINUE)) + return r; + } + return 0; } diff --git a/src/basic/recurse-dir.h b/src/basic/recurse-dir.h index 779c91e905..c10c8ddc9d 100644 --- a/src/basic/recurse-dir.h +++ b/src/basic/recurse-dir.h @@ -65,6 +65,7 @@ typedef enum RecurseDirFlags { RECURSE_DIR_ENSURE_TYPE = 1 << 2, /* guarantees that 'd_type' field of 'de' is not DT_UNKNOWN */ RECURSE_DIR_SAME_MOUNT = 1 << 3, /* skips over subdirectories that are submounts */ RECURSE_DIR_INODE_FD = 1 << 4, /* passes an opened inode fd (O_DIRECTORY fd in case of dirs, O_PATH otherwise) */ + RECURSE_DIR_TOPLEVEL = 1 << 5, /* call RECURSE_DIR_ENTER/RECURSE_DIR_LEAVE once for top-level dir, too, with dir_fd=-1 and NULL dirent */ } RecurseDirFlags; typedef struct DirectoryEntries { From db02190eec6fc2d2d9a0cde3a43e957ab0fd3977 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 9 Nov 2022 11:34:52 +0100 Subject: [PATCH 2/4] dissect: add a new "--mtree" switch for generating a BSD mtree(5) compatible file manifest --- src/dissect/dissect.c | 187 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 182 insertions(+), 5 deletions(-) diff --git a/src/dissect/dissect.c b/src/dissect/dissect.c index a1e9cb6add..81764690e9 100644 --- a/src/dissect/dissect.c +++ b/src/dissect/dissect.c @@ -19,6 +19,7 @@ #include "devnum-util.h" #include "dissect-image.h" #include "env-util.h" +#include "escape.h" #include "fd-util.h" #include "fileio.h" #include "format-table.h" @@ -38,6 +39,7 @@ #include "pretty-print.h" #include "process-util.h" #include "recurse-dir.h" +#include "sha256.h" #include "stat-util.h" #include "string-util.h" #include "strv.h" @@ -50,6 +52,7 @@ static enum { ACTION_MOUNT, ACTION_UMOUNT, ACTION_LIST, + ACTION_MTREE, ACTION_WITH, ACTION_COPY_FROM, ACTION_COPY_TO, @@ -87,6 +90,7 @@ static int help(void) { "%1$s [OPTIONS...] --mount IMAGE PATH\n" "%1$s [OPTIONS...] --umount PATH\n" "%1$s [OPTIONS...] --list IMAGE\n" + "%1$s [OPTIONS...] --mtree IMAGE\n" "%1$s [OPTIONS...] --with IMAGE [COMMAND…]\n" "%1$s [OPTIONS...] --copy-from IMAGE PATH [TARGET]\n" "%1$s [OPTIONS...] --copy-to IMAGE [SOURCE] PATH\n\n" @@ -118,6 +122,7 @@ static int help(void) { " -U Shortcut for --umount --rmdir\n" " -l --list List all the files and directories of the specified\n" " OS image\n" + " --mtree Show BSD mtree manifest of OS image\n" " --with Mount, run command, unmount\n" " -x --copy-from Copy files from image to host\n" " -a --copy-to Copy files from host to image\n" @@ -191,6 +196,7 @@ static int parse_argv(int argc, char *argv[]) { ARG_MKDIR, ARG_RMDIR, ARG_JSON, + ARG_MTREE, }; static const struct option options[] = { @@ -211,6 +217,7 @@ static int parse_argv(int argc, char *argv[]) { { "mkdir", no_argument, NULL, ARG_MKDIR }, { "rmdir", no_argument, NULL, ARG_RMDIR }, { "list", no_argument, NULL, 'l' }, + { "mtree", no_argument, NULL, ARG_MTREE }, { "copy-from", no_argument, NULL, 'x' }, { "copy-to", no_argument, NULL, 'a' }, { "json", required_argument, NULL, ARG_JSON }, @@ -278,6 +285,11 @@ static int parse_argv(int argc, char *argv[]) { arg_flags |= DISSECT_IMAGE_READ_ONLY; break; + case ARG_MTREE: + arg_action = ACTION_MTREE; + arg_flags |= DISSECT_IMAGE_READ_ONLY; + break; + case ARG_WITH: arg_action = ACTION_WITH; break; @@ -424,6 +436,7 @@ static int parse_argv(int argc, char *argv[]) { break; case ACTION_LIST: + case ACTION_MTREE: if (optind + 1 != argc) return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Expected an image file path as only argument."); @@ -815,7 +828,167 @@ static int list_print_item( return RECURSE_DIR_CONTINUE; } -static int action_list_or_copy(DissectedImage *m, LoopDevice *d) { +static int get_file_sha256(int inode_fd, uint8_t ret[static SHA256_DIGEST_SIZE]) { + _cleanup_close_ int fd = -1; + struct sha256_ctx ctx; + + /* convert O_PATH fd into a regular one */ + fd = fd_reopen(inode_fd, O_RDONLY|O_CLOEXEC); + if (fd < 0) + return fd; + + /* Calculating the SHA sum might be slow, hence let's flush STDOUT first, to give user an idea where we are slow. */ + fflush(stdout); + + sha256_init_ctx(&ctx); + + for (;;) { + uint8_t buffer[64 * 1024]; + ssize_t n; + + n = read(fd, buffer, sizeof(buffer)); + if (n < 0) + return -errno; + if (n == 0) + break; + + sha256_process_bytes(buffer, n, &ctx); + } + + sha256_finish_ctx(&ctx, ret); + return 0; +} + +static int mtree_print_item( + RecurseDirEvent event, + const char *path, + int dir_fd, + int inode_fd, + const struct dirent *de, + const struct statx *sx, + void *userdata) { + + int r; + + assert_se(path); + assert_se(sx); + + if (IN_SET(event, RECURSE_DIR_ENTER, RECURSE_DIR_ENTRY)) { + _cleanup_free_ char *escaped = NULL; + + if (isempty(path)) + path = "."; + else { + /* BSD mtree uses either C or octal escaping, and covers whitespace, comments and glob characters. We use C style escaping and follow suit */ + escaped = xescape(path, WHITESPACE COMMENTS GLOB_CHARS); + if (!escaped) + return log_oom(); + + path = escaped; + } + + printf("%s", isempty(path) ? "." : path); + + if (FLAGS_SET(sx->stx_mask, STATX_TYPE)) { + if (S_ISDIR(sx->stx_mode)) + printf("%s/%s", ansi_grey(), ansi_normal()); + + printf(" %stype=%s%s%s%s", + ansi_grey(), + ansi_normal(), + S_ISDIR(sx->stx_mode) ? ansi_highlight_blue() : + S_ISLNK(sx->stx_mode) ? ansi_highlight_cyan() : + (S_ISFIFO(sx->stx_mode) || S_ISCHR(sx->stx_mode) || S_ISBLK(sx->stx_mode)) ? ansi_highlight_yellow4() : + S_ISSOCK(sx->stx_mode) ? ansi_highlight_magenta() : "", + ASSERT_PTR(S_ISDIR(sx->stx_mode) ? "dir" : + S_ISREG(sx->stx_mode) ? "file" : + S_ISLNK(sx->stx_mode) ? "link" : + S_ISFIFO(sx->stx_mode) ? "fifo" : + S_ISBLK(sx->stx_mode) ? "block" : + S_ISCHR(sx->stx_mode) ? "char" : + S_ISSOCK(sx->stx_mode) ? "socket" : NULL), + ansi_normal()); + } + + if (FLAGS_SET(sx->stx_mask, STATX_MODE) && (!FLAGS_SET(sx->stx_mask, STATX_TYPE) || !S_ISLNK(sx->stx_mode))) + printf(" %smode=%s%04o", + ansi_grey(), + ansi_normal(), + (unsigned) (sx->stx_mode & 0777)); + + if (FLAGS_SET(sx->stx_mask, STATX_UID)) + printf(" %suid=%s" UID_FMT, + ansi_grey(), + ansi_normal(), + sx->stx_uid); + + if (FLAGS_SET(sx->stx_mask, STATX_GID)) + printf(" %sgid=%s" GID_FMT, + ansi_grey(), + ansi_normal(), + sx->stx_gid); + + if (FLAGS_SET(sx->stx_mask, STATX_TYPE|STATX_SIZE) && S_ISREG(sx->stx_mode)) { + printf(" %ssize=%s%" PRIu64, + ansi_grey(), + ansi_normal(), + (uint64_t) sx->stx_size); + + if (inode_fd >= 0 && sx->stx_size > 0) { + uint8_t hash[SHA256_DIGEST_SIZE]; + + r = get_file_sha256(inode_fd, hash); + if (r < 0) + log_warning_errno(r, "Failed to calculate file SHA256 sum for '%s', ignoring: %m", path); + else { + _cleanup_free_ char *h = NULL; + + h = hexmem(hash, sizeof(hash)); + if (!h) + return log_oom(); + + printf(" %ssha256sum=%s%s", + ansi_grey(), + ansi_normal(), + h); + } + } + } + + if (FLAGS_SET(sx->stx_mask, STATX_TYPE) && S_ISLNK(sx->stx_mode) && inode_fd >= 0) { + _cleanup_free_ char *target = NULL; + + r = readlinkat_malloc(inode_fd, "", &target); + if (r < 0) + log_warning_errno(r, "Failed to read symlink '%s', ignoring: %m", path); + else { + _cleanup_free_ char *target_escaped = NULL; + + target_escaped = xescape(target, WHITESPACE COMMENTS GLOB_CHARS); + if (!target_escaped) + return log_oom(); + + printf(" %slink=%s%s", + ansi_grey(), + ansi_normal(), + target_escaped); + } + } + + if (FLAGS_SET(sx->stx_mask, STATX_TYPE) && (S_ISBLK(sx->stx_mode) || S_ISCHR(sx->stx_mode))) + printf(" %sdevice=%slinux,%" PRIu64 ",%" PRIu64, + ansi_grey(), + ansi_normal(), + (uint64_t) sx->stx_rdev_major, + (uint64_t) sx->stx_rdev_minor); + + printf("\n"); + } + + return RECURSE_DIR_CONTINUE; +} + +static int action_list_or_mtree_or_copy(DissectedImage *m, LoopDevice *d) { _cleanup_(umount_and_rmdir_and_freep) char *mounted_dir = NULL; _cleanup_(rmdir_and_freep) char *created_dir = NULL; _cleanup_free_ char *temp = NULL; @@ -976,15 +1149,18 @@ static int action_list_or_copy(DissectedImage *m, LoopDevice *d) { } else { _cleanup_close_ int dfd = -1; - assert(arg_action == ACTION_LIST); - dfd = open(mounted_dir, O_DIRECTORY|O_CLOEXEC|O_RDONLY); if (dfd < 0) return log_error_errno(errno, "Failed to open mount directory: %m"); pager_open(arg_pager_flags); - r = recurse_dir(dfd, NULL, 0, UINT_MAX, RECURSE_DIR_SORT, list_print_item, NULL); + if (arg_action == ACTION_LIST) + r = recurse_dir(dfd, NULL, 0, UINT_MAX, RECURSE_DIR_SORT, list_print_item, NULL); + else if (arg_action == ACTION_MTREE) + r = recurse_dir(dfd, ".", STATX_TYPE|STATX_MODE|STATX_UID|STATX_GID|STATX_SIZE, UINT_MAX, RECURSE_DIR_SORT|RECURSE_DIR_INODE_FD|RECURSE_DIR_TOPLEVEL, mtree_print_item, NULL); + else + assert_not_reached(); if (r < 0) return log_error_errno(r, "Failed to list image: %m"); } @@ -1198,9 +1374,10 @@ static int run(int argc, char *argv[]) { break; case ACTION_LIST: + case ACTION_MTREE: case ACTION_COPY_FROM: case ACTION_COPY_TO: - r = action_list_or_copy(m, d); + r = action_list_or_mtree_or_copy(m, d); break; case ACTION_WITH: From b5b40106c7a733d8a88268a91fd08a2edb395cae Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 9 Nov 2022 12:12:02 +0100 Subject: [PATCH 3/4] man: document new systemd-dissect --mtree switch --- man/systemd-dissect.xml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/man/systemd-dissect.xml b/man/systemd-dissect.xml index f857ecacc7..0b6bf793ce 100644 --- a/man/systemd-dissect.xml +++ b/man/systemd-dissect.xml @@ -34,6 +34,9 @@ systemd-dissect OPTIONS IMAGE + + systemd-dissect OPTIONS IMAGE + systemd-dissect OPTIONS IMAGE COMMAND @@ -163,6 +166,23 @@ standard output. + + + + + Generates a BSD mtree8 + compatible file manifest of the specified disk image. This is useful for comparing disk image + contents in detail, including inode information and other metadata. While the generated manifest will + contain detailed inode information, it currently excludes extended attributes, file system + capabilities, MAC labels, chattr1 file + flags, btrfs subvolume information, and various other file metadata. File content information is + shown via a SHA256 digest. Additional fields might be added in future. Note that inode information + such as link counts, inode numbers and timestamps is excluded from the output on purpose, as it + typically complicates reproducibility. + + From baf6c9329622f95b1942bb149d5dbc7ff1bf1ef3 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 9 Nov 2022 12:31:36 +0100 Subject: [PATCH 4/4] test: add trivial test for systemd-dissect --mtree --- test/units/testsuite-50.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/units/testsuite-50.sh b/test/units/testsuite-50.sh index 10634a6ba9..a5c1eaba26 100755 --- a/test/units/testsuite-50.sh +++ b/test/units/testsuite-50.sh @@ -42,6 +42,7 @@ systemd-dissect "${image}.raw" | grep -q -F "MARKER=1" systemd-dissect "${image}.raw" | grep -q -F -f <(sed 's/"//g' "$os_release") systemd-dissect --list "${image}.raw" | grep -q '^etc/os-release$' +systemd-dissect --mtree "${image}.raw" | grep -q "./usr/bin/cat type=file mode=0755 uid=0 gid=0" read -r SHA256SUM1 _ < <(systemd-dissect --copy-from "${image}.raw" etc/os-release | sha256sum) test "$SHA256SUM1" != ""