sd-journal: move source files for sd-journal to src/libsystemd/sd-journal

src/libsystemd/meson.build

@@ -1,13 +1,72 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
+sd_journal_sources = files(
+        'sd-journal/audit-type.c',
+        'sd-journal/audit-type.h',
+        'sd-journal/catalog.c',
+        'sd-journal/catalog.h',
+        'sd-journal/compress.c',
+        'sd-journal/compress.h',
+        'sd-journal/journal-def.h',
+        'sd-journal/journal-file.c',
+        'sd-journal/journal-file.h',
+        'sd-journal/journal-internal.h',
+        'sd-journal/journal-send.c',
+        'sd-journal/journal-vacuum.c',
+        'sd-journal/journal-vacuum.h',
+        'sd-journal/journal-verify.c',
+        'sd-journal/journal-verify.h',
+        'sd-journal/lookup3.c',
+        'sd-journal/lookup3.h',
+        'sd-journal/mmap-cache.c',
+        'sd-journal/mmap-cache.h',
+        'sd-journal/sd-journal.c')
+
+if conf.get('HAVE_GCRYPT') == 1
+        sd_journal_sources += files(
+                'sd-journal/fsprg.c',
+                'sd-journal/fsprg.h',
+                'sd-journal/journal-authenticate.c',
+                'sd-journal/journal-authenticate.h')
+endif
+
+audit_type_includes = [config_h,
+                       missing_audit_h,
+                       'linux/audit.h']
+if conf.get('HAVE_AUDIT') == 1
+        audit_type_includes += 'libaudit.h'
+endif
+
+generate_audit_type_list = find_program('sd-journal/generate-audit_type-list.sh')
+audit_type_list_txt = custom_target(
+        'audit_type-list.txt',
+        output : 'audit_type-list.txt',
+        command : [generate_audit_type_list, cpp] + audit_type_includes,
+        capture : true)
+
+audit_type_to_name = custom_target(
+        'audit_type-to-name.h',
+        input : ['sd-journal/audit_type-to-name.awk', audit_type_list_txt],
+        output : 'audit_type-to-name.h',
+        command : [awk, '-f', '@INPUT0@', '@INPUT1@'],
+        capture : true)
+
+sd_journal_sources += [audit_type_to_name]
+
+############################################################
+
 id128_sources = files('''
         sd-id128/id128-util.c
         sd-id128/id128-util.h
         sd-id128/sd-id128.c
 '''.split())
 
+############################################################
+
 sd_daemon_sources = files('sd-daemon/sd-daemon.c')
 
+############################################################
+
 sd_event_sources = files('''
         sd-event/event-source.h
         sd-event/event-util.c
@@ -15,8 +74,12 @@ sd_event_sources = files('''
         sd-event/sd-event.c
 '''.split())
 
+############################################################
+
 sd_login_sources = files('sd-login/sd-login.c')
 
+############################################################
+
 libsystemd_sources = files('''
         sd-bus/bus-common-errors.c
         sd-bus/bus-common-errors.h
@@ -91,7 +154,7 @@ libsystemd_sources = files('''
         sd-resolve/resolve-private.h
         sd-resolve/sd-resolve.c
         sd-utf8/sd-utf8.c
-'''.split()) + id128_sources + sd_daemon_sources + sd_event_sources + sd_login_sources
+'''.split()) + sd_journal_sources + id128_sources + sd_daemon_sources + sd_event_sources + sd_login_sources
 
 disable_mempool_c = files('disable-mempool.c')
 

src/libsystemd/sd-journal/audit-type.c

@@ -0,0 +1,6 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include "audit-type.h"
+#include "missing_audit.h"
+
+#include "audit_type-to-name.h"

src/libsystemd/sd-journal/audit-type.h

@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include <stdio.h>
+
+#include "alloc-util.h"
+#include "macro.h"
+
+const char *audit_type_to_string(int type);
+int audit_type_from_string(const char *s);
+
+/* This is inspired by DNS TYPEnnn formatting */
+#define audit_type_name_alloca(type)                                    \
+        ({                                                              \
+                const char *_s_;                                        \
+                _s_ = audit_type_to_string(type);                       \
+                if (!_s_) {                                             \
+                        _s_ = newa(char, STRLEN("AUDIT") + DECIMAL_STR_MAX(int)); \
+                        sprintf((char*) _s_, "AUDIT%04i", type);        \
+                }                                                       \
+                _s_;                                                    \
+        })

src/libsystemd/sd-journal/audit_type-to-name.awk

@@ -0,0 +1,9 @@
+BEGIN{
+        print "const char *audit_type_to_string(int type) {\n\tswitch(type) {"
+}
+{
+        printf "        case AUDIT_%s: return \"%s\";\n", $1, $1
+}
+END{
+        print "        default: return NULL;\n\t}\n}\n"
+}

src/libsystemd/sd-journal/catalog.c

@@ -0,0 +1,742 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <locale.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "sd-id128.h"
+
+#include "alloc-util.h"
+#include "catalog.h"
+#include "conf-files.h"
+#include "fd-util.h"
+#include "fileio.h"
+#include "hashmap.h"
+#include "log.h"
+#include "memory-util.h"
+#include "mkdir.h"
+#include "path-util.h"
+#include "siphash24.h"
+#include "sort-util.h"
+#include "sparse-endian.h"
+#include "strbuf.h"
+#include "string-util.h"
+#include "strv.h"
+#include "tmpfile-util.h"
+
+const char * const catalog_file_dirs[] = {
+        "/usr/local/lib/systemd/catalog/",
+        "/usr/lib/systemd/catalog/",
+        NULL
+};
+
+#define CATALOG_SIGNATURE { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
+
+typedef struct CatalogHeader {
+        uint8_t signature[8];  /* "RHHHKSLP" */
+        le32_t compatible_flags;
+        le32_t incompatible_flags;
+        le64_t header_size;
+        le64_t n_items;
+        le64_t catalog_item_size;
+} CatalogHeader;
+
+typedef struct CatalogItem {
+        sd_id128_t id;
+        char language[32]; /* One byte is used for termination, so the maximum allowed
+                            * length of the string is actually 31 bytes. */
+        le64_t offset;
+} CatalogItem;
+
+static void catalog_hash_func(const CatalogItem *i, struct siphash *state) {
+        siphash24_compress(&i->id, sizeof(i->id), state);
+        siphash24_compress_string(i->language, state);
+}
+
+static int catalog_compare_func(const CatalogItem *a, const CatalogItem *b) {
+        unsigned k;
+        int r;
+
+        for (k = 0; k < ELEMENTSOF(b->id.bytes); k++) {
+                r = CMP(a->id.bytes[k], b->id.bytes[k]);
+                if (r != 0)
+                        return r;
+        }
+
+        return strcmp(a->language, b->language);
+}
+
+DEFINE_HASH_OPS(catalog_hash_ops, CatalogItem, catalog_hash_func, catalog_compare_func);
+
+static bool next_header(const char **s) {
+        const char *e;
+
+        e = strchr(*s, '\n');
+
+        /* Unexpected end */
+        if (!e)
+                return false;
+
+        /* End of headers */
+        if (e == *s)
+                return false;
+
+        *s = e + 1;
+        return true;
+}
+
+static const char *skip_header(const char *s) {
+        while (next_header(&s))
+                ;
+        return s;
+}
+
+static char *combine_entries(const char *one, const char *two) {
+        const char *b1, *b2;
+        size_t l1, l2, n;
+        char *dest, *p;
+
+        /* Find split point of headers to body */
+        b1 = skip_header(one);
+        b2 = skip_header(two);
+
+        l1 = strlen(one);
+        l2 = strlen(two);
+        dest = new(char, l1 + l2 + 1);
+        if (!dest) {
+                log_oom();
+                return NULL;
+        }
+
+        p = dest;
+
+        /* Headers from @one */
+        n = b1 - one;
+        p = mempcpy(p, one, n);
+
+        /* Headers from @two, these will only be found if not present above */
+        n = b2 - two;
+        p = mempcpy(p, two, n);
+
+        /* Body from @one */
+        n = l1 - (b1 - one);
+        if (n > 0) {
+                memcpy(p, b1, n);
+                p += n;
+
+        /* Body from @two */
+        } else {
+                n = l2 - (b2 - two);
+                memcpy(p, b2, n);
+                p += n;
+        }
+
+        assert(p - dest <= (ptrdiff_t)(l1 + l2));
+        p[0] = '\0';
+        return dest;
+}
+
+static int finish_item(
+                OrderedHashmap *h,
+                sd_id128_t id,
+                const char *language,
+                char *payload, size_t payload_size) {
+
+        _cleanup_free_ CatalogItem *i = NULL;
+        _cleanup_free_ char *prev = NULL, *combined = NULL;
+
+        assert(h);
+        assert(payload);
+        assert(payload_size > 0);
+
+        i = new0(CatalogItem, 1);
+        if (!i)
+                return log_oom();
+
+        i->id = id;
+        if (language) {
+                assert(strlen(language) > 1 && strlen(language) < 32);
+                strcpy(i->language, language);
+        }
+
+        prev = ordered_hashmap_get(h, i);
+        if (prev) {
+                /* Already have such an item, combine them */
+                combined = combine_entries(payload, prev);
+                if (!combined)
+                        return log_oom();
+
+                if (ordered_hashmap_update(h, i, combined) < 0)
+                        return log_oom();
+                combined = NULL;
+        } else {
+                /* A new item */
+                combined = memdup(payload, payload_size + 1);
+                if (!combined)
+                        return log_oom();
+
+                if (ordered_hashmap_put(h, i, combined) < 0)
+                        return log_oom();
+                i = NULL;
+                combined = NULL;
+        }
+
+        return 0;
+}
+
+int catalog_file_lang(const char* filename, char **lang) {
+        char *beg, *end, *_lang;
+
+        end = endswith(filename, ".catalog");
+        if (!end)
+                return 0;
+
+        beg = end - 1;
+        while (beg > filename && !IN_SET(*beg, '.', '/') && end - beg < 32)
+                beg--;
+
+        if (*beg != '.' || end <= beg + 1)
+                return 0;
+
+        _lang = strndup(beg + 1, end - beg - 1);
+        if (!_lang)
+                return -ENOMEM;
+
+        *lang = _lang;
+        return 1;
+}
+
+static int catalog_entry_lang(
+                const char* filename,
+                unsigned line,
+                const char* t,
+                const char* deflang,
+                char **ret) {
+
+        size_t c;
+        char *z;
+
+        c = strlen(t);
+        if (c < 2)
+                return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                                       "[%s:%u] Language too short.", filename, line);
+        if (c > 31)
+                return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                                       "[%s:%u] language too long.", filename, line);
+
+        if (deflang) {
+                if (streq(t, deflang)) {
+                        log_warning("[%s:%u] language specified unnecessarily", filename, line);
+                        return 0;
+                }
+
+                log_warning("[%s:%u] language differs from default for file", filename, line);
+        }
+
+        z = strdup(t);
+        if (!z)
+                return -ENOMEM;
+
+        *ret = z;
+        return 0;
+}
+
+int catalog_import_file(OrderedHashmap *h, const char *path) {
+        _cleanup_fclose_ FILE *f = NULL;
+        _cleanup_free_ char *payload = NULL;
+        size_t payload_size = 0, payload_allocated = 0;
+        unsigned n = 0;
+        sd_id128_t id;
+        _cleanup_free_ char *deflang = NULL, *lang = NULL;
+        bool got_id = false, empty_line = true;
+        int r;
+
+        assert(h);
+        assert(path);
+
+        f = fopen(path, "re");
+        if (!f)
+                return log_error_errno(errno, "Failed to open file %s: %m", path);
+
+        r = catalog_file_lang(path, &deflang);
+        if (r < 0)
+                log_error_errno(r, "Failed to determine language for file %s: %m", path);
+        if (r == 1)
+                log_debug("File %s has language %s.", path, deflang);
+
+        for (;;) {
+                _cleanup_free_ char *line = NULL;
+                size_t line_len;
+
+                r = read_line(f, LONG_LINE_MAX, &line);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to read file %s: %m", path);
+                if (r == 0)
+                        break;
+
+                n++;
+
+                if (isempty(line)) {
+                        empty_line = true;
+                        continue;
+                }
+
+                if (strchr(COMMENTS, line[0]))
+                        continue;
+
+                if (empty_line &&
+                    strlen(line) >= 2+1+32 &&
+                    line[0] == '-' &&
+                    line[1] == '-' &&
+                    line[2] == ' ' &&
+                    IN_SET(line[2+1+32], ' ', '\0')) {
+
+                        bool with_language;
+                        sd_id128_t jd;
+
+                        /* New entry */
+
+                        with_language = line[2+1+32] != '\0';
+                        line[2+1+32] = '\0';
+
+                        if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
+
+                                if (got_id) {
+                                        if (payload_size == 0)
+                                                return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                                                                       "[%s:%u] No payload text.",
+                                                                       path,
+                                                                       n);
+
+                                        r = finish_item(h, id, lang ?: deflang, payload, payload_size);
+                                        if (r < 0)
+                                                return r;
+
+                                        lang = mfree(lang);
+                                        payload_size = 0;
+                                }
+
+                                if (with_language) {
+                                        char *t;
+
+                                        t = strstrip(line + 2 + 1 + 32 + 1);
+                                        r = catalog_entry_lang(path, n, t, deflang, &lang);
+                                        if (r < 0)
+                                                return r;
+                                }
+
+                                got_id = true;
+                                empty_line = false;
+                                id = jd;
+
+                                continue;
+                        }
+                }
+
+                /* Payload */
+                if (!got_id)
+                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                                               "[%s:%u] Got payload before ID.",
+                                               path, n);
+
+                line_len = strlen(line);
+                if (!GREEDY_REALLOC(payload, payload_allocated,
+                                    payload_size + (empty_line ? 1 : 0) + line_len + 1 + 1))
+                        return log_oom();
+
+                if (empty_line)
+                        payload[payload_size++] = '\n';
+                memcpy(payload + payload_size, line, line_len);
+                payload_size += line_len;
+                payload[payload_size++] = '\n';
+                payload[payload_size] = '\0';
+
+                empty_line = false;
+        }
+
+        if (got_id) {
+                if (payload_size == 0)
+                        return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
+                                               "[%s:%u] No payload text.",
+                                               path, n);
+
+                r = finish_item(h, id, lang ?: deflang, payload, payload_size);
+                if (r < 0)
+                        return r;
+        }
+
+        return 0;
+}
+
+static int64_t write_catalog(
+                const char *database,
+                struct strbuf *sb,
+                CatalogItem *items,
+                size_t n) {
+
+        _cleanup_fclose_ FILE *w = NULL;
+        _cleanup_free_ char *p = NULL;
+        CatalogHeader header;
+        size_t k;
+        int r;
+
+        r = mkdir_parents(database, 0755);
+        if (r < 0)
+                return log_error_errno(r, "Failed to create parent directories of %s: %m", database);
+
+        r = fopen_temporary(database, &w, &p);
+        if (r < 0)
+                return log_error_errno(r, "Failed to open database for writing: %s: %m",
+                                       database);
+
+        header = (CatalogHeader) {
+                .signature = CATALOG_SIGNATURE,
+                .header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8)),
+                .catalog_item_size = htole64(sizeof(CatalogItem)),
+                .n_items = htole64(n),
+        };
+
+        r = -EIO;
+
+        k = fwrite(&header, 1, sizeof(header), w);
+        if (k != sizeof(header)) {
+                log_error("%s: failed to write header.", p);
+                goto error;
+        }
+
+        k = fwrite(items, 1, n * sizeof(CatalogItem), w);
+        if (k != n * sizeof(CatalogItem)) {
+                log_error("%s: failed to write database.", p);
+                goto error;
+        }
+
+        k = fwrite(sb->buf, 1, sb->len, w);
+        if (k != sb->len) {
+                log_error("%s: failed to write strings.", p);
+                goto error;
+        }
+
+        r = fflush_and_check(w);
+        if (r < 0) {
+                log_error_errno(r, "%s: failed to write database: %m", p);
+                goto error;
+        }
+
+        (void) fchmod(fileno(w), 0644);
+
+        if (rename(p, database) < 0) {
+                r = log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
+                goto error;
+        }
+
+        return ftello(w);
+
+error:
+        (void) unlink(p);
+        return r;
+}
+
+int catalog_update(const char* database, const char* root, const char* const* dirs) {
+        _cleanup_strv_free_ char **files = NULL;
+        char **f;
+        _cleanup_(strbuf_cleanupp) struct strbuf *sb = NULL;
+        _cleanup_ordered_hashmap_free_free_free_ OrderedHashmap *h = NULL;
+        _cleanup_free_ CatalogItem *items = NULL;
+        ssize_t offset;
+        char *payload;
+        CatalogItem *i;
+        unsigned n;
+        int r;
+        int64_t sz;
+
+        h = ordered_hashmap_new(&catalog_hash_ops);
+        sb = strbuf_new();
+        if (!h || !sb)
+                return log_oom();
+
+        r = conf_files_list_strv(&files, ".catalog", root, 0, dirs);
+        if (r < 0)
+                return log_error_errno(r, "Failed to get catalog files: %m");
+
+        STRV_FOREACH(f, files) {
+                log_debug("Reading file '%s'", *f);
+                r = catalog_import_file(h, *f);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to import file '%s': %m", *f);
+        }
+
+        if (ordered_hashmap_size(h) <= 0) {
+                log_info("No items in catalog.");
+                return 0;
+        } else
+                log_debug("Found %u items in catalog.", ordered_hashmap_size(h));
+
+        items = new(CatalogItem, ordered_hashmap_size(h));
+        if (!items)
+                return log_oom();
+
+        n = 0;
+        ORDERED_HASHMAP_FOREACH_KEY(payload, i, h) {
+                log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
+                          SD_ID128_FORMAT_VAL(i->id),
+                          isempty(i->language) ? "C" : i->language);
+
+                offset = strbuf_add_string(sb, payload, strlen(payload));
+                if (offset < 0)
+                        return log_oom();
+
+                i->offset = htole64((uint64_t) offset);
+                items[n++] = *i;
+        }
+
+        assert(n == ordered_hashmap_size(h));
+        typesafe_qsort(items, n, catalog_compare_func);
+
+        strbuf_complete(sb);
+
+        sz = write_catalog(database, sb, items, n);
+        if (sz < 0)
+                return log_error_errno(sz, "Failed to write %s: %m", database);
+
+        log_debug("%s: wrote %u items, with %zu bytes of strings, %"PRIi64" total size.",
+                  database, n, sb->len, sz);
+        return 0;
+}
+
+static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
+        _cleanup_close_ int fd = -1;
+        const CatalogHeader *h;
+        struct stat st;
+        void *p;
+
+        assert(_fd);
+        assert(_st);
+        assert(_p);
+
+        fd = open(database, O_RDONLY|O_CLOEXEC);
+        if (fd < 0)
+                return -errno;
+
+        if (fstat(fd, &st) < 0)
+                return -errno;
+
+        if (st.st_size < (off_t) sizeof(CatalogHeader))
+                return -EINVAL;
+
+        p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
+        if (p == MAP_FAILED)
+                return -errno;
+
+        h = p;
+        if (memcmp(h->signature, (const uint8_t[]) CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
+            le64toh(h->header_size) < sizeof(CatalogHeader) ||
+            le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
+            h->incompatible_flags != 0 ||
+            le64toh(h->n_items) <= 0 ||
+            st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
+                munmap(p, st.st_size);
+                return -EBADMSG;
+        }
+
+        *_fd = TAKE_FD(fd);
+        *_st = st;
+        *_p = p;
+
+        return 0;
+}
+
+static const char *find_id(void *p, sd_id128_t id) {
+        CatalogItem *f = NULL, key = { .id = id };
+        const CatalogHeader *h = p;
+        const char *loc;
+
+        loc = setlocale(LC_MESSAGES, NULL);
+        if (!isempty(loc) && !STR_IN_SET(loc, "C", "POSIX")) {
+                size_t len;
+
+                len = strcspn(loc, ".@");
+                if (len > sizeof(key.language) - 1)
+                        log_debug("LC_MESSAGES value too long, ignoring: \"%.*s\"", (int) len, loc);
+                else {
+                        strncpy(key.language, loc, len);
+                        key.language[len] = '\0';
+
+                        f = bsearch(&key,
+                                    (const uint8_t*) p + le64toh(h->header_size),
+                                    le64toh(h->n_items),
+                                    le64toh(h->catalog_item_size),
+                                    (comparison_fn_t) catalog_compare_func);
+                        if (!f) {
+                                char *e;
+
+                                e = strchr(key.language, '_');
+                                if (e) {
+                                        *e = 0;
+                                        f = bsearch(&key,
+                                                    (const uint8_t*) p + le64toh(h->header_size),
+                                                    le64toh(h->n_items),
+                                                    le64toh(h->catalog_item_size),
+                                                    (comparison_fn_t) catalog_compare_func);
+                                }
+                        }
+                }
+        }
+
+        if (!f) {
+                zero(key.language);
+                f = bsearch(&key,
+                            (const uint8_t*) p + le64toh(h->header_size),
+                            le64toh(h->n_items),
+                            le64toh(h->catalog_item_size),
+                            (comparison_fn_t) catalog_compare_func);
+        }
+
+        if (!f)
+                return NULL;
+
+        return (const char*) p +
+                le64toh(h->header_size) +
+                le64toh(h->n_items) * le64toh(h->catalog_item_size) +
+                le64toh(f->offset);
+}
+
+int catalog_get(const char* database, sd_id128_t id, char **_text) {
+        _cleanup_close_ int fd = -1;
+        void *p = NULL;
+        struct stat st = {};
+        char *text = NULL;
+        int r;
+        const char *s;
+
+        assert(_text);
+
+        r = open_mmap(database, &fd, &st, &p);
+        if (r < 0)
+                return r;
+
+        s = find_id(p, id);
+        if (!s) {
+                r = -ENOENT;
+                goto finish;
+        }
+
+        text = strdup(s);
+        if (!text) {
+                r = -ENOMEM;
+                goto finish;
+        }
+
+        *_text = text;
+        r = 0;
+
+finish:
+        if (p)
+                munmap(p, st.st_size);
+
+        return r;
+}
+
+static char *find_header(const char *s, const char *header) {
+
+        for (;;) {
+                const char *v;
+
+                v = startswith(s, header);
+                if (v) {
+                        v += strspn(v, WHITESPACE);
+                        return strndup(v, strcspn(v, NEWLINE));
+                }
+
+                if (!next_header(&s))
+                        return NULL;
+        }
+}
+
+static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
+        if (oneline) {
+                _cleanup_free_ char *subject = NULL, *defined_by = NULL;
+
+                subject = find_header(s, "Subject:");
+                defined_by = find_header(s, "Defined-By:");
+
+                fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
+                        SD_ID128_FORMAT_VAL(id),
+                        strna(defined_by), strna(subject));
+        } else
+                fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
+                        SD_ID128_FORMAT_VAL(id), s);
+}
+
+int catalog_list(FILE *f, const char *database, bool oneline) {
+        _cleanup_close_ int fd = -1;
+        void *p = NULL;
+        struct stat st;
+        const CatalogHeader *h;
+        const CatalogItem *items;
+        int r;
+        unsigned n;
+        sd_id128_t last_id;
+        bool last_id_set = false;
+
+        r = open_mmap(database, &fd, &st, &p);
+        if (r < 0)
+                return r;
+
+        h = p;
+        items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
+
+        for (n = 0; n < le64toh(h->n_items); n++) {
+                const char *s;
+
+                if (last_id_set && sd_id128_equal(last_id, items[n].id))
+                        continue;
+
+                assert_se(s = find_id(p, items[n].id));
+
+                dump_catalog_entry(f, items[n].id, s, oneline);
+
+                last_id_set = true;
+                last_id = items[n].id;
+        }
+
+        munmap(p, st.st_size);
+
+        return 0;
+}
+
+int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
+        char **item;
+        int r = 0;
+
+        STRV_FOREACH(item, items) {
+                sd_id128_t id;
+                int k;
+                _cleanup_free_ char *msg = NULL;
+
+                k = sd_id128_from_string(*item, &id);
+                if (k < 0) {
+                        log_error_errno(k, "Failed to parse id128 '%s': %m", *item);
+                        if (r == 0)
+                                r = k;
+                        continue;
+                }
+
+                k = catalog_get(database, id, &msg);
+                if (k < 0) {
+                        log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k,
+                                       "Failed to retrieve catalog entry for '%s': %m", *item);
+                        if (r == 0)
+                                r = k;
+                        continue;
+                }
+
+                dump_catalog_entry(f, id, msg, oneline);
+        }
+
+        return r;
+}

src/libsystemd/sd-journal/catalog.h

@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include <stdbool.h>
+#include <stdio.h>
+
+#include "sd-id128.h"
+
+#include "hashmap.h"
+#include "strbuf.h"
+
+int catalog_import_file(OrderedHashmap *h, const char *path);
+int catalog_update(const char* database, const char* root, const char* const* dirs);
+int catalog_get(const char* database, sd_id128_t id, char **data);
+int catalog_list(FILE *f, const char* database, bool oneline);
+int catalog_list_items(FILE *f, const char* database, bool oneline, char **items);
+int catalog_file_lang(const char *filename, char **lang);
+extern const char * const catalog_file_dirs[];
+extern const struct hash_ops catalog_hash_ops;

src/libsystemd/sd-journal/compress.h

@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include <unistd.h>
+
+#include "journal-def.h"
+
+const char* object_compressed_to_string(int compression);
+int object_compressed_from_string(const char *compression);
+
+int compress_blob_xz(const void *src, uint64_t src_size,
+                     void *dst, size_t dst_alloc_size, size_t *dst_size);
+int compress_blob_lz4(const void *src, uint64_t src_size,
+                      void *dst, size_t dst_alloc_size, size_t *dst_size);
+int compress_blob_zstd(const void *src, uint64_t src_size,
+                       void *dst, size_t dst_alloc_size, size_t *dst_size);
+
+static inline int compress_blob(const void *src, uint64_t src_size,
+                                void *dst, size_t dst_alloc_size, size_t *dst_size) {
+        int r;
+#if HAVE_ZSTD
+        r = compress_blob_zstd(src, src_size, dst, dst_alloc_size, dst_size);
+        if (r == 0)
+                return OBJECT_COMPRESSED_ZSTD;
+#elif HAVE_LZ4
+        r = compress_blob_lz4(src, src_size, dst, dst_alloc_size, dst_size);
+        if (r == 0)
+                return OBJECT_COMPRESSED_LZ4;
+#elif HAVE_XZ
+        r = compress_blob_xz(src, src_size, dst, dst_alloc_size, dst_size);
+        if (r == 0)
+                return OBJECT_COMPRESSED_XZ;
+#else
+        r = -EOPNOTSUPP;
+#endif
+        return r;
+}
+
+int decompress_blob_xz(const void *src, uint64_t src_size,
+                       void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max);
+int decompress_blob_lz4(const void *src, uint64_t src_size,
+                        void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max);
+int decompress_blob_zstd(const void *src, uint64_t src_size,
+                        void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max);
+int decompress_blob(int compression,
+                    const void *src, uint64_t src_size,
+                    void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max);
+
+int decompress_startswith_xz(const void *src, uint64_t src_size,
+                             void **buffer, size_t *buffer_size,
+                             const void *prefix, size_t prefix_len,
+                             uint8_t extra);
+int decompress_startswith_lz4(const void *src, uint64_t src_size,
+                              void **buffer, size_t *buffer_size,
+                              const void *prefix, size_t prefix_len,
+                              uint8_t extra);
+int decompress_startswith_zstd(const void *src, uint64_t src_size,
+                               void **buffer, size_t *buffer_size,
+                               const void *prefix, size_t prefix_len,
+                               uint8_t extra);
+int decompress_startswith(int compression,
+                          const void *src, uint64_t src_size,
+                          void **buffer, size_t *buffer_size,
+                          const void *prefix, size_t prefix_len,
+                          uint8_t extra);
+
+int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes);
+int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes);
+int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes);
+
+int decompress_stream_xz(int fdf, int fdt, uint64_t max_size);
+int decompress_stream_lz4(int fdf, int fdt, uint64_t max_size);
+int decompress_stream_zstd(int fdf, int fdt, uint64_t max_size);
+
+#if HAVE_ZSTD
+#  define compress_stream compress_stream_zstd
+#  define COMPRESSED_EXT ".zst"
+#elif HAVE_LZ4
+#  define compress_stream compress_stream_lz4
+#  define COMPRESSED_EXT ".lz4"
+#elif HAVE_XZ
+#  define compress_stream compress_stream_xz
+#  define COMPRESSED_EXT ".xz"
+#else
+static inline int compress_stream(int fdf, int fdt, uint64_t max_size) {
+        return -EOPNOTSUPP;
+}
+#  define COMPRESSED_EXT ""
+#endif
+
+int decompress_stream(const char *filename, int fdf, int fdt, uint64_t max_bytes);

src/libsystemd/sd-journal/fsprg.c

@@ -0,0 +1,378 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later
+ *
+ * fsprg v0.1  -  (seekable) forward-secure pseudorandom generator
+ * Copyright © 2012 B. Poettering
+ * Contact: fsprg@point-at-infinity.org
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+ */
+
+/*
+ * See "Practical Secure Logging: Seekable Sequential Key Generators"
+ * by G. A. Marson, B. Poettering for details:
+ *
+ * http://eprint.iacr.org/2013/397
+ */
+
+#include <string.h>
+
+#include "fsprg.h"
+#include "gcrypt-util.h"
+#include "memory-util.h"
+
+#define ISVALID_SECPAR(secpar) (((secpar) % 16 == 0) && ((secpar) >= 16) && ((secpar) <= 16384))
+#define VALIDATE_SECPAR(secpar) assert(ISVALID_SECPAR(secpar));
+
+#define RND_HASH GCRY_MD_SHA256
+#define RND_GEN_P 0x01
+#define RND_GEN_Q 0x02
+#define RND_GEN_X 0x03
+
+#pragma GCC diagnostic ignored "-Wpointer-arith"
+/* TODO: remove void* arithmetic and this work-around */
+
+/******************************************************************************/
+
+static void mpi_export(void *buf, size_t buflen, const gcry_mpi_t x) {
+        unsigned len;
+        size_t nwritten;
+
+        assert(gcry_mpi_cmp_ui(x, 0) >= 0);
+        len = (gcry_mpi_get_nbits(x) + 7) / 8;
+        assert(len <= buflen);
+        memzero(buf, buflen);
+        gcry_mpi_print(GCRYMPI_FMT_USG, buf + (buflen - len), len, &nwritten, x);
+        assert(nwritten == len);
+}
+
+static gcry_mpi_t mpi_import(const void *buf, size_t buflen) {
+        gcry_mpi_t h;
+        _unused_ unsigned len;
+
+        assert_se(gcry_mpi_scan(&h, GCRYMPI_FMT_USG, buf, buflen, NULL) == 0);
+        len = (gcry_mpi_get_nbits(h) + 7) / 8;
+        assert(len <= buflen);
+        assert(gcry_mpi_cmp_ui(h, 0) >= 0);
+
+        return h;
+}
+
+static void uint64_export(void *buf, size_t buflen, uint64_t x) {
+        assert(buflen == 8);
+        ((uint8_t*) buf)[0] = (x >> 56) & 0xff;
+        ((uint8_t*) buf)[1] = (x >> 48) & 0xff;
+        ((uint8_t*) buf)[2] = (x >> 40) & 0xff;
+        ((uint8_t*) buf)[3] = (x >> 32) & 0xff;
+        ((uint8_t*) buf)[4] = (x >> 24) & 0xff;
+        ((uint8_t*) buf)[5] = (x >> 16) & 0xff;
+        ((uint8_t*) buf)[6] = (x >>  8) & 0xff;
+        ((uint8_t*) buf)[7] = (x >>  0) & 0xff;
+}
+
+_pure_ static uint64_t uint64_import(const void *buf, size_t buflen) {
+        assert(buflen == 8);
+        return
+                (uint64_t)(((uint8_t*) buf)[0]) << 56 |
+                (uint64_t)(((uint8_t*) buf)[1]) << 48 |
+                (uint64_t)(((uint8_t*) buf)[2]) << 40 |
+                (uint64_t)(((uint8_t*) buf)[3]) << 32 |
+                (uint64_t)(((uint8_t*) buf)[4]) << 24 |
+                (uint64_t)(((uint8_t*) buf)[5]) << 16 |
+                (uint64_t)(((uint8_t*) buf)[6]) <<  8 |
+                (uint64_t)(((uint8_t*) buf)[7]) <<  0;
+}
+
+/* deterministically generate from seed/idx a string of buflen pseudorandom bytes */
+static void det_randomize(void *buf, size_t buflen, const void *seed, size_t seedlen, uint32_t idx) {
+        gcry_md_hd_t hd, hd2;
+        size_t olen, cpylen;
+        uint32_t ctr;
+
+        olen = gcry_md_get_algo_dlen(RND_HASH);
+        gcry_md_open(&hd, RND_HASH, 0);
+        gcry_md_write(hd, seed, seedlen);
+        gcry_md_putc(hd, (idx >> 24) & 0xff);
+        gcry_md_putc(hd, (idx >> 16) & 0xff);
+        gcry_md_putc(hd, (idx >>  8) & 0xff);
+        gcry_md_putc(hd, (idx >>  0) & 0xff);
+
+        for (ctr = 0; buflen; ctr++) {
+                gcry_md_copy(&hd2, hd);
+                gcry_md_putc(hd2, (ctr >> 24) & 0xff);
+                gcry_md_putc(hd2, (ctr >> 16) & 0xff);
+                gcry_md_putc(hd2, (ctr >>  8) & 0xff);
+                gcry_md_putc(hd2, (ctr >>  0) & 0xff);
+                gcry_md_final(hd2);
+                cpylen = (buflen < olen) ? buflen : olen;
+                memcpy(buf, gcry_md_read(hd2, RND_HASH), cpylen);
+                gcry_md_close(hd2);
+                buf += cpylen;
+                buflen -= cpylen;
+        }
+        gcry_md_close(hd);
+}
+
+/* deterministically generate from seed/idx a prime of length `bits' that is 3 (mod 4) */
+static gcry_mpi_t genprime3mod4(int bits, const void *seed, size_t seedlen, uint32_t idx) {
+        size_t buflen = bits / 8;
+        uint8_t buf[buflen];
+        gcry_mpi_t p;
+
+        assert(bits % 8 == 0);
+        assert(buflen > 0);
+
+        det_randomize(buf, buflen, seed, seedlen, idx);
+        buf[0] |= 0xc0; /* set upper two bits, so that n=pq has maximum size */
+        buf[buflen - 1] |= 0x03; /* set lower two bits, to have result 3 (mod 4) */
+
+        p = mpi_import(buf, buflen);
+        while (gcry_prime_check(p, 0))
+                gcry_mpi_add_ui(p, p, 4);
+
+        return p;
+}
+
+/* deterministically generate from seed/idx a quadratic residue (mod n) */
+static gcry_mpi_t gensquare(const gcry_mpi_t n, const void *seed, size_t seedlen, uint32_t idx, unsigned secpar) {
+        size_t buflen = secpar / 8;
+        uint8_t buf[buflen];
+        gcry_mpi_t x;
+
+        det_randomize(buf, buflen, seed, seedlen, idx);
+        buf[0] &= 0x7f; /* clear upper bit, so that we have x < n */
+        x = mpi_import(buf, buflen);
+        assert(gcry_mpi_cmp(x, n) < 0);
+        gcry_mpi_mulm(x, x, x, n);
+        return x;
+}
+
+/* compute 2^m (mod phi(p)), for a prime p */
+static gcry_mpi_t twopowmodphi(uint64_t m, const gcry_mpi_t p) {
+        gcry_mpi_t phi, r;
+        int n;
+
+        phi = gcry_mpi_new(0);
+        gcry_mpi_sub_ui(phi, p, 1);
+
+        /* count number of used bits in m */
+        for (n = 0; (1ULL << n) <= m; n++)
+                ;
+
+        r = gcry_mpi_new(0);
+        gcry_mpi_set_ui(r, 1);
+        while (n) { /* square and multiply algorithm for fast exponentiation */
+                n--;
+                gcry_mpi_mulm(r, r, r, phi);
+                if (m & ((uint64_t)1 << n)) {
+                        gcry_mpi_add(r, r, r);
+                        if (gcry_mpi_cmp(r, phi) >= 0)
+                                gcry_mpi_sub(r, r, phi);
+                }
+        }
+
+        gcry_mpi_release(phi);
+        return r;
+}
+
+/* Decompose $x \in Z_n$ into $(xp,xq) \in Z_p \times Z_q$ using Chinese Remainder Theorem */
+static void CRT_decompose(gcry_mpi_t *xp, gcry_mpi_t *xq, const gcry_mpi_t x, const gcry_mpi_t p, const gcry_mpi_t q) {
+        *xp = gcry_mpi_new(0);
+        *xq = gcry_mpi_new(0);
+        gcry_mpi_mod(*xp, x, p);
+        gcry_mpi_mod(*xq, x, q);
+}
+
+/* Compose $(xp,xq) \in Z_p \times Z_q$ into $x \in Z_n$ using Chinese Remainder Theorem */
+static void CRT_compose(gcry_mpi_t *x, const gcry_mpi_t xp, const gcry_mpi_t xq, const gcry_mpi_t p, const gcry_mpi_t q) {
+        gcry_mpi_t a, u;
+
+        a = gcry_mpi_new(0);
+        u = gcry_mpi_new(0);
+        *x = gcry_mpi_new(0);
+        gcry_mpi_subm(a, xq, xp, q);
+        gcry_mpi_invm(u, p, q);
+        gcry_mpi_mulm(a, a, u, q); /* a = (xq - xp) / p  (mod q) */
+        gcry_mpi_mul(*x, p, a);
+        gcry_mpi_add(*x, *x, xp); /* x = p * ((xq - xp) / p mod q) + xp */
+        gcry_mpi_release(a);
+        gcry_mpi_release(u);
+}
+
+/******************************************************************************/
+
+size_t FSPRG_mskinbytes(unsigned _secpar) {
+        VALIDATE_SECPAR(_secpar);
+        return 2 + 2 * (_secpar / 2) / 8; /* to store header,p,q */
+}
+
+size_t FSPRG_mpkinbytes(unsigned _secpar) {
+        VALIDATE_SECPAR(_secpar);
+        return 2 + _secpar / 8; /* to store header,n */
+}
+
+size_t FSPRG_stateinbytes(unsigned _secpar) {
+        VALIDATE_SECPAR(_secpar);
+        return 2 + 2 * _secpar / 8 + 8; /* to store header,n,x,epoch */
+}
+
+static void store_secpar(void *buf, uint16_t secpar) {
+        secpar = secpar / 16 - 1;
+        ((uint8_t*) buf)[0] = (secpar >> 8) & 0xff;
+        ((uint8_t*) buf)[1] = (secpar >> 0) & 0xff;
+}
+
+static uint16_t read_secpar(const void *buf) {
+        uint16_t secpar;
+        secpar =
+                (uint16_t)(((uint8_t*) buf)[0]) << 8 |
+                (uint16_t)(((uint8_t*) buf)[1]) << 0;
+        return 16 * (secpar + 1);
+}
+
+void FSPRG_GenMK(void *msk, void *mpk, const void *seed, size_t seedlen, unsigned _secpar) {
+        uint8_t iseed[FSPRG_RECOMMENDED_SEEDLEN];
+        gcry_mpi_t n, p, q;
+        uint16_t secpar;
+
+        VALIDATE_SECPAR(_secpar);
+        secpar = _secpar;
+
+        initialize_libgcrypt(false);
+
+        if (!seed) {
+                gcry_randomize(iseed, FSPRG_RECOMMENDED_SEEDLEN, GCRY_STRONG_RANDOM);
+                seed = iseed;
+                seedlen = FSPRG_RECOMMENDED_SEEDLEN;
+        }
+
+        p = genprime3mod4(secpar / 2, seed, seedlen, RND_GEN_P);
+        q = genprime3mod4(secpar / 2, seed, seedlen, RND_GEN_Q);
+
+        if (msk) {
+                store_secpar(msk + 0, secpar);
+                mpi_export(msk + 2 + 0 * (secpar / 2) / 8, (secpar / 2) / 8, p);
+                mpi_export(msk + 2 + 1 * (secpar / 2) / 8, (secpar / 2) / 8, q);
+        }
+
+        if (mpk) {
+                n = gcry_mpi_new(0);
+                gcry_mpi_mul(n, p, q);
+                assert(gcry_mpi_get_nbits(n) == secpar);
+
+                store_secpar(mpk + 0, secpar);
+                mpi_export(mpk + 2, secpar / 8, n);
+
+                gcry_mpi_release(n);
+        }
+
+        gcry_mpi_release(p);
+        gcry_mpi_release(q);
+}
+
+void FSPRG_GenState0(void *state, const void *mpk, const void *seed, size_t seedlen) {
+        gcry_mpi_t n, x;
+        uint16_t secpar;
+
+        initialize_libgcrypt(false);
+
+        secpar = read_secpar(mpk + 0);
+        n = mpi_import(mpk + 2, secpar / 8);
+        x = gensquare(n, seed, seedlen, RND_GEN_X, secpar);
+
+        memcpy(state, mpk, 2 + secpar / 8);
+        mpi_export(state + 2 + 1 * secpar / 8, secpar / 8, x);
+        memzero(state + 2 + 2 * secpar / 8, 8);
+
+        gcry_mpi_release(n);
+        gcry_mpi_release(x);
+}
+
+void FSPRG_Evolve(void *state) {
+        gcry_mpi_t n, x;
+        uint16_t secpar;
+        uint64_t epoch;
+
+        initialize_libgcrypt(false);
+
+        secpar = read_secpar(state + 0);
+        n = mpi_import(state + 2 + 0 * secpar / 8, secpar / 8);
+        x = mpi_import(state + 2 + 1 * secpar / 8, secpar / 8);
+        epoch = uint64_import(state + 2 + 2 * secpar / 8, 8);
+
+        gcry_mpi_mulm(x, x, x, n);
+        epoch++;
+
+        mpi_export(state + 2 + 1 * secpar / 8, secpar / 8, x);
+        uint64_export(state + 2 + 2 * secpar / 8, 8, epoch);
+
+        gcry_mpi_release(n);
+        gcry_mpi_release(x);
+}
+
+uint64_t FSPRG_GetEpoch(const void *state) {
+        uint16_t secpar;
+        secpar = read_secpar(state + 0);
+        return uint64_import(state + 2 + 2 * secpar / 8, 8);
+}
+
+void FSPRG_Seek(void *state, uint64_t epoch, const void *msk, const void *seed, size_t seedlen) {
+        gcry_mpi_t p, q, n, x, xp, xq, kp, kq, xm;
+        uint16_t secpar;
+
+        initialize_libgcrypt(false);
+
+        secpar = read_secpar(msk + 0);
+        p  = mpi_import(msk + 2 + 0 * (secpar / 2) / 8, (secpar / 2) / 8);
+        q  = mpi_import(msk + 2 + 1 * (secpar / 2) / 8, (secpar / 2) / 8);
+
+        n = gcry_mpi_new(0);
+        gcry_mpi_mul(n, p, q);
+
+        x = gensquare(n, seed, seedlen, RND_GEN_X, secpar);
+        CRT_decompose(&xp, &xq, x, p, q); /* split (mod n) into (mod p) and (mod q) using CRT */
+
+        kp = twopowmodphi(epoch, p); /* compute 2^epoch (mod phi(p)) */
+        kq = twopowmodphi(epoch, q); /* compute 2^epoch (mod phi(q)) */
+
+        gcry_mpi_powm(xp, xp, kp, p); /* compute x^(2^epoch) (mod p) */
+        gcry_mpi_powm(xq, xq, kq, q); /* compute x^(2^epoch) (mod q) */
+
+        CRT_compose(&xm, xp, xq, p, q); /* combine (mod p) and (mod q) to (mod n) using CRT */
+
+        store_secpar(state + 0, secpar);
+        mpi_export(state + 2 + 0 * secpar / 8, secpar / 8, n);
+        mpi_export(state + 2 + 1 * secpar / 8, secpar / 8, xm);
+        uint64_export(state + 2 + 2 * secpar / 8, 8, epoch);
+
+        gcry_mpi_release(p);
+        gcry_mpi_release(q);
+        gcry_mpi_release(n);
+        gcry_mpi_release(x);
+        gcry_mpi_release(xp);
+        gcry_mpi_release(xq);
+        gcry_mpi_release(kp);
+        gcry_mpi_release(kq);
+        gcry_mpi_release(xm);
+}
+
+void FSPRG_GetKey(const void *state, void *key, size_t keylen, uint32_t idx) {
+        uint16_t secpar;
+
+        initialize_libgcrypt(false);
+
+        secpar = read_secpar(state + 0);
+        det_randomize(key, keylen, state + 2, 2 * secpar / 8 + 8, idx);
+}

src/libsystemd/sd-journal/fsprg.h

@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+/*
+ * fsprg v0.1  -  (seekable) forward-secure pseudorandom generator
+ * Copyright © 2012 B. Poettering
+ * Contact: fsprg@point-at-infinity.org
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+ */
+
+#include <inttypes.h>
+#include <sys/types.h>
+
+#include "macro.h"
+#include "util.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define FSPRG_RECOMMENDED_SECPAR 1536
+#define FSPRG_RECOMMENDED_SEEDLEN (96/8)
+
+size_t FSPRG_mskinbytes(unsigned secpar) _const_;
+size_t FSPRG_mpkinbytes(unsigned secpar) _const_;
+size_t FSPRG_stateinbytes(unsigned secpar) _const_;
+
+/* Setup msk and mpk. Providing seed != NULL makes this algorithm deterministic. */
+void FSPRG_GenMK(void *msk, void *mpk, const void *seed, size_t seedlen, unsigned secpar);
+
+/* Initialize state deterministically in dependence on seed. */
+/* Note: in case one wants to run only one GenState0 per GenMK it is safe to use
+   the same seed for both GenMK and GenState0.
+*/
+void FSPRG_GenState0(void *state, const void *mpk, const void *seed, size_t seedlen);
+
+void FSPRG_Evolve(void *state);
+
+uint64_t FSPRG_GetEpoch(const void *state) _pure_;
+
+/* Seek to any arbitrary state (by providing msk together with seed from GenState0). */
+void FSPRG_Seek(void *state, uint64_t epoch, const void *msk, const void *seed, size_t seedlen);
+
+void FSPRG_GetKey(const void *state, void *key, size_t keylen, uint32_t idx);
+
+#ifdef __cplusplus
+}
+#endif

src/libsystemd/sd-journal/generate-audit_type-list.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -eu
+
+cpp="$1"
+shift
+
+includes=""
+for i in "$@"; do
+    includes="$includes -include $i"
+done
+
+$cpp -dM $includes - </dev/null | \
+    grep -vE 'AUDIT_.*(FIRST|LAST)_' | \
+    sed -r -n 's/^#define\s+AUDIT_(\w+)\s+([0-9]{4})\s*$$/\1\t\2/p' | \
+    sort -k2

src/libsystemd/sd-journal/journal-authenticate.c

@@ -0,0 +1,536 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <fcntl.h>
+#include <sys/mman.h>
+
+#include "fd-util.h"
+#include "fsprg.h"
+#include "gcrypt-util.h"
+#include "hexdecoct.h"
+#include "journal-authenticate.h"
+#include "journal-def.h"
+#include "journal-file.h"
+#include "memory-util.h"
+#include "time-util.h"
+
+static uint64_t journal_file_tag_seqnum(JournalFile *f) {
+        uint64_t r;
+
+        assert(f);
+
+        r = le64toh(f->header->n_tags) + 1;
+        f->header->n_tags = htole64(r);
+
+        return r;
+}
+
+int journal_file_append_tag(JournalFile *f) {
+        Object *o;
+        uint64_t p;
+        int r;
+
+        assert(f);
+
+        if (!f->seal)
+                return 0;
+
+        if (!f->hmac_running)
+                return 0;
+
+        assert(f->hmac);
+
+        r = journal_file_append_object(f, OBJECT_TAG, sizeof(struct TagObject), &o, &p);
+        if (r < 0)
+                return r;
+
+        o->tag.seqnum = htole64(journal_file_tag_seqnum(f));
+        o->tag.epoch = htole64(FSPRG_GetEpoch(f->fsprg_state));
+
+        log_debug("Writing tag %"PRIu64" for epoch %"PRIu64"",
+                  le64toh(o->tag.seqnum),
+                  FSPRG_GetEpoch(f->fsprg_state));
+
+        /* Add the tag object itself, so that we can protect its
+         * header. This will exclude the actual hash value in it */
+        r = journal_file_hmac_put_object(f, OBJECT_TAG, o, p);
+        if (r < 0)
+                return r;
+
+        /* Get the HMAC tag and store it in the object */
+        memcpy(o->tag.tag, gcry_md_read(f->hmac, 0), TAG_LENGTH);
+        f->hmac_running = false;
+
+        return 0;
+}
+
+int journal_file_hmac_start(JournalFile *f) {
+        uint8_t key[256 / 8]; /* Let's pass 256 bit from FSPRG to HMAC */
+        assert(f);
+
+        if (!f->seal)
+                return 0;
+
+        if (f->hmac_running)
+                return 0;
+
+        /* Prepare HMAC for next cycle */
+        gcry_md_reset(f->hmac);
+        FSPRG_GetKey(f->fsprg_state, key, sizeof(key), 0);
+        gcry_md_setkey(f->hmac, key, sizeof(key));
+
+        f->hmac_running = true;
+
+        return 0;
+}
+
+static int journal_file_get_epoch(JournalFile *f, uint64_t realtime, uint64_t *epoch) {
+        uint64_t t;
+
+        assert(f);
+        assert(epoch);
+        assert(f->seal);
+
+        if (f->fss_start_usec == 0 ||
+            f->fss_interval_usec == 0)
+                return -EOPNOTSUPP;
+
+        if (realtime < f->fss_start_usec)
+                return -ESTALE;
+
+        t = realtime - f->fss_start_usec;
+        t = t / f->fss_interval_usec;
+
+        *epoch = t;
+        return 0;
+}
+
+static int journal_file_fsprg_need_evolve(JournalFile *f, uint64_t realtime) {
+        uint64_t goal, epoch;
+        int r;
+        assert(f);
+
+        if (!f->seal)
+                return 0;
+
+        r = journal_file_get_epoch(f, realtime, &goal);
+        if (r < 0)
+                return r;
+
+        epoch = FSPRG_GetEpoch(f->fsprg_state);
+        if (epoch > goal)
+                return -ESTALE;
+
+        return epoch != goal;
+}
+
+int journal_file_fsprg_evolve(JournalFile *f, uint64_t realtime) {
+        uint64_t goal, epoch;
+        int r;
+
+        assert(f);
+
+        if (!f->seal)
+                return 0;
+
+        r = journal_file_get_epoch(f, realtime, &goal);
+        if (r < 0)
+                return r;
+
+        epoch = FSPRG_GetEpoch(f->fsprg_state);
+        if (epoch < goal)
+                log_debug("Evolving FSPRG key from epoch %"PRIu64" to %"PRIu64".", epoch, goal);
+
+        for (;;) {
+                if (epoch > goal)
+                        return -ESTALE;
+                if (epoch == goal)
+                        return 0;
+
+                FSPRG_Evolve(f->fsprg_state);
+                epoch = FSPRG_GetEpoch(f->fsprg_state);
+        }
+}
+
+int journal_file_fsprg_seek(JournalFile *f, uint64_t goal) {
+        void *msk;
+        uint64_t epoch;
+
+        assert(f);
+
+        if (!f->seal)
+                return 0;
+
+        assert(f->fsprg_seed);
+
+        if (f->fsprg_state) {
+                /* Cheaper... */
+
+                epoch = FSPRG_GetEpoch(f->fsprg_state);
+                if (goal == epoch)
+                        return 0;
+
+                if (goal == epoch+1) {
+                        FSPRG_Evolve(f->fsprg_state);
+                        return 0;
+                }
+        } else {
+                f->fsprg_state_size = FSPRG_stateinbytes(FSPRG_RECOMMENDED_SECPAR);
+                f->fsprg_state = malloc(f->fsprg_state_size);
+
+                if (!f->fsprg_state)
+                        return -ENOMEM;
+        }
+
+        log_debug("Seeking FSPRG key to %"PRIu64".", goal);
+
+        msk = alloca(FSPRG_mskinbytes(FSPRG_RECOMMENDED_SECPAR));
+        FSPRG_GenMK(msk, NULL, f->fsprg_seed, f->fsprg_seed_size, FSPRG_RECOMMENDED_SECPAR);
+        FSPRG_Seek(f->fsprg_state, goal, msk, f->fsprg_seed, f->fsprg_seed_size);
+        return 0;
+}
+
+int journal_file_maybe_append_tag(JournalFile *f, uint64_t realtime) {
+        int r;
+
+        assert(f);
+
+        if (!f->seal)
+                return 0;
+
+        if (realtime <= 0)
+                realtime = now(CLOCK_REALTIME);
+
+        r = journal_file_fsprg_need_evolve(f, realtime);
+        if (r <= 0)
+                return 0;
+
+        r = journal_file_append_tag(f);
+        if (r < 0)
+                return r;
+
+        r = journal_file_fsprg_evolve(f, realtime);
+        if (r < 0)
+                return r;
+
+        return 0;
+}
+
+int journal_file_hmac_put_object(JournalFile *f, ObjectType type, Object *o, uint64_t p) {
+        int r;
+
+        assert(f);
+
+        if (!f->seal)
+                return 0;
+
+        r = journal_file_hmac_start(f);
+        if (r < 0)
+                return r;
+
+        if (!o) {
+                r = journal_file_move_to_object(f, type, p, &o);
+                if (r < 0)
+                        return r;
+        } else {
+                if (type > OBJECT_UNUSED && o->object.type != type)
+                        return -EBADMSG;
+        }
+
+        gcry_md_write(f->hmac, o, offsetof(ObjectHeader, payload));
+
+        switch (o->object.type) {
+
+        case OBJECT_DATA:
+                /* All but hash and payload are mutable */
+                gcry_md_write(f->hmac, &o->data.hash, sizeof(o->data.hash));
+                gcry_md_write(f->hmac, o->data.payload, le64toh(o->object.size) - offsetof(DataObject, payload));
+                break;
+
+        case OBJECT_FIELD:
+                /* Same here */
+                gcry_md_write(f->hmac, &o->field.hash, sizeof(o->field.hash));
+                gcry_md_write(f->hmac, o->field.payload, le64toh(o->object.size) - offsetof(FieldObject, payload));
+                break;
+
+        case OBJECT_ENTRY:
+                /* All */
+                gcry_md_write(f->hmac, &o->entry.seqnum, le64toh(o->object.size) - offsetof(EntryObject, seqnum));
+                break;
+
+        case OBJECT_FIELD_HASH_TABLE:
+        case OBJECT_DATA_HASH_TABLE:
+        case OBJECT_ENTRY_ARRAY:
+                /* Nothing: everything is mutable */
+                break;
+
+        case OBJECT_TAG:
+                /* All but the tag itself */
+                gcry_md_write(f->hmac, &o->tag.seqnum, sizeof(o->tag.seqnum));
+                gcry_md_write(f->hmac, &o->tag.epoch, sizeof(o->tag.epoch));
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+int journal_file_hmac_put_header(JournalFile *f) {
+        int r;
+
+        assert(f);
+
+        if (!f->seal)
+                return 0;
+
+        r = journal_file_hmac_start(f);
+        if (r < 0)
+                return r;
+
+        /* All but state+reserved, boot_id, arena_size,
+         * tail_object_offset, n_objects, n_entries,
+         * tail_entry_seqnum, head_entry_seqnum, entry_array_offset,
+         * head_entry_realtime, tail_entry_realtime,
+         * tail_entry_monotonic, n_data, n_fields, n_tags,
+         * n_entry_arrays. */
+
+        gcry_md_write(f->hmac, f->header->signature, offsetof(Header, state) - offsetof(Header, signature));
+        gcry_md_write(f->hmac, &f->header->file_id, offsetof(Header, boot_id) - offsetof(Header, file_id));
+        gcry_md_write(f->hmac, &f->header->seqnum_id, offsetof(Header, arena_size) - offsetof(Header, seqnum_id));
+        gcry_md_write(f->hmac, &f->header->data_hash_table_offset, offsetof(Header, tail_object_offset) - offsetof(Header, data_hash_table_offset));
+
+        return 0;
+}
+
+int journal_file_fss_load(JournalFile *f) {
+        int r, fd = -1;
+        char *p = NULL;
+        struct stat st;
+        FSSHeader *m = NULL;
+        sd_id128_t machine;
+
+        assert(f);
+
+        if (!f->seal)
+                return 0;
+
+        r = sd_id128_get_machine(&machine);
+        if (r < 0)
+                return r;
+
+        if (asprintf(&p, "/var/log/journal/" SD_ID128_FORMAT_STR "/fss",
+                     SD_ID128_FORMAT_VAL(machine)) < 0)
+                return -ENOMEM;
+
+        fd = open(p, O_RDWR|O_CLOEXEC|O_NOCTTY, 0600);
+        if (fd < 0) {
+                if (errno != ENOENT)
+                        log_error_errno(errno, "Failed to open %s: %m", p);
+
+                r = -errno;
+                goto finish;
+        }
+
+        if (fstat(fd, &st) < 0) {
+                r = -errno;
+                goto finish;
+        }
+
+        if (st.st_size < (off_t) sizeof(FSSHeader)) {
+                r = -ENODATA;
+                goto finish;
+        }
+
+        m = mmap(NULL, PAGE_ALIGN(sizeof(FSSHeader)), PROT_READ, MAP_SHARED, fd, 0);
+        if (m == MAP_FAILED) {
+                m = NULL;
+                r = -errno;
+                goto finish;
+        }
+
+        if (memcmp(m->signature, FSS_HEADER_SIGNATURE, 8) != 0) {
+                r = -EBADMSG;
+                goto finish;
+        }
+
+        if (m->incompatible_flags != 0) {
+                r = -EPROTONOSUPPORT;
+                goto finish;
+        }
+
+        if (le64toh(m->header_size) < sizeof(FSSHeader)) {
+                r = -EBADMSG;
+                goto finish;
+        }
+
+        if (le64toh(m->fsprg_state_size) != FSPRG_stateinbytes(le16toh(m->fsprg_secpar))) {
+                r = -EBADMSG;
+                goto finish;
+        }
+
+        f->fss_file_size = le64toh(m->header_size) + le64toh(m->fsprg_state_size);
+        if ((uint64_t) st.st_size < f->fss_file_size) {
+                r = -ENODATA;
+                goto finish;
+        }
+
+        if (!sd_id128_equal(machine, m->machine_id)) {
+                r = -EHOSTDOWN;
+                goto finish;
+        }
+
+        if (le64toh(m->start_usec) <= 0 ||
+            le64toh(m->interval_usec) <= 0) {
+                r = -EBADMSG;
+                goto finish;
+        }
+
+        f->fss_file = mmap(NULL, PAGE_ALIGN(f->fss_file_size), PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+        if (f->fss_file == MAP_FAILED) {
+                f->fss_file = NULL;
+                r = -errno;
+                goto finish;
+        }
+
+        f->fss_start_usec = le64toh(f->fss_file->start_usec);
+        f->fss_interval_usec = le64toh(f->fss_file->interval_usec);
+
+        f->fsprg_state = (uint8_t*) f->fss_file + le64toh(f->fss_file->header_size);
+        f->fsprg_state_size = le64toh(f->fss_file->fsprg_state_size);
+
+        r = 0;
+
+finish:
+        if (m)
+                munmap(m, PAGE_ALIGN(sizeof(FSSHeader)));
+
+        safe_close(fd);
+        free(p);
+
+        return r;
+}
+
+int journal_file_hmac_setup(JournalFile *f) {
+        gcry_error_t e;
+
+        if (!f->seal)
+                return 0;
+
+        initialize_libgcrypt(true);
+
+        e = gcry_md_open(&f->hmac, GCRY_MD_SHA256, GCRY_MD_FLAG_HMAC);
+        if (e != 0)
+                return -EOPNOTSUPP;
+
+        return 0;
+}
+
+int journal_file_append_first_tag(JournalFile *f) {
+        int r;
+        uint64_t p;
+
+        if (!f->seal)
+                return 0;
+
+        log_debug("Calculating first tag...");
+
+        r = journal_file_hmac_put_header(f);
+        if (r < 0)
+                return r;
+
+        p = le64toh(f->header->field_hash_table_offset);
+        if (p < offsetof(Object, hash_table.items))
+                return -EINVAL;
+        p -= offsetof(Object, hash_table.items);
+
+        r = journal_file_hmac_put_object(f, OBJECT_FIELD_HASH_TABLE, NULL, p);
+        if (r < 0)
+                return r;
+
+        p = le64toh(f->header->data_hash_table_offset);
+        if (p < offsetof(Object, hash_table.items))
+                return -EINVAL;
+        p -= offsetof(Object, hash_table.items);
+
+        r = journal_file_hmac_put_object(f, OBJECT_DATA_HASH_TABLE, NULL, p);
+        if (r < 0)
+                return r;
+
+        r = journal_file_append_tag(f);
+        if (r < 0)
+                return r;
+
+        return 0;
+}
+
+int journal_file_parse_verification_key(JournalFile *f, const char *key) {
+        uint8_t *seed;
+        size_t seed_size, c;
+        const char *k;
+        int r;
+        unsigned long long start, interval;
+
+        seed_size = FSPRG_RECOMMENDED_SEEDLEN;
+        seed = malloc(seed_size);
+        if (!seed)
+                return -ENOMEM;
+
+        k = key;
+        for (c = 0; c < seed_size; c++) {
+                int x, y;
+
+                while (*k == '-')
+                        k++;
+
+                x = unhexchar(*k);
+                if (x < 0) {
+                        free(seed);
+                        return -EINVAL;
+                }
+                k++;
+                y = unhexchar(*k);
+                if (y < 0) {
+                        free(seed);
+                        return -EINVAL;
+                }
+                k++;
+
+                seed[c] = (uint8_t) (x * 16 + y);
+        }
+
+        if (*k != '/') {
+                free(seed);
+                return -EINVAL;
+        }
+        k++;
+
+        r = sscanf(k, "%llx-%llx", &start, &interval);
+        if (r != 2) {
+                free(seed);
+                return -EINVAL;
+        }
+
+        f->fsprg_seed = seed;
+        f->fsprg_seed_size = seed_size;
+
+        f->fss_start_usec = start * interval;
+        f->fss_interval_usec = interval;
+
+        return 0;
+}
+
+bool journal_file_next_evolve_usec(JournalFile *f, usec_t *u) {
+        uint64_t epoch;
+
+        assert(f);
+        assert(u);
+
+        if (!f->seal)
+                return false;
+
+        epoch = FSPRG_GetEpoch(f->fsprg_state);
+
+        *u = (usec_t) (f->fss_start_usec + f->fss_interval_usec * epoch + f->fss_interval_usec);
+
+        return true;
+}

src/libsystemd/sd-journal/journal-authenticate.h

@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include <stdbool.h>
+
+#include "journal-file.h"
+
+int journal_file_append_tag(JournalFile *f);
+int journal_file_maybe_append_tag(JournalFile *f, uint64_t realtime);
+int journal_file_append_first_tag(JournalFile *f);
+
+int journal_file_hmac_setup(JournalFile *f);
+int journal_file_hmac_start(JournalFile *f);
+int journal_file_hmac_put_header(JournalFile *f);
+int journal_file_hmac_put_object(JournalFile *f, ObjectType type, Object *o, uint64_t p);
+
+int journal_file_fss_load(JournalFile *f);
+int journal_file_parse_verification_key(JournalFile *f, const char *key);
+
+int journal_file_fsprg_evolve(JournalFile *f, uint64_t realtime);
+int journal_file_fsprg_seek(JournalFile *f, uint64_t epoch);
+
+bool journal_file_next_evolve_usec(JournalFile *f, usec_t *u);

src/libsystemd/sd-journal/journal-def.h

@@ -0,0 +1,252 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include "sd-id128.h"
+
+#include "macro.h"
+#include "sparse-endian.h"
+
+/*
+ * If you change this file you probably should also change its documentation:
+ *
+ * https://systemd.io/JOURNAL_FILE_FORMAT
+ */
+
+typedef struct Header Header;
+
+typedef struct ObjectHeader ObjectHeader;
+typedef union Object Object;
+
+typedef struct DataObject DataObject;
+typedef struct FieldObject FieldObject;
+typedef struct EntryObject EntryObject;
+typedef struct HashTableObject HashTableObject;
+typedef struct EntryArrayObject EntryArrayObject;
+typedef struct TagObject TagObject;
+
+typedef struct EntryItem EntryItem;
+typedef struct HashItem HashItem;
+
+typedef struct FSSHeader FSSHeader;
+
+/* Object types */
+typedef enum ObjectType {
+        OBJECT_UNUSED, /* also serves as "any type" or "additional context" */
+        OBJECT_DATA,
+        OBJECT_FIELD,
+        OBJECT_ENTRY,
+        OBJECT_DATA_HASH_TABLE,
+        OBJECT_FIELD_HASH_TABLE,
+        OBJECT_ENTRY_ARRAY,
+        OBJECT_TAG,
+        _OBJECT_TYPE_MAX
+} ObjectType;
+
+/* Object flags */
+enum {
+        OBJECT_COMPRESSED_XZ   = 1 << 0,
+        OBJECT_COMPRESSED_LZ4  = 1 << 1,
+        OBJECT_COMPRESSED_ZSTD = 1 << 2,
+        OBJECT_COMPRESSION_MASK = (OBJECT_COMPRESSED_XZ | OBJECT_COMPRESSED_LZ4 | OBJECT_COMPRESSED_ZSTD),
+        _OBJECT_COMPRESSED_MAX = OBJECT_COMPRESSION_MASK,
+};
+
+struct ObjectHeader {
+        uint8_t type;
+        uint8_t flags;
+        uint8_t reserved[6];
+        le64_t size;
+        uint8_t payload[];
+} _packed_;
+
+#define DataObject__contents {                                          \
+        ObjectHeader object;                                            \
+        le64_t hash;                                                    \
+        le64_t next_hash_offset;                                        \
+        le64_t next_field_offset;                                       \
+        le64_t entry_offset; /* the first array entry we store inline */ \
+        le64_t entry_array_offset;                                      \
+        le64_t n_entries;                                               \
+        uint8_t payload[];                                              \
+        }
+
+struct DataObject DataObject__contents;
+struct DataObject__packed DataObject__contents _packed_;
+assert_cc(sizeof(struct DataObject) == sizeof(struct DataObject__packed));
+
+#define FieldObject__contents {                 \
+        ObjectHeader object;                    \
+        le64_t hash;                            \
+        le64_t next_hash_offset;                \
+        le64_t head_data_offset;                \
+        uint8_t payload[];                      \
+}
+
+struct FieldObject FieldObject__contents;
+struct FieldObject__packed FieldObject__contents _packed_;
+assert_cc(sizeof(struct FieldObject) == sizeof(struct FieldObject__packed));
+
+struct EntryItem {
+        le64_t object_offset;
+        le64_t hash;
+} _packed_;
+
+#define EntryObject__contents { \
+        ObjectHeader object;    \
+        le64_t seqnum;          \
+        le64_t realtime;        \
+        le64_t monotonic;       \
+        sd_id128_t boot_id;     \
+        le64_t xor_hash;        \
+        EntryItem items[];      \
+        }
+
+struct EntryObject EntryObject__contents;
+struct EntryObject__packed EntryObject__contents _packed_;
+assert_cc(sizeof(struct EntryObject) == sizeof(struct EntryObject__packed));
+
+struct HashItem {
+        le64_t head_hash_offset;
+        le64_t tail_hash_offset;
+} _packed_;
+
+struct HashTableObject {
+        ObjectHeader object;
+        HashItem items[];
+} _packed_;
+
+struct EntryArrayObject {
+        ObjectHeader object;
+        le64_t next_entry_array_offset;
+        le64_t items[];
+} _packed_;
+
+#define TAG_LENGTH (256/8)
+
+struct TagObject {
+        ObjectHeader object;
+        le64_t seqnum;
+        le64_t epoch;
+        uint8_t tag[TAG_LENGTH]; /* SHA-256 HMAC */
+} _packed_;
+
+union Object {
+        ObjectHeader object;
+        DataObject data;
+        FieldObject field;
+        EntryObject entry;
+        HashTableObject hash_table;
+        EntryArrayObject entry_array;
+        TagObject tag;
+};
+
+enum {
+        STATE_OFFLINE = 0,
+        STATE_ONLINE = 1,
+        STATE_ARCHIVED = 2,
+        _STATE_MAX
+};
+
+/* Header flags */
+enum {
+        HEADER_INCOMPATIBLE_COMPRESSED_XZ   = 1 << 0,
+        HEADER_INCOMPATIBLE_COMPRESSED_LZ4  = 1 << 1,
+        HEADER_INCOMPATIBLE_KEYED_HASH      = 1 << 2,
+        HEADER_INCOMPATIBLE_COMPRESSED_ZSTD = 1 << 3,
+};
+
+#define HEADER_INCOMPATIBLE_ANY               \
+        (HEADER_INCOMPATIBLE_COMPRESSED_XZ |  \
+         HEADER_INCOMPATIBLE_COMPRESSED_LZ4 | \
+         HEADER_INCOMPATIBLE_KEYED_HASH |     \
+         HEADER_INCOMPATIBLE_COMPRESSED_ZSTD)
+
+#if HAVE_XZ && HAVE_LZ4 && HAVE_ZSTD
+#  define HEADER_INCOMPATIBLE_SUPPORTED HEADER_INCOMPATIBLE_ANY
+#elif HAVE_XZ && HAVE_LZ4
+#  define HEADER_INCOMPATIBLE_SUPPORTED (HEADER_INCOMPATIBLE_COMPRESSED_XZ|HEADER_INCOMPATIBLE_COMPRESSED_LZ4|HEADER_INCOMPATIBLE_KEYED_HASH)
+#elif HAVE_XZ && HAVE_ZSTD
+#  define HEADER_INCOMPATIBLE_SUPPORTED (HEADER_INCOMPATIBLE_COMPRESSED_XZ|HEADER_INCOMPATIBLE_COMPRESSED_ZSTD|HEADER_INCOMPATIBLE_KEYED_HASH)
+#elif HAVE_LZ4 && HAVE_ZSTD
+#  define HEADER_INCOMPATIBLE_SUPPORTED (HEADER_INCOMPATIBLE_COMPRESSED_LZ4|HEADER_INCOMPATIBLE_COMPRESSED_ZSTD|HEADER_INCOMPATIBLE_KEYED_HASH)
+#elif HAVE_XZ
+#  define HEADER_INCOMPATIBLE_SUPPORTED (HEADER_INCOMPATIBLE_COMPRESSED_XZ|HEADER_INCOMPATIBLE_KEYED_HASH)
+#elif HAVE_LZ4
+#  define HEADER_INCOMPATIBLE_SUPPORTED (HEADER_INCOMPATIBLE_COMPRESSED_LZ4|HEADER_INCOMPATIBLE_KEYED_HASH)
+#elif HAVE_ZSTD
+#  define HEADER_INCOMPATIBLE_SUPPORTED (HEADER_INCOMPATIBLE_COMPRESSED_ZSTD|HEADER_INCOMPATIBLE_KEYED_HASH)
+#else
+#  define HEADER_INCOMPATIBLE_SUPPORTED HEADER_INCOMPATIBLE_KEYED_HASH
+#endif
+
+enum {
+        HEADER_COMPATIBLE_SEALED = 1 << 0,
+};
+
+#define HEADER_COMPATIBLE_ANY HEADER_COMPATIBLE_SEALED
+#if HAVE_GCRYPT
+#  define HEADER_COMPATIBLE_SUPPORTED HEADER_COMPATIBLE_SEALED
+#else
+#  define HEADER_COMPATIBLE_SUPPORTED 0
+#endif
+
+#define HEADER_SIGNATURE                                                \
+        ((const char[]) { 'L', 'P', 'K', 'S', 'H', 'H', 'R', 'H' })
+
+#define struct_Header__contents {                       \
+        uint8_t signature[8]; /* "LPKSHHRH" */          \
+        le32_t compatible_flags;                        \
+        le32_t incompatible_flags;                      \
+        uint8_t state;                                  \
+        uint8_t reserved[7];                            \
+        sd_id128_t file_id;                             \
+        sd_id128_t machine_id;                          \
+        sd_id128_t boot_id;    /* last writer */        \
+        sd_id128_t seqnum_id;                           \
+        le64_t header_size;                             \
+        le64_t arena_size;                              \
+        le64_t data_hash_table_offset;                  \
+        le64_t data_hash_table_size;                    \
+        le64_t field_hash_table_offset;                 \
+        le64_t field_hash_table_size;                   \
+        le64_t tail_object_offset;                      \
+        le64_t n_objects;                               \
+        le64_t n_entries;                               \
+        le64_t tail_entry_seqnum;                       \
+        le64_t head_entry_seqnum;                       \
+        le64_t entry_array_offset;                      \
+        le64_t head_entry_realtime;                     \
+        le64_t tail_entry_realtime;                     \
+        le64_t tail_entry_monotonic;                    \
+        /* Added in 187 */                              \
+        le64_t n_data;                                  \
+        le64_t n_fields;                                \
+        /* Added in 189 */                              \
+        le64_t n_tags;                                  \
+        le64_t n_entry_arrays;                          \
+        /* Added in 246 */                              \
+        le64_t data_hash_chain_depth;                   \
+        le64_t field_hash_chain_depth;                  \
+        }
+
+struct Header struct_Header__contents;
+struct Header__packed struct_Header__contents _packed_;
+assert_cc(sizeof(struct Header) == sizeof(struct Header__packed));
+assert_cc(sizeof(struct Header) == 256);
+
+#define FSS_HEADER_SIGNATURE                                            \
+        ((const char[]) { 'K', 'S', 'H', 'H', 'R', 'H', 'L', 'P' })
+
+struct FSSHeader {
+        uint8_t signature[8]; /* "KSHHRHLP" */
+        le32_t compatible_flags;
+        le32_t incompatible_flags;
+        sd_id128_t machine_id;
+        sd_id128_t boot_id;    /* last writer */
+        le64_t header_size;
+        le64_t start_usec;
+        le64_t interval_usec;
+        le16_t fsprg_secpar;
+        le16_t reserved[3];
+        le64_t fsprg_state_size;
+} _packed_;

src/libsystemd/sd-journal/journal-file.h

@@ -0,0 +1,275 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include <inttypes.h>
+#include <sys/uio.h>
+
+#if HAVE_GCRYPT
+#  include <gcrypt.h>
+#endif
+
+#include "sd-event.h"
+#include "sd-id128.h"
+
+#include "hashmap.h"
+#include "journal-def.h"
+#include "mmap-cache.h"
+#include "sparse-endian.h"
+#include "time-util.h"
+
+typedef struct JournalMetrics {
+        /* For all these: -1 means "pick automatically", and 0 means "no limit enforced" */
+        uint64_t max_size;     /* how large journal files grow at max */
+        uint64_t min_size;     /* how large journal files grow at least */
+        uint64_t max_use;      /* how much disk space to use in total at max, keep_free permitting */
+        uint64_t min_use;      /* how much disk space to use in total at least, even if keep_free says not to */
+        uint64_t keep_free;    /* how much to keep free on disk */
+        uint64_t n_max_files;  /* how many files to keep around at max */
+} JournalMetrics;
+
+typedef enum direction {
+        DIRECTION_UP,
+        DIRECTION_DOWN
+} direction_t;
+
+typedef enum LocationType {
+        /* The first and last entries, resp. */
+        LOCATION_HEAD,
+        LOCATION_TAIL,
+
+        /* We already read the entry we currently point to, and the
+         * next one to read should probably not be this one again. */
+        LOCATION_DISCRETE,
+
+        /* We should seek to the precise location specified, and
+         * return it, as we haven't read it yet. */
+        LOCATION_SEEK
+} LocationType;
+
+typedef enum OfflineState {
+        OFFLINE_JOINED,
+        OFFLINE_SYNCING,
+        OFFLINE_OFFLINING,
+        OFFLINE_CANCEL,
+        OFFLINE_AGAIN_FROM_SYNCING,
+        OFFLINE_AGAIN_FROM_OFFLINING,
+        OFFLINE_DONE
+} OfflineState;
+
+typedef struct JournalFile {
+        int fd;
+        MMapFileDescriptor *cache_fd;
+
+        mode_t mode;
+
+        int flags;
+        bool writable:1;
+        bool compress_xz:1;
+        bool compress_lz4:1;
+        bool compress_zstd:1;
+        bool seal:1;
+        bool defrag_on_close:1;
+        bool close_fd:1;
+        bool archive:1;
+        bool keyed_hash:1;
+
+        direction_t last_direction;
+        LocationType location_type;
+        uint64_t last_n_entries;
+
+        char *path;
+        struct stat last_stat;
+        usec_t last_stat_usec;
+
+        Header *header;
+        HashItem *data_hash_table;
+        HashItem *field_hash_table;
+
+        uint64_t current_offset;
+        uint64_t current_seqnum;
+        uint64_t current_realtime;
+        uint64_t current_monotonic;
+        sd_id128_t current_boot_id;
+        uint64_t current_xor_hash;
+
+        JournalMetrics metrics;
+        MMapCache *mmap;
+
+        sd_event_source *post_change_timer;
+        usec_t post_change_timer_period;
+
+        OrderedHashmap *chain_cache;
+
+        pthread_t offline_thread;
+        volatile OfflineState offline_state;
+
+        unsigned last_seen_generation;
+
+        uint64_t compress_threshold_bytes;
+#if HAVE_COMPRESSION
+        void *compress_buffer;
+        size_t compress_buffer_size;
+#endif
+
+#if HAVE_GCRYPT
+        gcry_md_hd_t hmac;
+        bool hmac_running;
+
+        FSSHeader *fss_file;
+        size_t fss_file_size;
+
+        uint64_t fss_start_usec;
+        uint64_t fss_interval_usec;
+
+        void *fsprg_state;
+        size_t fsprg_state_size;
+
+        void *fsprg_seed;
+        size_t fsprg_seed_size;
+#endif
+} JournalFile;
+
+int journal_file_open(
+                int fd,
+                const char *fname,
+                int flags,
+                mode_t mode,
+                bool compress,
+                uint64_t compress_threshold_bytes,
+                bool seal,
+                JournalMetrics *metrics,
+                MMapCache *mmap_cache,
+                Set *deferred_closes,
+                JournalFile *template,
+                JournalFile **ret);
+
+int journal_file_set_offline(JournalFile *f, bool wait);
+bool journal_file_is_offlining(JournalFile *f);
+JournalFile* journal_file_close(JournalFile *j);
+int journal_file_fstat(JournalFile *f);
+DEFINE_TRIVIAL_CLEANUP_FUNC(JournalFile*, journal_file_close);
+
+int journal_file_open_reliably(
+                const char *fname,
+                int flags,
+                mode_t mode,
+                bool compress,
+                uint64_t compress_threshold_bytes,
+                bool seal,
+                JournalMetrics *metrics,
+                MMapCache *mmap_cache,
+                Set *deferred_closes,
+                JournalFile *template,
+                JournalFile **ret);
+
+#define ALIGN64(x) (((x) + 7ULL) & ~7ULL)
+#define VALID64(x) (((x) & 7ULL) == 0ULL)
+
+/* Use six characters to cover the offsets common in smallish journal
+ * files without adding too many zeros. */
+#define OFSfmt "%06"PRIx64
+
+static inline bool VALID_REALTIME(uint64_t u) {
+        /* This considers timestamps until the year 3112 valid. That should be plenty room... */
+        return u > 0 && u < (1ULL << 55);
+}
+
+static inline bool VALID_MONOTONIC(uint64_t u) {
+        /* This considers timestamps until 1142 years of runtime valid. */
+        return u < (1ULL << 55);
+}
+
+static inline bool VALID_EPOCH(uint64_t u) {
+        /* This allows changing the key for 1142 years, every usec. */
+        return u < (1ULL << 55);
+}
+
+#define JOURNAL_HEADER_CONTAINS(h, field) \
+        (le64toh((h)->header_size) >= offsetof(Header, field) + sizeof((h)->field))
+
+#define JOURNAL_HEADER_SEALED(h) \
+        FLAGS_SET(le32toh((h)->compatible_flags), HEADER_COMPATIBLE_SEALED)
+
+#define JOURNAL_HEADER_COMPRESSED_XZ(h) \
+        FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_COMPRESSED_XZ)
+
+#define JOURNAL_HEADER_COMPRESSED_LZ4(h) \
+        FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_COMPRESSED_LZ4)
+
+#define JOURNAL_HEADER_COMPRESSED_ZSTD(h) \
+        FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_COMPRESSED_ZSTD)
+
+#define JOURNAL_HEADER_KEYED_HASH(h) \
+        FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_KEYED_HASH)
+
+int journal_file_move_to_object(JournalFile *f, ObjectType type, uint64_t offset, Object **ret);
+
+uint64_t journal_file_entry_n_items(Object *o) _pure_;
+uint64_t journal_file_entry_array_n_items(Object *o) _pure_;
+uint64_t journal_file_hash_table_n_items(Object *o) _pure_;
+
+int journal_file_append_object(JournalFile *f, ObjectType type, uint64_t size, Object **ret, uint64_t *offset);
+int journal_file_append_entry(
+                JournalFile *f,
+                const dual_timestamp *ts,
+                const sd_id128_t *boot_id,
+                const struct iovec iovec[], unsigned n_iovec,
+                uint64_t *seqno,
+                Object **ret,
+                uint64_t *offset);
+
+int journal_file_find_data_object(JournalFile *f, const void *data, uint64_t size, Object **ret, uint64_t *offset);
+int journal_file_find_data_object_with_hash(JournalFile *f, const void *data, uint64_t size, uint64_t hash, Object **ret, uint64_t *offset);
+
+int journal_file_find_field_object(JournalFile *f, const void *field, uint64_t size, Object **ret, uint64_t *offset);
+int journal_file_find_field_object_with_hash(JournalFile *f, const void *field, uint64_t size, uint64_t hash, Object **ret, uint64_t *offset);
+
+void journal_file_reset_location(JournalFile *f);
+void journal_file_save_location(JournalFile *f, Object *o, uint64_t offset);
+int journal_file_compare_locations(JournalFile *af, JournalFile *bf);
+int journal_file_next_entry(JournalFile *f, uint64_t p, direction_t direction, Object **ret, uint64_t *offset);
+
+int journal_file_next_entry_for_data(JournalFile *f, Object *o, uint64_t p, uint64_t data_offset, direction_t direction, Object **ret, uint64_t *offset);
+
+int journal_file_move_to_entry_by_seqnum(JournalFile *f, uint64_t seqnum, direction_t direction, Object **ret, uint64_t *offset);
+int journal_file_move_to_entry_by_realtime(JournalFile *f, uint64_t realtime, direction_t direction, Object **ret, uint64_t *offset);
+int journal_file_move_to_entry_by_monotonic(JournalFile *f, sd_id128_t boot_id, uint64_t monotonic, direction_t direction, Object **ret, uint64_t *offset);
+
+int journal_file_move_to_entry_by_offset_for_data(JournalFile *f, uint64_t data_offset, uint64_t p, direction_t direction, Object **ret, uint64_t *offset);
+int journal_file_move_to_entry_by_seqnum_for_data(JournalFile *f, uint64_t data_offset, uint64_t seqnum, direction_t direction, Object **ret, uint64_t *offset);
+int journal_file_move_to_entry_by_realtime_for_data(JournalFile *f, uint64_t data_offset, uint64_t realtime, direction_t direction, Object **ret, uint64_t *offset);
+int journal_file_move_to_entry_by_monotonic_for_data(JournalFile *f, uint64_t data_offset, sd_id128_t boot_id, uint64_t monotonic, direction_t direction, Object **ret, uint64_t *offset);
+
+int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p);
+
+void journal_file_dump(JournalFile *f);
+void journal_file_print_header(JournalFile *f);
+
+int journal_file_archive(JournalFile *f);
+JournalFile* journal_initiate_close(JournalFile *f, Set *deferred_closes);
+int journal_file_rotate(JournalFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes);
+
+int journal_file_dispose(int dir_fd, const char *fname);
+
+void journal_file_post_change(JournalFile *f);
+int journal_file_enable_post_change_timer(JournalFile *f, sd_event *e, usec_t t);
+
+void journal_reset_metrics(JournalMetrics *m);
+void journal_default_metrics(JournalMetrics *m, int fd);
+
+int journal_file_get_cutoff_realtime_usec(JournalFile *f, usec_t *from, usec_t *to);
+int journal_file_get_cutoff_monotonic_usec(JournalFile *f, sd_id128_t boot, usec_t *from, usec_t *to);
+
+bool journal_file_rotate_suggested(JournalFile *f, usec_t max_file_usec);
+
+int journal_file_map_data_hash_table(JournalFile *f);
+int journal_file_map_field_hash_table(JournalFile *f);
+
+static inline bool JOURNAL_FILE_COMPRESS(JournalFile *f) {
+        assert(f);
+        return f->compress_xz || f->compress_lz4 || f->compress_zstd;
+}
+
+uint64_t journal_file_hash_data(JournalFile *f, const void *data, size_t sz);
+
+bool journal_field_valid(const char *p, size_t l, bool allow_protected);

src/libsystemd/sd-journal/journal-internal.h

@@ -0,0 +1,138 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include <inttypes.h>
+#include <stdbool.h>
+#include <sys/types.h>
+
+#include "sd-id128.h"
+#include "sd-journal.h"
+
+#include "hashmap.h"
+#include "journal-def.h"
+#include "journal-file.h"
+#include "list.h"
+#include "set.h"
+
+typedef struct Match Match;
+typedef struct Location Location;
+typedef struct Directory Directory;
+
+typedef enum MatchType {
+        MATCH_DISCRETE,
+        MATCH_OR_TERM,
+        MATCH_AND_TERM
+} MatchType;
+
+struct Match {
+        MatchType type;
+        Match *parent;
+        LIST_FIELDS(Match, matches);
+
+        /* For concrete matches */
+        char *data;
+        size_t size;
+        uint64_t hash; /* old-style jenkins hash. New-style siphash is different per file, hence won't be cached here */
+
+        /* For terms */
+        LIST_HEAD(Match, matches);
+};
+
+struct Location {
+        LocationType type;
+
+        bool seqnum_set:1;
+        bool realtime_set:1;
+        bool monotonic_set:1;
+        bool xor_hash_set:1;
+
+        uint64_t seqnum;
+        sd_id128_t seqnum_id;
+
+        uint64_t realtime;
+
+        uint64_t monotonic;
+        sd_id128_t boot_id;
+
+        uint64_t xor_hash;
+};
+
+struct Directory {
+        char *path;
+        int wd;
+        bool is_root;
+        unsigned last_seen_generation;
+};
+
+struct sd_journal {
+        int toplevel_fd;
+
+        char *path;
+        char *prefix;
+        char *namespace;
+
+        OrderedHashmap *files;
+        IteratedCache *files_cache;
+        MMapCache *mmap;
+
+        Location current_location;
+
+        JournalFile *current_file;
+        uint64_t current_field;
+
+        Match *level0, *level1, *level2;
+
+        pid_t original_pid;
+
+        int inotify_fd;
+        unsigned current_invalidate_counter, last_invalidate_counter;
+        usec_t last_process_usec;
+        unsigned generation;
+
+        /* Iterating through unique fields and their data values */
+        char *unique_field;
+        JournalFile *unique_file;
+        uint64_t unique_offset;
+
+        /* Iterating through known fields */
+        JournalFile *fields_file;
+        uint64_t fields_offset;
+        uint64_t fields_hash_table_index;
+        char *fields_buffer;
+        size_t fields_buffer_allocated;
+
+        int flags;
+
+        bool on_network:1;
+        bool no_new_files:1;
+        bool no_inotify:1;
+        bool unique_file_lost:1; /* File we were iterating over got
+                                    removed, and there were no more
+                                    files, so sd_j_enumerate_unique
+                                    will return a value equal to 0. */
+        bool fields_file_lost:1;
+        bool has_runtime_files:1;
+        bool has_persistent_files:1;
+
+        size_t data_threshold;
+
+        Hashmap *directories_by_path;
+        Hashmap *directories_by_wd;
+
+        Hashmap *errors;
+};
+
+char *journal_make_match_string(sd_journal *j);
+void journal_print_header(sd_journal *j);
+
+#define JOURNAL_FOREACH_DATA_RETVAL(j, data, l, retval)                     \
+        for (sd_journal_restart_data(j); ((retval) = sd_journal_enumerate_data((j), &(data), &(l))) > 0; )
+
+/* All errors that we might encounter while extracting a field that are not real errors,
+ * but only mean that the field is too large or we don't support the compression. */
+static inline bool JOURNAL_ERRNO_IS_UNAVAILABLE_FIELD(int r) {
+        return IN_SET(abs(r),
+                      ENOBUFS,          /* Field or decompressed field too large */
+                      E2BIG,            /* Field too large for pointer width */
+                      EPROTONOSUPPORT); /* Unsupported compression */
+}

src/libsystemd/sd-journal/journal-send.c

@@ -0,0 +1,569 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <printf.h>
+#include <stddef.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+#define SD_JOURNAL_SUPPRESS_LOCATION
+
+#include "sd-journal.h"
+
+#include "alloc-util.h"
+#include "errno-util.h"
+#include "fd-util.h"
+#include "io-util.h"
+#include "fileio.h"
+#include "memfd-util.h"
+#include "socket-util.h"
+#include "stdio-util.h"
+#include "string-util.h"
+#include "tmpfile-util.h"
+
+#define SNDBUF_SIZE (8*1024*1024)
+
+#define ALLOCA_CODE_FUNC(f, func)                 \
+        do {                                      \
+                size_t _fl;                       \
+                const char *_func = (func);       \
+                char **_f = &(f);                 \
+                _fl = strlen(_func) + 1;          \
+                *_f = newa(char, _fl + 10);       \
+                memcpy(*_f, "CODE_FUNC=", 10);    \
+                memcpy(*_f + 10, _func, _fl);     \
+        } while (false)
+
+/* We open a single fd, and we'll share it with the current process,
+ * all its threads, and all its subprocesses. This means we need to
+ * initialize it atomically, and need to operate on it atomically
+ * never assuming we are the only user */
+
+static int journal_fd(void) {
+        int fd;
+        static int fd_plus_one = 0;
+
+retry:
+        if (fd_plus_one > 0)
+                return fd_plus_one - 1;
+
+        fd = socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0);
+        if (fd < 0)
+                return -errno;
+
+        fd_inc_sndbuf(fd, SNDBUF_SIZE);
+
+        if (!__sync_bool_compare_and_swap(&fd_plus_one, 0, fd+1)) {
+                safe_close(fd);
+                goto retry;
+        }
+
+        return fd;
+}
+
+_public_ int sd_journal_print(int priority, const char *format, ...) {
+        int r;
+        va_list ap;
+
+        va_start(ap, format);
+        r = sd_journal_printv(priority, format, ap);
+        va_end(ap);
+
+        return r;
+}
+
+_public_ int sd_journal_printv(int priority, const char *format, va_list ap) {
+        char p[STRLEN("PRIORITY=") + DECIMAL_STR_MAX(int) + 1];
+        char sbuf[LINE_MAX + 8] = "MESSAGE=";
+        struct iovec iov[2];
+        int len;
+        va_list aq;
+        char *buffer = sbuf;
+
+        assert_return(priority >= 0, -EINVAL);
+        assert_return(priority <= 7, -EINVAL);
+        assert_return(format, -EINVAL);
+
+        xsprintf(p, "PRIORITY=%i", priority & LOG_PRIMASK);
+
+        va_copy(aq, ap);
+        len = vsnprintf(buffer + 8, LINE_MAX, format, aq);
+        va_end(aq);
+
+        if (len >= (int)LONG_LINE_MAX - 8)
+                return -ENOBUFS;
+
+        /* Allocate large buffer to accommodate big message */
+        if (len >= LINE_MAX) {
+                buffer = alloca(len + 9);
+                memcpy(buffer, "MESSAGE=", 8);
+                assert_se(vsnprintf(buffer + 8, len + 1, format, ap) == len);
+        }
+
+        /* Strip trailing whitespace, keep prefix whitespace. */
+        (void) strstrip(buffer);
+
+        /* Suppress empty lines */
+        if (isempty(buffer + 8))
+                return 0;
+
+        iov[0] = IOVEC_MAKE_STRING(buffer);
+        iov[1] = IOVEC_MAKE_STRING(p);
+
+        return sd_journal_sendv(iov, 2);
+}
+
+_printf_(1, 0) static int fill_iovec_sprintf(const char *format, va_list ap, int extra, struct iovec **_iov) {
+        PROTECT_ERRNO;
+        int r, n = 0, i = 0, j;
+        struct iovec *iov = NULL;
+
+        assert(_iov);
+
+        if (extra > 0) {
+                n = MAX(extra * 2, extra + 4);
+                iov = malloc0(n * sizeof(struct iovec));
+                if (!iov) {
+                        r = -ENOMEM;
+                        goto fail;
+                }
+
+                i = extra;
+        }
+
+        while (format) {
+                struct iovec *c;
+                char *buffer;
+                va_list aq;
+
+                if (i >= n) {
+                        n = MAX(i*2, 4);
+                        c = reallocarray(iov, n, sizeof(struct iovec));
+                        if (!c) {
+                                r = -ENOMEM;
+                                goto fail;
+                        }
+
+                        iov = c;
+                }
+
+                va_copy(aq, ap);
+                if (vasprintf(&buffer, format, aq) < 0) {
+                        va_end(aq);
+                        r = -ENOMEM;
+                        goto fail;
+                }
+                va_end(aq);
+
+                VA_FORMAT_ADVANCE(format, ap);
+
+                (void) strstrip(buffer); /* strip trailing whitespace, keep prefixing whitespace */
+
+                iov[i++] = IOVEC_MAKE_STRING(buffer);
+
+                format = va_arg(ap, char *);
+        }
+
+        *_iov = iov;
+
+        return i;
+
+fail:
+        for (j = 0; j < i; j++)
+                free(iov[j].iov_base);
+
+        free(iov);
+
+        return r;
+}
+
+_public_ int sd_journal_send(const char *format, ...) {
+        int r, i, j;
+        va_list ap;
+        struct iovec *iov = NULL;
+
+        va_start(ap, format);
+        i = fill_iovec_sprintf(format, ap, 0, &iov);
+        va_end(ap);
+
+        if (_unlikely_(i < 0)) {
+                r = i;
+                goto finish;
+        }
+
+        r = sd_journal_sendv(iov, i);
+
+finish:
+        for (j = 0; j < i; j++)
+                free(iov[j].iov_base);
+
+        free(iov);
+
+        return r;
+}
+
+_public_ int sd_journal_sendv(const struct iovec *iov, int n) {
+        PROTECT_ERRNO;
+        int fd, r;
+        _cleanup_close_ int buffer_fd = -1;
+        struct iovec *w;
+        uint64_t *l;
+        int i, j = 0;
+        static const union sockaddr_union sa = {
+                .un.sun_family = AF_UNIX,
+                .un.sun_path = "/run/systemd/journal/socket",
+        };
+        struct msghdr mh = {
+                .msg_name = (struct sockaddr*) &sa.sa,
+                .msg_namelen = SOCKADDR_UN_LEN(sa.un),
+        };
+        ssize_t k;
+        bool have_syslog_identifier = false;
+        bool seal = true;
+
+        assert_return(iov, -EINVAL);
+        assert_return(n > 0, -EINVAL);
+
+        w = newa(struct iovec, n * 5 + 3);
+        l = newa(uint64_t, n);
+
+        for (i = 0; i < n; i++) {
+                char *c, *nl;
+
+                if (_unlikely_(!iov[i].iov_base || iov[i].iov_len <= 1))
+                        return -EINVAL;
+
+                c = memchr(iov[i].iov_base, '=', iov[i].iov_len);
+                if (_unlikely_(!c || c == iov[i].iov_base))
+                        return -EINVAL;
+
+                have_syslog_identifier = have_syslog_identifier ||
+                        (c == (char *) iov[i].iov_base + 17 &&
+                         startswith(iov[i].iov_base, "SYSLOG_IDENTIFIER"));
+
+                nl = memchr(iov[i].iov_base, '\n', iov[i].iov_len);
+                if (nl) {
+                        if (_unlikely_(nl < c))
+                                return -EINVAL;
+
+                        /* Already includes a newline? Bummer, then
+                         * let's write the variable name, then a
+                         * newline, then the size (64bit LE), followed
+                         * by the data and a final newline */
+
+                        w[j++] = IOVEC_MAKE(iov[i].iov_base, c - (char*) iov[i].iov_base);
+                        w[j++] = IOVEC_MAKE_STRING("\n");
+
+                        l[i] = htole64(iov[i].iov_len - (c - (char*) iov[i].iov_base) - 1);
+                        w[j++] = IOVEC_MAKE(&l[i], sizeof(uint64_t));
+
+                        w[j++] = IOVEC_MAKE(c + 1, iov[i].iov_len - (c - (char*) iov[i].iov_base) - 1);
+                } else
+                        /* Nothing special? Then just add the line and
+                         * append a newline */
+                        w[j++] = iov[i];
+
+                w[j++] = IOVEC_MAKE_STRING("\n");
+        }
+
+        if (!have_syslog_identifier &&
+            string_is_safe(program_invocation_short_name)) {
+
+                /* Implicitly add program_invocation_short_name, if it
+                 * is not set explicitly. We only do this for
+                 * program_invocation_short_name, and nothing else
+                 * since everything else is much nicer to retrieve
+                 * from the outside. */
+
+                w[j++] = IOVEC_MAKE_STRING("SYSLOG_IDENTIFIER=");
+                w[j++] = IOVEC_MAKE_STRING(program_invocation_short_name);
+                w[j++] = IOVEC_MAKE_STRING("\n");
+        }
+
+        fd = journal_fd();
+        if (_unlikely_(fd < 0))
+                return fd;
+
+        mh.msg_iov = w;
+        mh.msg_iovlen = j;
+
+        k = sendmsg(fd, &mh, MSG_NOSIGNAL);
+        if (k >= 0)
+                return 0;
+
+        /* Fail silently if the journal is not available */
+        if (errno == ENOENT)
+                return 0;
+
+        if (!IN_SET(errno, EMSGSIZE, ENOBUFS))
+                return -errno;
+
+        /* Message doesn't fit... Let's dump the data in a memfd or
+         * temporary file and just pass a file descriptor of it to the
+         * other side.
+         *
+         * For the temporary files we use /dev/shm instead of /tmp
+         * here, since we want this to be a tmpfs, and one that is
+         * available from early boot on and where unprivileged users
+         * can create files. */
+        buffer_fd = memfd_new(NULL);
+        if (buffer_fd < 0) {
+                if (buffer_fd == -ENOSYS) {
+                        buffer_fd = open_tmpfile_unlinkable("/dev/shm", O_RDWR | O_CLOEXEC);
+                        if (buffer_fd < 0)
+                                return buffer_fd;
+
+                        seal = false;
+                } else
+                        return buffer_fd;
+        }
+
+        n = writev(buffer_fd, w, j);
+        if (n < 0)
+                return -errno;
+
+        if (seal) {
+                r = memfd_set_sealed(buffer_fd);
+                if (r < 0)
+                        return r;
+        }
+
+        r = send_one_fd_sa(fd, buffer_fd, mh.msg_name, mh.msg_namelen, 0);
+        if (r == -ENOENT)
+                /* Fail silently if the journal is not available */
+                return 0;
+        return r;
+}
+
+static int fill_iovec_perror_and_send(const char *message, int skip, struct iovec iov[]) {
+        PROTECT_ERRNO;
+        size_t n, k;
+
+        k = isempty(message) ? 0 : strlen(message) + 2;
+        n = 8 + k + 256 + 1;
+
+        for (;;) {
+                char buffer[n];
+                char* j;
+
+                errno = 0;
+                j = strerror_r(_saved_errno_, buffer + 8 + k, n - 8 - k);
+                if (errno == 0) {
+                        char error[STRLEN("ERRNO=") + DECIMAL_STR_MAX(int) + 1];
+
+                        if (j != buffer + 8 + k)
+                                memmove(buffer + 8 + k, j, strlen(j)+1);
+
+                        memcpy(buffer, "MESSAGE=", 8);
+
+                        if (k > 0) {
+                                memcpy(buffer + 8, message, k - 2);
+                                memcpy(buffer + 8 + k - 2, ": ", 2);
+                        }
+
+                        xsprintf(error, "ERRNO=%i", _saved_errno_);
+
+                        assert_cc(3 == LOG_ERR);
+                        iov[skip+0] = IOVEC_MAKE_STRING("PRIORITY=3");
+                        iov[skip+1] = IOVEC_MAKE_STRING(buffer);
+                        iov[skip+2] = IOVEC_MAKE_STRING(error);
+
+                        return sd_journal_sendv(iov, skip + 3);
+                }
+
+                if (errno != ERANGE)
+                        return -errno;
+
+                n *= 2;
+        }
+}
+
+_public_ int sd_journal_perror(const char *message) {
+        struct iovec iovec[3];
+
+        return fill_iovec_perror_and_send(message, 0, iovec);
+}
+
+_public_ int sd_journal_stream_fd(const char *identifier, int priority, int level_prefix) {
+        static const union sockaddr_union sa = {
+                .un.sun_family = AF_UNIX,
+                .un.sun_path = "/run/systemd/journal/stdout",
+        };
+        _cleanup_close_ int fd = -1;
+        char *header;
+        size_t l;
+        int r;
+
+        assert_return(priority >= 0, -EINVAL);
+        assert_return(priority <= 7, -EINVAL);
+
+        fd = socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC, 0);
+        if (fd < 0)
+                return -errno;
+
+        r = connect(fd, &sa.sa, SOCKADDR_UN_LEN(sa.un));
+        if (r < 0)
+                return -errno;
+
+        if (shutdown(fd, SHUT_RD) < 0)
+                return -errno;
+
+        (void) fd_inc_sndbuf(fd, SNDBUF_SIZE);
+
+        identifier = strempty(identifier);
+
+        l = strlen(identifier);
+        header = newa(char, l + 1 + 1 + 2 + 2 + 2 + 2 + 2);
+
+        memcpy(header, identifier, l);
+        header[l++] = '\n';
+        header[l++] = '\n'; /* unit id */
+        header[l++] = '0' + priority;
+        header[l++] = '\n';
+        header[l++] = '0' + !!level_prefix;
+        header[l++] = '\n';
+        header[l++] = '0';
+        header[l++] = '\n';
+        header[l++] = '0';
+        header[l++] = '\n';
+        header[l++] = '0';
+        header[l++] = '\n';
+
+        r = loop_write(fd, header, l, false);
+        if (r < 0)
+                return r;
+
+        return TAKE_FD(fd);
+}
+
+_public_ int sd_journal_print_with_location(int priority, const char *file, const char *line, const char *func, const char *format, ...) {
+        int r;
+        va_list ap;
+
+        va_start(ap, format);
+        r = sd_journal_printv_with_location(priority, file, line, func, format, ap);
+        va_end(ap);
+
+        return r;
+}
+
+_public_ int sd_journal_printv_with_location(int priority, const char *file, const char *line, const char *func, const char *format, va_list ap) {
+        char p[STRLEN("PRIORITY=") + DECIMAL_STR_MAX(int) + 1];
+        char sbuf[LINE_MAX + 8] = "MESSAGE=";
+        struct iovec iov[5];
+        char *f;
+        int len;
+        char *buffer = sbuf;
+        va_list aq;
+
+        assert_return(priority >= 0, -EINVAL);
+        assert_return(priority <= 7, -EINVAL);
+        assert_return(format, -EINVAL);
+
+        xsprintf(p, "PRIORITY=%i", priority & LOG_PRIMASK);
+
+        va_copy(aq, ap);
+        len = vsnprintf(buffer + 8, LINE_MAX, format, aq);
+        va_end(aq);
+
+        if (len >= (int)LONG_LINE_MAX - 8)
+                return -ENOBUFS;
+
+        /* Allocate large buffer to accommodate big message */
+        if (len >= LINE_MAX) {
+                buffer = alloca(len + 9);
+                memcpy(buffer, "MESSAGE=", 8);
+                assert_se(vsnprintf(buffer + 8, len + 1, format, ap) == len);
+        }
+
+        /* Strip trailing whitespace, keep prefixing whitespace */
+        (void) strstrip(buffer);
+
+        /* Suppress empty lines */
+        if (isempty(buffer + 8))
+                return 0;
+
+        /* func is initialized from __func__ which is not a macro, but
+         * a static const char[], hence cannot easily be prefixed with
+         * CODE_FUNC=, hence let's do it manually here. */
+        ALLOCA_CODE_FUNC(f, func);
+
+        iov[0] = IOVEC_MAKE_STRING(buffer);
+        iov[1] = IOVEC_MAKE_STRING(p);
+        iov[2] = IOVEC_MAKE_STRING(file);
+        iov[3] = IOVEC_MAKE_STRING(line);
+        iov[4] = IOVEC_MAKE_STRING(f);
+
+        return sd_journal_sendv(iov, ELEMENTSOF(iov));
+}
+
+_public_ int sd_journal_send_with_location(const char *file, const char *line, const char *func, const char *format, ...) {
+        _cleanup_free_ struct iovec *iov = NULL;
+        int r, i, j;
+        va_list ap;
+        char *f;
+
+        va_start(ap, format);
+        i = fill_iovec_sprintf(format, ap, 3, &iov);
+        va_end(ap);
+
+        if (_unlikely_(i < 0)) {
+                r = i;
+                goto finish;
+        }
+
+        ALLOCA_CODE_FUNC(f, func);
+
+        iov[0] = IOVEC_MAKE_STRING(file);
+        iov[1] = IOVEC_MAKE_STRING(line);
+        iov[2] = IOVEC_MAKE_STRING(f);
+
+        r = sd_journal_sendv(iov, i);
+
+finish:
+        for (j = 3; j < i; j++)
+                free(iov[j].iov_base);
+
+        return r;
+}
+
+_public_ int sd_journal_sendv_with_location(
+                const char *file, const char *line,
+                const char *func,
+                const struct iovec *iov, int n) {
+
+        struct iovec *niov;
+        char *f;
+
+        assert_return(iov, -EINVAL);
+        assert_return(n > 0, -EINVAL);
+
+        niov = newa(struct iovec, n + 3);
+        memcpy(niov, iov, sizeof(struct iovec) * n);
+
+        ALLOCA_CODE_FUNC(f, func);
+
+        niov[n++] = IOVEC_MAKE_STRING(file);
+        niov[n++] = IOVEC_MAKE_STRING(line);
+        niov[n++] = IOVEC_MAKE_STRING(f);
+
+        return sd_journal_sendv(niov, n);
+}
+
+_public_ int sd_journal_perror_with_location(
+                const char *file, const char *line,
+                const char *func,
+                const char *message) {
+
+        struct iovec iov[6];
+        char *f;
+
+        ALLOCA_CODE_FUNC(f, func);
+
+        iov[0] = IOVEC_MAKE_STRING(file);
+        iov[1] = IOVEC_MAKE_STRING(line);
+        iov[2] = IOVEC_MAKE_STRING(f);
+
+        return fill_iovec_perror_and_send(message, 3, iov);
+}

src/libsystemd/sd-journal/journal-vacuum.c

@@ -0,0 +1,321 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "sd-id128.h"
+
+#include "alloc-util.h"
+#include "dirent-util.h"
+#include "fd-util.h"
+#include "format-util.h"
+#include "fs-util.h"
+#include "journal-def.h"
+#include "journal-file.h"
+#include "journal-vacuum.h"
+#include "sort-util.h"
+#include "string-util.h"
+#include "time-util.h"
+#include "xattr-util.h"
+
+struct vacuum_info {
+        uint64_t usage;
+        char *filename;
+
+        uint64_t realtime;
+
+        sd_id128_t seqnum_id;
+        uint64_t seqnum;
+        bool have_seqnum;
+};
+
+static int vacuum_compare(const struct vacuum_info *a, const struct vacuum_info *b) {
+        int r;
+
+        if (a->have_seqnum && b->have_seqnum &&
+            sd_id128_equal(a->seqnum_id, b->seqnum_id))
+                return CMP(a->seqnum, b->seqnum);
+
+        r = CMP(a->realtime, b->realtime);
+        if (r != 0)
+                return r;
+
+        if (a->have_seqnum && b->have_seqnum)
+                return memcmp(&a->seqnum_id, &b->seqnum_id, 16);
+
+        return strcmp(a->filename, b->filename);
+}
+
+static void patch_realtime(
+                int fd,
+                const char *fn,
+                const struct stat *st,
+                unsigned long long *realtime) {
+
+        usec_t x, crtime = 0;
+
+        /* The timestamp was determined by the file name, but let's
+         * see if the file might actually be older than the file name
+         * suggested... */
+
+        assert(fd >= 0);
+        assert(fn);
+        assert(st);
+        assert(realtime);
+
+        x = timespec_load(&st->st_ctim);
+        if (x > 0 && x != USEC_INFINITY && x < *realtime)
+                *realtime = x;
+
+        x = timespec_load(&st->st_atim);
+        if (x > 0 && x != USEC_INFINITY && x < *realtime)
+                *realtime = x;
+
+        x = timespec_load(&st->st_mtim);
+        if (x > 0 && x != USEC_INFINITY && x < *realtime)
+                *realtime = x;
+
+        /* Let's read the original creation time, if possible. Ideally
+         * we'd just query the creation time the FS might provide, but
+         * unfortunately there's currently no sane API to query
+         * it. Hence let's implement this manually... */
+
+        if (fd_getcrtime_at(fd, fn, &crtime, 0) >= 0) {
+                if (crtime < *realtime)
+                        *realtime = crtime;
+        }
+}
+
+static int journal_file_empty(int dir_fd, const char *name) {
+        _cleanup_close_ int fd;
+        struct stat st;
+        le64_t n_entries;
+        ssize_t n;
+
+        fd = openat(dir_fd, name, O_RDONLY|O_CLOEXEC|O_NOFOLLOW|O_NONBLOCK|O_NOATIME);
+        if (fd < 0) {
+                /* Maybe failed due to O_NOATIME and lack of privileges? */
+                fd = openat(dir_fd, name, O_RDONLY|O_CLOEXEC|O_NOFOLLOW|O_NONBLOCK);
+                if (fd < 0)
+                        return -errno;
+        }
+
+        if (fstat(fd, &st) < 0)
+                return -errno;
+
+        /* If an offline file doesn't even have a header we consider it empty */
+        if (st.st_size < (off_t) sizeof(Header))
+                return 1;
+
+        /* If the number of entries is empty, we consider it empty, too */
+        n = pread(fd, &n_entries, sizeof(n_entries), offsetof(Header, n_entries));
+        if (n < 0)
+                return -errno;
+        if (n != sizeof(n_entries))
+                return -EIO;
+
+        return le64toh(n_entries) <= 0;
+}
+
+int journal_directory_vacuum(
+                const char *directory,
+                uint64_t max_use,
+                uint64_t n_max_files,
+                usec_t max_retention_usec,
+                usec_t *oldest_usec,
+                bool verbose) {
+
+        uint64_t sum = 0, freed = 0, n_active_files = 0;
+        size_t n_list = 0, n_allocated = 0, i;
+        _cleanup_closedir_ DIR *d = NULL;
+        struct vacuum_info *list = NULL;
+        usec_t retention_limit = 0;
+        char sbytes[FORMAT_BYTES_MAX];
+        struct dirent *de;
+        int r;
+
+        assert(directory);
+
+        if (max_use <= 0 && max_retention_usec <= 0 && n_max_files <= 0)
+                return 0;
+
+        if (max_retention_usec > 0)
+                retention_limit = usec_sub_unsigned(now(CLOCK_REALTIME), max_retention_usec);
+
+        d = opendir(directory);
+        if (!d)
+                return -errno;
+
+        FOREACH_DIRENT_ALL(de, d, r = -errno; goto finish) {
+
+                unsigned long long seqnum = 0, realtime;
+                _cleanup_free_ char *p = NULL;
+                sd_id128_t seqnum_id;
+                bool have_seqnum;
+                uint64_t size;
+                struct stat st;
+                size_t q;
+
+                if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0) {
+                        log_debug_errno(errno, "Failed to stat file %s while vacuuming, ignoring: %m", de->d_name);
+                        continue;
+                }
+
+                if (!S_ISREG(st.st_mode))
+                        continue;
+
+                q = strlen(de->d_name);
+
+                if (endswith(de->d_name, ".journal")) {
+
+                        /* Vacuum archived files. Active files are
+                         * left around */
+
+                        if (q < 1 + 32 + 1 + 16 + 1 + 16 + 8) {
+                                n_active_files++;
+                                continue;
+                        }
+
+                        if (de->d_name[q-8-16-1] != '-' ||
+                            de->d_name[q-8-16-1-16-1] != '-' ||
+                            de->d_name[q-8-16-1-16-1-32-1] != '@') {
+                                n_active_files++;
+                                continue;
+                        }
+
+                        p = strdup(de->d_name);
+                        if (!p) {
+                                r = -ENOMEM;
+                                goto finish;
+                        }
+
+                        de->d_name[q-8-16-1-16-1] = 0;
+                        if (sd_id128_from_string(de->d_name + q-8-16-1-16-1-32, &seqnum_id) < 0) {
+                                n_active_files++;
+                                continue;
+                        }
+
+                        if (sscanf(de->d_name + q-8-16-1-16, "%16llx-%16llx.journal", &seqnum, &realtime) != 2) {
+                                n_active_files++;
+                                continue;
+                        }
+
+                        have_seqnum = true;
+
+                } else if (endswith(de->d_name, ".journal~")) {
+                        unsigned long long tmp;
+
+                        /* Vacuum corrupted files */
+
+                        if (q < 1 + 16 + 1 + 16 + 8 + 1) {
+                                n_active_files++;
+                                continue;
+                        }
+
+                        if (de->d_name[q-1-8-16-1] != '-' ||
+                            de->d_name[q-1-8-16-1-16-1] != '@') {
+                                n_active_files++;
+                                continue;
+                        }
+
+                        p = strdup(de->d_name);
+                        if (!p) {
+                                r = -ENOMEM;
+                                goto finish;
+                        }
+
+                        if (sscanf(de->d_name + q-1-8-16-1-16, "%16llx-%16llx.journal~", &realtime, &tmp) != 2) {
+                                n_active_files++;
+                                continue;
+                        }
+
+                        have_seqnum = false;
+                } else {
+                        /* We do not vacuum unknown files! */
+                        log_debug("Not vacuuming unknown file %s.", de->d_name);
+                        continue;
+                }
+
+                size = 512UL * (uint64_t) st.st_blocks;
+
+                r = journal_file_empty(dirfd(d), p);
+                if (r < 0) {
+                        log_debug_errno(r, "Failed check if %s is empty, ignoring: %m", p);
+                        continue;
+                }
+                if (r > 0) {
+                        /* Always vacuum empty non-online files. */
+
+                        r = unlinkat_deallocate(dirfd(d), p, 0);
+                        if (r >= 0) {
+
+                                log_full(verbose ? LOG_INFO : LOG_DEBUG,
+                                         "Deleted empty archived journal %s/%s (%s).", directory, p, format_bytes(sbytes, sizeof(sbytes), size));
+
+                                freed += size;
+                        } else if (r != -ENOENT)
+                                log_warning_errno(r, "Failed to delete empty archived journal %s/%s: %m", directory, p);
+
+                        continue;
+                }
+
+                patch_realtime(dirfd(d), p, &st, &realtime);
+
+                if (!GREEDY_REALLOC(list, n_allocated, n_list + 1)) {
+                        r = -ENOMEM;
+                        goto finish;
+                }
+
+                list[n_list++] = (struct vacuum_info) {
+                        .filename = TAKE_PTR(p),
+                        .usage = size,
+                        .seqnum = seqnum,
+                        .realtime = realtime,
+                        .seqnum_id = seqnum_id,
+                        .have_seqnum = have_seqnum,
+                };
+
+                sum += size;
+        }
+
+        typesafe_qsort(list, n_list, vacuum_compare);
+
+        for (i = 0; i < n_list; i++) {
+                uint64_t left;
+
+                left = n_active_files + n_list - i;
+
+                if ((max_retention_usec <= 0 || list[i].realtime >= retention_limit) &&
+                    (max_use <= 0 || sum <= max_use) &&
+                    (n_max_files <= 0 || left <= n_max_files))
+                        break;
+
+                r = unlinkat_deallocate(dirfd(d), list[i].filename, 0);
+                if (r >= 0) {
+                        log_full(verbose ? LOG_INFO : LOG_DEBUG, "Deleted archived journal %s/%s (%s).", directory, list[i].filename, format_bytes(sbytes, sizeof(sbytes), list[i].usage));
+                        freed += list[i].usage;
+
+                        if (list[i].usage < sum)
+                                sum -= list[i].usage;
+                        else
+                                sum = 0;
+
+                } else if (r != -ENOENT)
+                        log_warning_errno(r, "Failed to delete archived journal %s/%s: %m", directory, list[i].filename);
+        }
+
+        if (oldest_usec && i < n_list && (*oldest_usec == 0 || list[i].realtime < *oldest_usec))
+                *oldest_usec = list[i].realtime;
+
+        r = 0;
+
+finish:
+        for (i = 0; i < n_list; i++)
+                free(list[i].filename);
+        free(list);
+
+        log_full(verbose ? LOG_INFO : LOG_DEBUG, "Vacuuming done, freed %s of archived journals from %s.", format_bytes(sbytes, sizeof(sbytes), freed), directory);
+
+        return r;
+}

src/libsystemd/sd-journal/journal-vacuum.h

@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include <inttypes.h>
+#include <stdbool.h>
+
+#include "time-util.h"
+
+int journal_directory_vacuum(const char *directory, uint64_t max_use, uint64_t n_max_files, usec_t max_retention_usec, usec_t *oldest_usec, bool verbose);

src/libsystemd/sd-journal/journal-verify.h

@@ -0,0 +1,6 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include "journal-file.h"
+
+int journal_file_verify(JournalFile *f, const char *key, usec_t *first_contained, usec_t *last_validated, usec_t *last_contained, bool show_progress);

src/libsystemd/sd-journal/lookup3.h

@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: LicenseRef-lookup3-public-domain */
+#pragma once
+
+#include <inttypes.h>
+#include <sys/types.h>
+
+#include "macro.h"
+
+uint32_t jenkins_hashword(const uint32_t *k, size_t length, uint32_t initval) _pure_;
+void jenkins_hashword2(const uint32_t *k, size_t length, uint32_t *pc, uint32_t *pb);
+
+uint32_t jenkins_hashlittle(const void *key, size_t length, uint32_t initval) _pure_;
+void jenkins_hashlittle2(const void *key, size_t length, uint32_t *pc, uint32_t *pb);
+
+uint32_t jenkins_hashbig(const void *key, size_t length, uint32_t initval) _pure_;
+
+static inline uint64_t jenkins_hash64(const void *data, size_t length) {
+        uint32_t a = 0, b = 0;
+
+        jenkins_hashlittle2(data, length, &a, &b);
+
+        return ((uint64_t) a << 32ULL) | (uint64_t) b;
+}

src/libsystemd/sd-journal/mmap-cache.c

@@ -0,0 +1,648 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <errno.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+
+#include "alloc-util.h"
+#include "errno-util.h"
+#include "fd-util.h"
+#include "hashmap.h"
+#include "list.h"
+#include "log.h"
+#include "macro.h"
+#include "memory-util.h"
+#include "mmap-cache.h"
+#include "sigbus.h"
+
+typedef struct Window Window;
+typedef struct Context Context;
+
+struct Window {
+        MMapCache *cache;
+
+        bool invalidated:1;
+        bool keep_always:1;
+        bool in_unused:1;
+
+        void *ptr;
+        uint64_t offset;
+        size_t size;
+
+        MMapFileDescriptor *fd;
+
+        LIST_FIELDS(Window, by_fd);
+        LIST_FIELDS(Window, unused);
+
+        LIST_HEAD(Context, contexts);
+};
+
+struct Context {
+        MMapCache *cache;
+        unsigned id;
+        Window *window;
+
+        LIST_FIELDS(Context, by_window);
+};
+
+struct MMapFileDescriptor {
+        MMapCache *cache;
+        int fd;
+        int prot;
+        bool sigbus;
+        LIST_HEAD(Window, windows);
+};
+
+struct MMapCache {
+        unsigned n_ref;
+        unsigned n_windows;
+
+        unsigned n_context_cache_hit, n_window_list_hit, n_missed;
+
+        Hashmap *fds;
+        Context *contexts[MMAP_CACHE_MAX_CONTEXTS];
+
+        LIST_HEAD(Window, unused);
+        Window *last_unused;
+};
+
+#define WINDOWS_MIN 64
+
+#if ENABLE_DEBUG_MMAP_CACHE
+/* Tiny windows increase mmap activity and the chance of exposing unsafe use. */
+# define WINDOW_SIZE (page_size())
+#else
+# define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
+#endif
+
+MMapCache* mmap_cache_new(void) {
+        MMapCache *m;
+
+        m = new0(MMapCache, 1);
+        if (!m)
+                return NULL;
+
+        m->n_ref = 1;
+        return m;
+}
+
+static void window_unlink(Window *w) {
+        Context *c;
+
+        assert(w);
+
+        if (w->ptr)
+                munmap(w->ptr, w->size);
+
+        if (w->fd)
+                LIST_REMOVE(by_fd, w->fd->windows, w);
+
+        if (w->in_unused) {
+                if (w->cache->last_unused == w)
+                        w->cache->last_unused = w->unused_prev;
+
+                LIST_REMOVE(unused, w->cache->unused, w);
+        }
+
+        LIST_FOREACH(by_window, c, w->contexts) {
+                assert(c->window == w);
+                c->window = NULL;
+        }
+}
+
+static void window_invalidate(Window *w) {
+        assert(w);
+        assert(w->fd);
+
+        if (w->invalidated)
+                return;
+
+        /* Replace the window with anonymous pages. This is useful
+         * when we hit a SIGBUS and want to make sure the file cannot
+         * trigger any further SIGBUS, possibly overrunning the sigbus
+         * queue. */
+
+        assert_se(mmap(w->ptr, w->size, w->fd->prot, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0) == w->ptr);
+        w->invalidated = true;
+}
+
+static void window_free(Window *w) {
+        assert(w);
+
+        window_unlink(w);
+        w->cache->n_windows--;
+        free(w);
+}
+
+_pure_ static bool window_matches(Window *w, uint64_t offset, size_t size) {
+        assert(w);
+        assert(size > 0);
+
+        return
+                offset >= w->offset &&
+                offset + size <= w->offset + w->size;
+}
+
+_pure_ static bool window_matches_fd(Window *w, MMapFileDescriptor *f, uint64_t offset, size_t size) {
+        assert(w);
+        assert(f);
+
+        return
+                w->fd == f &&
+                window_matches(w, offset, size);
+}
+
+static Window *window_add(MMapCache *m, MMapFileDescriptor *f, bool keep_always, uint64_t offset, size_t size, void *ptr) {
+        Window *w;
+
+        assert(m);
+        assert(f);
+
+        if (!m->last_unused || m->n_windows <= WINDOWS_MIN) {
+
+                /* Allocate a new window */
+                w = new(Window, 1);
+                if (!w)
+                        return NULL;
+                m->n_windows++;
+        } else {
+
+                /* Reuse an existing one */
+                w = m->last_unused;
+                window_unlink(w);
+        }
+
+        *w = (Window) {
+                .cache = m,
+                .fd = f,
+                .keep_always = keep_always,
+                .offset = offset,
+                .size = size,
+                .ptr = ptr,
+        };
+
+        LIST_PREPEND(by_fd, f->windows, w);
+
+        return w;
+}
+
+static void context_detach_window(Context *c) {
+        Window *w;
+
+        assert(c);
+
+        if (!c->window)
+                return;
+
+        w = TAKE_PTR(c->window);
+        LIST_REMOVE(by_window, w->contexts, c);
+
+        if (!w->contexts && !w->keep_always) {
+                /* Not used anymore? */
+#if ENABLE_DEBUG_MMAP_CACHE
+                /* Unmap unused windows immediately to expose use-after-unmap
+                 * by SIGSEGV. */
+                window_free(w);
+#else
+                LIST_PREPEND(unused, c->cache->unused, w);
+                if (!c->cache->last_unused)
+                        c->cache->last_unused = w;
+
+                w->in_unused = true;
+#endif
+        }
+}
+
+static void context_attach_window(Context *c, Window *w) {
+        assert(c);
+        assert(w);
+
+        if (c->window == w)
+                return;
+
+        context_detach_window(c);
+
+        if (w->in_unused) {
+                /* Used again? */
+                LIST_REMOVE(unused, c->cache->unused, w);
+                if (c->cache->last_unused == w)
+                        c->cache->last_unused = w->unused_prev;
+
+                w->in_unused = false;
+        }
+
+        c->window = w;
+        LIST_PREPEND(by_window, w->contexts, c);
+}
+
+static Context *context_add(MMapCache *m, unsigned id) {
+        Context *c;
+
+        assert(m);
+
+        c = m->contexts[id];
+        if (c)
+                return c;
+
+        c = new0(Context, 1);
+        if (!c)
+                return NULL;
+
+        c->cache = m;
+        c->id = id;
+
+        assert(!m->contexts[id]);
+        m->contexts[id] = c;
+
+        return c;
+}
+
+static void context_free(Context *c) {
+        assert(c);
+
+        context_detach_window(c);
+
+        if (c->cache) {
+                assert(c->cache->contexts[c->id] == c);
+                c->cache->contexts[c->id] = NULL;
+        }
+
+        free(c);
+}
+
+static MMapCache *mmap_cache_free(MMapCache *m) {
+        int i;
+
+        assert(m);
+
+        for (i = 0; i < MMAP_CACHE_MAX_CONTEXTS; i++)
+                if (m->contexts[i])
+                        context_free(m->contexts[i]);
+
+        hashmap_free(m->fds);
+
+        while (m->unused)
+                window_free(m->unused);
+
+        return mfree(m);
+}
+
+DEFINE_TRIVIAL_REF_UNREF_FUNC(MMapCache, mmap_cache, mmap_cache_free);
+
+static int make_room(MMapCache *m) {
+        assert(m);
+
+        if (!m->last_unused)
+                return 0;
+
+        window_free(m->last_unused);
+        return 1;
+}
+
+static int try_context(
+                MMapCache *m,
+                MMapFileDescriptor *f,
+                unsigned context,
+                bool keep_always,
+                uint64_t offset,
+                size_t size,
+                void **ret) {
+
+        Context *c;
+
+        assert(m);
+        assert(m->n_ref > 0);
+        assert(f);
+        assert(size > 0);
+        assert(ret);
+
+        c = m->contexts[context];
+        if (!c)
+                return 0;
+
+        assert(c->id == context);
+
+        if (!c->window)
+                return 0;
+
+        if (!window_matches_fd(c->window, f, offset, size)) {
+
+                /* Drop the reference to the window, since it's unnecessary now */
+                context_detach_window(c);
+                return 0;
+        }
+
+        if (c->window->fd->sigbus)
+                return -EIO;
+
+        c->window->keep_always = c->window->keep_always || keep_always;
+
+        *ret = (uint8_t*) c->window->ptr + (offset - c->window->offset);
+
+        return 1;
+}
+
+static int find_mmap(
+                MMapCache *m,
+                MMapFileDescriptor *f,
+                unsigned context,
+                bool keep_always,
+                uint64_t offset,
+                size_t size,
+                void **ret) {
+
+        Window *w;
+        Context *c;
+
+        assert(m);
+        assert(m->n_ref > 0);
+        assert(f);
+        assert(size > 0);
+
+        if (f->sigbus)
+                return -EIO;
+
+        LIST_FOREACH(by_fd, w, f->windows)
+                if (window_matches(w, offset, size))
+                        break;
+
+        if (!w)
+                return 0;
+
+        c = context_add(m, context);
+        if (!c)
+                return -ENOMEM;
+
+        context_attach_window(c, w);
+        w->keep_always = w->keep_always || keep_always;
+
+        *ret = (uint8_t*) w->ptr + (offset - w->offset);
+
+        return 1;
+}
+
+static int mmap_try_harder(MMapCache *m, void *addr, MMapFileDescriptor *f, int flags, uint64_t offset, size_t size, void **res) {
+        void *ptr;
+
+        assert(m);
+        assert(f);
+        assert(res);
+
+        for (;;) {
+                int r;
+
+                ptr = mmap(addr, size, f->prot, flags, f->fd, offset);
+                if (ptr != MAP_FAILED)
+                        break;
+                if (errno != ENOMEM)
+                        return negative_errno();
+
+                r = make_room(m);
+                if (r < 0)
+                        return r;
+                if (r == 0)
+                        return -ENOMEM;
+        }
+
+        *res = ptr;
+        return 0;
+}
+
+static int add_mmap(
+                MMapCache *m,
+                MMapFileDescriptor *f,
+                unsigned context,
+                bool keep_always,
+                uint64_t offset,
+                size_t size,
+                struct stat *st,
+                void **ret) {
+
+        uint64_t woffset, wsize;
+        Context *c;
+        Window *w;
+        void *d;
+        int r;
+
+        assert(m);
+        assert(m->n_ref > 0);
+        assert(f);
+        assert(size > 0);
+        assert(ret);
+
+        woffset = offset & ~((uint64_t) page_size() - 1ULL);
+        wsize = size + (offset - woffset);
+        wsize = PAGE_ALIGN(wsize);
+
+        if (wsize < WINDOW_SIZE) {
+                uint64_t delta;
+
+                delta = PAGE_ALIGN((WINDOW_SIZE - wsize) / 2);
+
+                if (delta > offset)
+                        woffset = 0;
+                else
+                        woffset -= delta;
+
+                wsize = WINDOW_SIZE;
+        }
+
+        if (st) {
+                /* Memory maps that are larger then the files
+                   underneath have undefined behavior. Hence, clamp
+                   things to the file size if we know it */
+
+                if (woffset >= (uint64_t) st->st_size)
+                        return -EADDRNOTAVAIL;
+
+                if (woffset + wsize > (uint64_t) st->st_size)
+                        wsize = PAGE_ALIGN(st->st_size - woffset);
+        }
+
+        r = mmap_try_harder(m, NULL, f, MAP_SHARED, woffset, wsize, &d);
+        if (r < 0)
+                return r;
+
+        c = context_add(m, context);
+        if (!c)
+                goto outofmem;
+
+        w = window_add(m, f, keep_always, woffset, wsize, d);
+        if (!w)
+                goto outofmem;
+
+        context_attach_window(c, w);
+
+        *ret = (uint8_t*) w->ptr + (offset - w->offset);
+
+        return 1;
+
+outofmem:
+        (void) munmap(d, wsize);
+        return -ENOMEM;
+}
+
+int mmap_cache_get(
+                MMapCache *m,
+                MMapFileDescriptor *f,
+                unsigned context,
+                bool keep_always,
+                uint64_t offset,
+                size_t size,
+                struct stat *st,
+                void **ret) {
+
+        int r;
+
+        assert(m);
+        assert(m->n_ref > 0);
+        assert(f);
+        assert(size > 0);
+        assert(ret);
+        assert(context < MMAP_CACHE_MAX_CONTEXTS);
+
+        /* Check whether the current context is the right one already */
+        r = try_context(m, f, context, keep_always, offset, size, ret);
+        if (r != 0) {
+                m->n_context_cache_hit++;
+                return r;
+        }
+
+        /* Search for a matching mmap */
+        r = find_mmap(m, f, context, keep_always, offset, size, ret);
+        if (r != 0) {
+                m->n_window_list_hit++;
+                return r;
+        }
+
+        m->n_missed++;
+
+        /* Create a new mmap */
+        return add_mmap(m, f, context, keep_always, offset, size, st, ret);
+}
+
+void mmap_cache_stats_log_debug(MMapCache *m) {
+        assert(m);
+
+        log_debug("mmap cache statistics: %u context cache hit, %u window list hit, %u miss", m->n_context_cache_hit, m->n_window_list_hit, m->n_missed);
+}
+
+static void mmap_cache_process_sigbus(MMapCache *m) {
+        bool found = false;
+        MMapFileDescriptor *f;
+        int r;
+
+        assert(m);
+
+        /* Iterate through all triggered pages and mark their files as
+         * invalidated */
+        for (;;) {
+                bool ours;
+                void *addr;
+
+                r = sigbus_pop(&addr);
+                if (_likely_(r == 0))
+                        break;
+                if (r < 0) {
+                        log_error_errno(r, "SIGBUS handling failed: %m");
+                        abort();
+                }
+
+                ours = false;
+                HASHMAP_FOREACH(f, m->fds) {
+                        Window *w;
+
+                        LIST_FOREACH(by_fd, w, f->windows) {
+                                if ((uint8_t*) addr >= (uint8_t*) w->ptr &&
+                                    (uint8_t*) addr < (uint8_t*) w->ptr + w->size) {
+                                        found = ours = f->sigbus = true;
+                                        break;
+                                }
+                        }
+
+                        if (ours)
+                                break;
+                }
+
+                /* Didn't find a matching window, give up */
+                if (!ours) {
+                        log_error("Unknown SIGBUS page, aborting.");
+                        abort();
+                }
+        }
+
+        /* The list of triggered pages is now empty. Now, let's remap
+         * all windows of the triggered file to anonymous maps, so
+         * that no page of the file in question is triggered again, so
+         * that we can be sure not to hit the queue size limit. */
+        if (_likely_(!found))
+                return;
+
+        HASHMAP_FOREACH(f, m->fds) {
+                Window *w;
+
+                if (!f->sigbus)
+                        continue;
+
+                LIST_FOREACH(by_fd, w, f->windows)
+                        window_invalidate(w);
+        }
+}
+
+bool mmap_cache_got_sigbus(MMapCache *m, MMapFileDescriptor *f) {
+        assert(m);
+        assert(f);
+
+        mmap_cache_process_sigbus(m);
+
+        return f->sigbus;
+}
+
+MMapFileDescriptor* mmap_cache_add_fd(MMapCache *m, int fd, int prot) {
+        MMapFileDescriptor *f;
+        int r;
+
+        assert(m);
+        assert(fd >= 0);
+
+        f = hashmap_get(m->fds, FD_TO_PTR(fd));
+        if (f)
+                return f;
+
+        r = hashmap_ensure_allocated(&m->fds, NULL);
+        if (r < 0)
+                return NULL;
+
+        f = new0(MMapFileDescriptor, 1);
+        if (!f)
+                return NULL;
+
+        f->cache = m;
+        f->fd = fd;
+        f->prot = prot;
+
+        r = hashmap_put(m->fds, FD_TO_PTR(fd), f);
+        if (r < 0)
+                return mfree(f);
+
+        return f;
+}
+
+void mmap_cache_free_fd(MMapCache *m, MMapFileDescriptor *f) {
+        assert(m);
+        assert(f);
+
+        /* Make sure that any queued SIGBUS are first dispatched, so
+         * that we don't end up with a SIGBUS entry we cannot relate
+         * to any existing memory map */
+
+        mmap_cache_process_sigbus(m);
+
+        while (f->windows)
+                window_free(f->windows);
+
+        if (f->cache)
+                assert_se(hashmap_remove(f->cache->fds, FD_TO_PTR(f->fd)));
+
+        free(f);
+}

src/libsystemd/sd-journal/mmap-cache.h

@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include <stdbool.h>
+#include <sys/stat.h>
+
+/* One context per object type, plus one of the header, plus one "additional" one */
+#define MMAP_CACHE_MAX_CONTEXTS 9
+
+typedef struct MMapCache MMapCache;
+typedef struct MMapFileDescriptor MMapFileDescriptor;
+
+MMapCache* mmap_cache_new(void);
+MMapCache* mmap_cache_ref(MMapCache *m);
+MMapCache* mmap_cache_unref(MMapCache *m);
+
+int mmap_cache_get(
+        MMapCache *m,
+        MMapFileDescriptor *f,
+        unsigned context,
+        bool keep_always,
+        uint64_t offset,
+        size_t size,
+        struct stat *st,
+        void **ret);
+MMapFileDescriptor * mmap_cache_add_fd(MMapCache *m, int fd, int prot);
+void mmap_cache_free_fd(MMapCache *m, MMapFileDescriptor *f);
+
+void mmap_cache_stats_log_debug(MMapCache *m);
+
+bool mmap_cache_got_sigbus(MMapCache *m, MMapFileDescriptor *f);

src/libsystemd/sd-journal/test-audit-type.c

@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <stdio.h>
+#include <linux/audit.h>
+
+#include "audit-type.h"
+
+static void print_audit_label(int i) {
+        const char *name;
+
+        name = audit_type_name_alloca(i);
+        /* This is a separate function only because of alloca */
+        printf("%i → %s → %s\n", i, audit_type_to_string(i), name);
+}
+
+static void test_audit_type(void) {
+        int i;
+
+        for (i = 0; i <= AUDIT_KERNEL; i++)
+                print_audit_label(i);
+}
+
+int main(int argc, char **argv) {
+        test_audit_type();
+        return 0;
+}

src/libsystemd/sd-journal/test-catalog.c

@@ -0,0 +1,235 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <locale.h>
+#include <unistd.h>
+
+#include "sd-messages.h"
+
+#include "alloc-util.h"
+#include "catalog.h"
+#include "fd-util.h"
+#include "fs-util.h"
+#include "log.h"
+#include "macro.h"
+#include "path-util.h"
+#include "string-util.h"
+#include "strv.h"
+#include "tests.h"
+#include "tmpfile-util.h"
+#include "util.h"
+
+static char** catalog_dirs = NULL;
+static const char *no_catalog_dirs[] = {
+        "/bin/hopefully/with/no/catalog",
+        NULL
+};
+
+static OrderedHashmap* test_import(const char* contents, ssize_t size, int code) {
+        _cleanup_(unlink_tempfilep) char name[] = "/tmp/test-catalog.XXXXXX";
+        _cleanup_close_ int fd;
+        OrderedHashmap *h;
+
+        if (size < 0)
+                size = strlen(contents);
+
+        assert_se(h = ordered_hashmap_new(&catalog_hash_ops));
+
+        fd = mkostemp_safe(name);
+        assert_se(fd >= 0);
+        assert_se(write(fd, contents, size) == size);
+
+        assert_se(catalog_import_file(h, name) == code);
+
+        return h;
+}
+
+static void test_catalog_import_invalid(void) {
+        _cleanup_ordered_hashmap_free_free_free_ OrderedHashmap *h = NULL;
+
+        h = test_import("xxx", -1, -EINVAL);
+        assert_se(ordered_hashmap_isempty(h));
+}
+
+static void test_catalog_import_badid(void) {
+        _cleanup_ordered_hashmap_free_free_free_ OrderedHashmap *h = NULL;
+        const char *input =
+"-- 0027229ca0644181a76c4e92458afaff dededededededededededededededede\n" \
+"Subject: message\n" \
+"\n" \
+"payload\n";
+        h = test_import(input, -1, -EINVAL);
+}
+
+static void test_catalog_import_one(void) {
+        _cleanup_ordered_hashmap_free_free_free_ OrderedHashmap *h = NULL;
+        char *payload;
+
+        const char *input =
+"-- 0027229ca0644181a76c4e92458afaff dededededededededededededededed\n" \
+"Subject: message\n" \
+"\n" \
+"payload\n";
+        const char *expect =
+"Subject: message\n" \
+"\n" \
+"payload\n";
+
+        h = test_import(input, -1, 0);
+        assert_se(ordered_hashmap_size(h) == 1);
+
+        ORDERED_HASHMAP_FOREACH(payload, h) {
+                printf("expect: %s\n", expect);
+                printf("actual: %s\n", payload);
+                assert_se(streq(expect, payload));
+        }
+}
+
+static void test_catalog_import_merge(void) {
+        _cleanup_ordered_hashmap_free_free_free_ OrderedHashmap *h = NULL;
+        char *payload;
+
+        const char *input =
+"-- 0027229ca0644181a76c4e92458afaff dededededededededededededededed\n" \
+"Subject: message\n" \
+"Defined-By: me\n" \
+"\n" \
+"payload\n" \
+"\n" \
+"-- 0027229ca0644181a76c4e92458afaff dededededededededededededededed\n" \
+"Subject: override subject\n" \
+"X-Header: hello\n" \
+"\n" \
+"override payload\n";
+
+        const char *combined =
+"Subject: override subject\n" \
+"X-Header: hello\n" \
+"Subject: message\n" \
+"Defined-By: me\n" \
+"\n" \
+"override payload\n";
+
+        h = test_import(input, -1, 0);
+        assert_se(ordered_hashmap_size(h) == 1);
+
+        ORDERED_HASHMAP_FOREACH(payload, h)
+                assert_se(streq(combined, payload));
+}
+
+static void test_catalog_import_merge_no_body(void) {
+        _cleanup_ordered_hashmap_free_free_free_ OrderedHashmap *h = NULL;
+        char *payload;
+
+        const char *input =
+"-- 0027229ca0644181a76c4e92458afaff dededededededededededededededed\n" \
+"Subject: message\n" \
+"Defined-By: me\n" \
+"\n" \
+"payload\n" \
+"\n" \
+"-- 0027229ca0644181a76c4e92458afaff dededededededededededededededed\n" \
+"Subject: override subject\n" \
+"X-Header: hello\n" \
+"\n";
+
+        const char *combined =
+"Subject: override subject\n" \
+"X-Header: hello\n" \
+"Subject: message\n" \
+"Defined-By: me\n" \
+"\n" \
+"payload\n";
+
+        h = test_import(input, -1, 0);
+        assert_se(ordered_hashmap_size(h) == 1);
+
+        ORDERED_HASHMAP_FOREACH(payload, h)
+                assert_se(streq(combined, payload));
+}
+
+static void test_catalog_update(const char *database) {
+        int r;
+
+        /* Test what happens if there are no files. */
+        r = catalog_update(database, NULL, NULL);
+        assert_se(r == 0);
+
+        /* Test what happens if there are no files in the directory. */
+        r = catalog_update(database, NULL, no_catalog_dirs);
+        assert_se(r == 0);
+
+        /* Make sure that we at least have some files loaded or the
+         * catalog_list below will fail. */
+        r = catalog_update(database, NULL, (const char * const *) catalog_dirs);
+        assert_se(r == 0);
+}
+
+static void test_catalog_file_lang(void) {
+        _cleanup_free_ char *lang = NULL, *lang2 = NULL, *lang3 = NULL, *lang4 = NULL;
+
+        assert_se(catalog_file_lang("systemd.de_DE.catalog", &lang) == 1);
+        assert_se(streq(lang, "de_DE"));
+
+        assert_se(catalog_file_lang("systemd..catalog", &lang2) == 0);
+        assert_se(lang2 == NULL);
+
+        assert_se(catalog_file_lang("systemd.fr.catalog", &lang2) == 1);
+        assert_se(streq(lang2, "fr"));
+
+        assert_se(catalog_file_lang("systemd.fr.catalog.gz", &lang3) == 0);
+        assert_se(lang3 == NULL);
+
+        assert_se(catalog_file_lang("systemd.01234567890123456789012345678901.catalog", &lang3) == 0);
+        assert_se(lang3 == NULL);
+
+        assert_se(catalog_file_lang("systemd.0123456789012345678901234567890.catalog", &lang3) == 1);
+        assert_se(streq(lang3, "0123456789012345678901234567890"));
+
+        assert_se(catalog_file_lang("/x/y/systemd.catalog", &lang4) == 0);
+        assert_se(lang4 == NULL);
+
+        assert_se(catalog_file_lang("/x/y/systemd.ru_RU.catalog", &lang4) == 1);
+        assert_se(streq(lang4, "ru_RU"));
+}
+
+int main(int argc, char *argv[]) {
+        _cleanup_(unlink_tempfilep) char database[] = "/tmp/test-catalog.XXXXXX";
+        _cleanup_free_ char *text = NULL;
+        int r;
+
+        setlocale(LC_ALL, "de_DE.UTF-8");
+
+        test_setup_logging(LOG_DEBUG);
+
+        /* If test-catalog is located at the build directory, then use catalogs in that.
+         * If it is not, e.g. installed by systemd-tests package, then use installed catalogs. */
+        catalog_dirs = STRV_MAKE(get_catalog_dir());
+
+        assert_se(access(catalog_dirs[0], F_OK) >= 0);
+        log_notice("Using catalog directory '%s'", catalog_dirs[0]);
+
+        test_catalog_file_lang();
+
+        test_catalog_import_invalid();
+        test_catalog_import_badid();
+        test_catalog_import_one();
+        test_catalog_import_merge();
+        test_catalog_import_merge_no_body();
+
+        assert_se(mkostemp_safe(database) >= 0);
+
+        test_catalog_update(database);
+
+        r = catalog_list(stdout, database, true);
+        assert_se(r >= 0);
+
+        r = catalog_list(stdout, database, false);
+        assert_se(r >= 0);
+
+        assert_se(catalog_get(database, SD_MESSAGE_COREDUMP, &text) >= 0);
+        printf(">>>%s<<<\n", text);
+
+        return 0;
+}

src/libsystemd/sd-journal/test-compress-benchmark.c

@@ -0,0 +1,179 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include "alloc-util.h"
+#include "compress.h"
+#include "env-util.h"
+#include "macro.h"
+#include "memory-util.h"
+#include "nulstr-util.h"
+#include "parse-util.h"
+#include "process-util.h"
+#include "random-util.h"
+#include "string-util.h"
+#include "tests.h"
+
+typedef int (compress_t)(const void *src, uint64_t src_size, void *dst,
+                         size_t dst_alloc_size, size_t *dst_size);
+typedef int (decompress_t)(const void *src, uint64_t src_size,
+                           void **dst, size_t *dst_alloc_size, size_t* dst_size, size_t dst_max);
+
+#if HAVE_COMPRESSION
+
+static usec_t arg_duration;
+static size_t arg_start;
+
+#define MAX_SIZE (1024*1024LU)
+#define PRIME 1048571  /* A prime close enough to one megabyte that mod 4 == 3 */
+
+static size_t _permute(size_t x) {
+        size_t residue;
+
+        if (x >= PRIME)
+                return x;
+
+        residue = x*x % PRIME;
+        if (x <= PRIME / 2)
+                return residue;
+        else
+                return PRIME - residue;
+}
+
+static size_t permute(size_t x) {
+        return _permute((_permute(x) + arg_start) % MAX_SIZE ^ 0xFF345);
+}
+
+static char* make_buf(size_t count, const char *type) {
+        char *buf;
+        size_t i;
+
+        buf = malloc(count);
+        assert_se(buf);
+
+        if (streq(type, "zeros"))
+                memzero(buf, count);
+        else if (streq(type, "simple"))
+                for (i = 0; i < count; i++)
+                        buf[i] = 'a' + i % ('z' - 'a' + 1);
+        else if (streq(type, "random")) {
+                size_t step = count / 10;
+
+                random_bytes(buf, step);
+                memzero(buf + 1*step, step);
+                random_bytes(buf + 2*step, step);
+                memzero(buf + 3*step, step);
+                random_bytes(buf + 4*step, step);
+                memzero(buf + 5*step, step);
+                random_bytes(buf + 6*step, step);
+                memzero(buf + 7*step, step);
+                random_bytes(buf + 8*step, step);
+                memzero(buf + 9*step, step);
+        } else
+                assert_not_reached("here");
+
+        return buf;
+}
+
+static void test_compress_decompress(const char* label, const char* type,
+                                     compress_t compress, decompress_t decompress) {
+        usec_t n, n2 = 0;
+        float dt;
+
+        _cleanup_free_ char *text, *buf;
+        _cleanup_free_ void *buf2 = NULL;
+        size_t buf2_allocated = 0;
+        size_t skipped = 0, compressed = 0, total = 0;
+
+        text = make_buf(MAX_SIZE, type);
+        buf = calloc(MAX_SIZE + 1, 1);
+        assert_se(text && buf);
+
+        n = now(CLOCK_MONOTONIC);
+
+        for (size_t i = 0; i <= MAX_SIZE; i++) {
+                size_t j = 0, k = 0, size;
+                int r;
+
+                size = permute(i);
+                if (size == 0)
+                        continue;
+
+                log_debug("%s %zu %zu", type, i, size);
+
+                memzero(buf, MIN(size + 1000, MAX_SIZE));
+
+                r = compress(text, size, buf, size, &j);
+                /* assume compression must be successful except for small or random inputs */
+                assert_se(r == 0 || (size < 2048 && r == -ENOBUFS) || streq(type, "random"));
+
+                /* check for overwrites */
+                assert_se(buf[size] == 0);
+                if (r != 0) {
+                        skipped += size;
+                        continue;
+                }
+
+                assert_se(j > 0);
+                if (j >= size)
+                        log_error("%s \"compressed\" %zu -> %zu", label, size, j);
+
+                r = decompress(buf, j, &buf2, &buf2_allocated, &k, 0);
+                assert_se(r == 0);
+                assert_se(buf2_allocated >= k);
+                assert_se(k == size);
+
+                assert_se(memcmp(text, buf2, size) == 0);
+
+                total += size;
+                compressed += j;
+
+                n2 = now(CLOCK_MONOTONIC);
+                if (n2 - n > arg_duration)
+                        break;
+        }
+
+        dt = (n2-n) / 1e6;
+
+        log_info("%s/%s: compressed & decompressed %zu bytes in %.2fs (%.2fMiB/s), "
+                 "mean compression %.2f%%, skipped %zu bytes",
+                 label, type, total, dt,
+                 total / 1024. / 1024 / dt,
+                 100 - compressed * 100. / total,
+                 skipped);
+}
+#endif
+
+int main(int argc, char *argv[]) {
+#if HAVE_COMPRESSION
+        test_setup_logging(LOG_INFO);
+
+        if (argc >= 2) {
+                unsigned x;
+
+                assert_se(safe_atou(argv[1], &x) >= 0);
+                arg_duration = x * USEC_PER_SEC;
+        } else
+                arg_duration = slow_tests_enabled() ?
+                        2 * USEC_PER_SEC : USEC_PER_SEC / 50;
+
+        if (argc == 3)
+                (void) safe_atozu(argv[2], &arg_start);
+        else
+                arg_start = getpid_cached();
+
+        const char *i;
+        NULSTR_FOREACH(i, "zeros\0simple\0random\0") {
+#if HAVE_XZ
+                test_compress_decompress("XZ", i, compress_blob_xz, decompress_blob_xz);
+#endif
+#if HAVE_LZ4
+                test_compress_decompress("LZ4", i, compress_blob_lz4, decompress_blob_lz4);
+#endif
+#if HAVE_ZSTD
+                test_compress_decompress("ZSTD", i, compress_blob_zstd, decompress_blob_zstd);
+#endif
+        }
+        return 0;
+#else
+        return log_tests_skipped("No compression feature is enabled");
+#endif
+}

src/libsystemd/sd-journal/test-compress.c

@@ -0,0 +1,372 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <sys/stat.h>
+
+#if HAVE_LZ4
+#include <lz4.h>
+#endif
+
+#include "alloc-util.h"
+#include "compress.h"
+#include "fd-util.h"
+#include "fs-util.h"
+#include "macro.h"
+#include "memory-util.h"
+#include "path-util.h"
+#include "random-util.h"
+#include "tests.h"
+#include "tmpfile-util.h"
+
+#if HAVE_XZ
+# define XZ_OK 0
+#else
+# define XZ_OK -EPROTONOSUPPORT
+#endif
+
+#if HAVE_LZ4
+# define LZ4_OK 0
+#else
+# define LZ4_OK -EPROTONOSUPPORT
+#endif
+
+#define HUGE_SIZE (4096*1024)
+
+typedef int (compress_blob_t)(const void *src, uint64_t src_size,
+                              void *dst, size_t dst_alloc_size, size_t *dst_size);
+typedef int (decompress_blob_t)(const void *src, uint64_t src_size,
+                                void **dst, size_t *dst_alloc_size,
+                                size_t* dst_size, size_t dst_max);
+typedef int (decompress_sw_t)(const void *src, uint64_t src_size,
+                              void **buffer, size_t *buffer_size,
+                              const void *prefix, size_t prefix_len,
+                              uint8_t extra);
+
+typedef int (compress_stream_t)(int fdf, int fdt, uint64_t max_bytes);
+typedef int (decompress_stream_t)(int fdf, int fdt, uint64_t max_size);
+
+#if HAVE_COMPRESSION
+_unused_ static void test_compress_decompress(const char *compression,
+                                              compress_blob_t compress,
+                                              decompress_blob_t decompress,
+                                              const char *data,
+                                              size_t data_len,
+                                              bool may_fail) {
+        char compressed[512];
+        size_t csize, usize = 0;
+        _cleanup_free_ char *decompressed = NULL;
+        int r;
+
+        log_info("/* testing %s %s blob compression/decompression */",
+                 compression, data);
+
+        r = compress(data, data_len, compressed, sizeof(compressed), &csize);
+        if (r == -ENOBUFS) {
+                log_info_errno(r, "compression failed: %m");
+                assert_se(may_fail);
+        } else {
+                assert_se(r == 0);
+                r = decompress(compressed, csize,
+                               (void **) &decompressed, &usize, &csize, 0);
+                assert_se(r == 0);
+                assert_se(decompressed);
+                assert_se(memcmp(decompressed, data, data_len) == 0);
+        }
+
+        r = decompress("garbage", 7,
+                       (void **) &decompressed, &usize, &csize, 0);
+        assert_se(r < 0);
+
+        /* make sure to have the minimal lz4 compressed size */
+        r = decompress("00000000\1g", 9,
+                       (void **) &decompressed, &usize, &csize, 0);
+        assert_se(r < 0);
+
+        r = decompress("\100000000g", 9,
+                       (void **) &decompressed, &usize, &csize, 0);
+        assert_se(r < 0);
+
+        memzero(decompressed, usize);
+}
+
+_unused_ static void test_decompress_startswith(const char *compression,
+                                                compress_blob_t compress,
+                                                decompress_sw_t decompress_sw,
+                                                const char *data,
+                                                size_t data_len,
+                                                bool may_fail) {
+
+        char *compressed;
+        _cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL;
+        size_t csize, usize = 0, len;
+        int r;
+
+        log_info("/* testing decompress_startswith with %s on %.20s text */",
+                 compression, data);
+
+#define BUFSIZE_1 512
+#define BUFSIZE_2 20000
+
+        compressed = compressed1 = malloc(BUFSIZE_1);
+        assert_se(compressed1);
+        r = compress(data, data_len, compressed, BUFSIZE_1, &csize);
+        if (r == -ENOBUFS) {
+                log_info_errno(r, "compression failed: %m");
+                assert_se(may_fail);
+
+                compressed = compressed2 = malloc(BUFSIZE_2);
+                assert_se(compressed2);
+                r = compress(data, data_len, compressed, BUFSIZE_2, &csize);
+                assert(r == 0);
+        }
+        assert_se(r == 0);
+
+        len = strlen(data);
+
+        r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
+        assert_se(r > 0);
+        r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, 'w');
+        assert_se(r == 0);
+        r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, "barbarbar", 9, ' ');
+        assert_se(r == 0);
+        r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, data[len-1]);
+        assert_se(r > 0);
+        r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, 'w');
+        assert_se(r == 0);
+        r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
+        assert_se(r > 0);
+}
+
+_unused_ static void test_decompress_startswith_short(const char *compression,
+                                                      compress_blob_t compress,
+                                                      decompress_sw_t decompress_sw) {
+
+#define TEXT "HUGE=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+
+        char buf[1024];
+        size_t i, csize;
+        int r;
+
+        log_info("/* %s with %s */", __func__, compression);
+
+        r = compress(TEXT, sizeof TEXT, buf, sizeof buf, &csize);
+        assert_se(r == 0);
+
+        for (i = 1; i < strlen(TEXT); i++) {
+                size_t alloc_size = i;
+                _cleanup_free_ void *buf2 = NULL;
+
+                assert_se(buf2 = malloc(i));
+
+                assert_se(decompress_sw(buf, csize, &buf2, &alloc_size, TEXT, i, TEXT[i]) == 1);
+                assert_se(decompress_sw(buf, csize, &buf2, &alloc_size, TEXT, i, 'y') == 0);
+        }
+}
+
+_unused_ static void test_compress_stream(const char *compression,
+                                          const char *cat,
+                                          compress_stream_t compress,
+                                          decompress_stream_t decompress,
+                                          const char *srcfile) {
+
+        _cleanup_close_ int src = -1, dst = -1, dst2 = -1;
+        _cleanup_(unlink_tempfilep) char
+                pattern[] = "/tmp/systemd-test.compressed.XXXXXX",
+                pattern2[] = "/tmp/systemd-test.compressed.XXXXXX";
+        int r;
+        _cleanup_free_ char *cmd = NULL, *cmd2 = NULL;
+        struct stat st = {};
+
+        r = find_executable(cat, NULL);
+        if (r < 0) {
+                log_error_errno(r, "Skipping %s, could not find %s binary: %m", __func__, cat);
+                return;
+        }
+
+        log_debug("/* testing %s compression */", compression);
+
+        log_debug("/* create source from %s */", srcfile);
+
+        assert_se((src = open(srcfile, O_RDONLY|O_CLOEXEC)) >= 0);
+
+        log_debug("/* test compression */");
+
+        assert_se((dst = mkostemp_safe(pattern)) >= 0);
+
+        assert_se(compress(src, dst, -1) == 0);
+
+        if (cat) {
+                assert_se(asprintf(&cmd, "%s %s | diff %s -", cat, pattern, srcfile) > 0);
+                assert_se(system(cmd) == 0);
+        }
+
+        log_debug("/* test decompression */");
+
+        assert_se((dst2 = mkostemp_safe(pattern2)) >= 0);
+
+        assert_se(stat(srcfile, &st) == 0);
+
+        assert_se(lseek(dst, 0, SEEK_SET) == 0);
+        r = decompress(dst, dst2, st.st_size);
+        assert_se(r == 0);
+
+        assert_se(asprintf(&cmd2, "diff %s %s", srcfile, pattern2) > 0);
+        assert_se(system(cmd2) == 0);
+
+        log_debug("/* test faulty decompression */");
+
+        assert_se(lseek(dst, 1, SEEK_SET) == 1);
+        r = decompress(dst, dst2, st.st_size);
+        assert_se(IN_SET(r, 0, -EBADMSG));
+
+        assert_se(lseek(dst, 0, SEEK_SET) == 0);
+        assert_se(lseek(dst2, 0, SEEK_SET) == 0);
+        r = decompress(dst, dst2, st.st_size - 1);
+        assert_se(r == -EFBIG);
+}
+#endif
+
+#if HAVE_LZ4
+static void test_lz4_decompress_partial(void) {
+        char buf[20000], buf2[100];
+        size_t buf_size = sizeof(buf), compressed;
+        int r;
+        _cleanup_free_ char *huge = NULL;
+
+        log_debug("/* %s */", __func__);
+
+        assert_se(huge = malloc(HUGE_SIZE));
+        memcpy(huge, "HUGE=", STRLEN("HUGE="));
+        memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
+        huge[HUGE_SIZE - 1] = '\0';
+
+        r = LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size);
+        assert_se(r >= 0);
+        compressed = r;
+        log_info("Compressed %i → %zu", HUGE_SIZE, compressed);
+
+        r = LZ4_decompress_safe(buf, huge, r, HUGE_SIZE);
+        assert_se(r >= 0);
+        log_info("Decompressed → %i", r);
+
+        r = LZ4_decompress_safe_partial(buf, huge,
+                                        compressed,
+                                        12, HUGE_SIZE);
+        assert_se(r >= 0);
+        log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE, r);
+
+        for (size_t size = 1; size < sizeof(buf2); size++) {
+                /* This failed in older lz4s but works in newer ones. */
+                r = LZ4_decompress_safe_partial(buf, buf2, compressed, size, size);
+                log_info("Decompressed partial %zu/%zu → %i (%s)", size, size, r,
+                                                                   r < 0 ? "bad" : "good");
+                if (r >= 0 && LZ4_versionNumber() >= 10803)
+                        /* lz4 <= 1.8.2 should fail that test, let's only check for newer ones */
+                        assert_se(memcmp(buf2, huge, r) == 0);
+        }
+}
+#endif
+
+int main(int argc, char *argv[]) {
+#if HAVE_COMPRESSION
+        _unused_ const char text[] =
+                "text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
+                "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
+
+        /* The file to test compression on can be specified as the first argument */
+        const char *srcfile = argc > 1 ? argv[1] : argv[0];
+
+        char data[512] = "random\0";
+
+        _cleanup_free_ char *huge = NULL;
+
+        assert_se(huge = malloc(HUGE_SIZE));
+        memcpy(huge, "HUGE=", STRLEN("HUGE="));
+        memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
+        huge[HUGE_SIZE - 1] = '\0';
+
+        test_setup_logging(LOG_DEBUG);
+
+        random_bytes(data + 7, sizeof(data) - 7);
+
+#if HAVE_XZ
+        test_compress_decompress("XZ", compress_blob_xz, decompress_blob_xz,
+                                 text, sizeof(text), false);
+        test_compress_decompress("XZ", compress_blob_xz, decompress_blob_xz,
+                                 data, sizeof(data), true);
+
+        test_decompress_startswith("XZ",
+                                   compress_blob_xz, decompress_startswith_xz,
+                                   text, sizeof(text), false);
+        test_decompress_startswith("XZ",
+                                   compress_blob_xz, decompress_startswith_xz,
+                                   data, sizeof(data), true);
+        test_decompress_startswith("XZ",
+                                   compress_blob_xz, decompress_startswith_xz,
+                                   huge, HUGE_SIZE, true);
+
+        test_compress_stream("XZ", "xzcat",
+                             compress_stream_xz, decompress_stream_xz, srcfile);
+
+        test_decompress_startswith_short("XZ", compress_blob_xz, decompress_startswith_xz);
+
+#else
+        log_info("/* XZ test skipped */");
+#endif
+
+#if HAVE_LZ4
+        test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4,
+                                 text, sizeof(text), false);
+        test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4,
+                                 data, sizeof(data), true);
+
+        test_decompress_startswith("LZ4",
+                                   compress_blob_lz4, decompress_startswith_lz4,
+                                   text, sizeof(text), false);
+        test_decompress_startswith("LZ4",
+                                   compress_blob_lz4, decompress_startswith_lz4,
+                                   data, sizeof(data), true);
+        test_decompress_startswith("LZ4",
+                                   compress_blob_lz4, decompress_startswith_lz4,
+                                   huge, HUGE_SIZE, true);
+
+        test_compress_stream("LZ4", "lz4cat",
+                             compress_stream_lz4, decompress_stream_lz4, srcfile);
+
+        test_lz4_decompress_partial();
+
+        test_decompress_startswith_short("LZ4", compress_blob_lz4, decompress_startswith_lz4);
+
+#else
+        log_info("/* LZ4 test skipped */");
+#endif
+
+#if HAVE_ZSTD
+        test_compress_decompress("ZSTD", compress_blob_zstd, decompress_blob_zstd,
+                                 text, sizeof(text), false);
+        test_compress_decompress("ZSTD", compress_blob_zstd, decompress_blob_zstd,
+                                 data, sizeof(data), true);
+
+        test_decompress_startswith("ZSTD",
+                                   compress_blob_zstd, decompress_startswith_zstd,
+                                   text, sizeof(text), false);
+        test_decompress_startswith("ZSTD",
+                                   compress_blob_zstd, decompress_startswith_zstd,
+                                   data, sizeof(data), true);
+        test_decompress_startswith("ZSTD",
+                                   compress_blob_zstd, decompress_startswith_zstd,
+                                   huge, HUGE_SIZE, true);
+
+        test_compress_stream("ZSTD", "zstdcat",
+                             compress_stream_zstd, decompress_stream_zstd, srcfile);
+
+        test_decompress_startswith_short("ZSTD", compress_blob_zstd, decompress_startswith_zstd);
+#else
+        log_info("/* ZSTD test skipped */");
+#endif
+
+        return 0;
+#else
+        log_info("/* XZ, LZ4 and ZSTD tests skipped */");
+        return EXIT_TEST_SKIP;
+#endif
+}

src/libsystemd/sd-journal/test-journal-enum.c

@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <stdio.h>
+
+#include "sd-journal.h"
+
+#include "journal-internal.h"
+#include "log.h"
+#include "macro.h"
+#include "tests.h"
+
+int main(int argc, char *argv[]) {
+        unsigned n = 0;
+        _cleanup_(sd_journal_closep) sd_journal *j = NULL;
+
+        test_setup_logging(LOG_DEBUG);
+
+        assert_se(sd_journal_open(&j, SD_JOURNAL_LOCAL_ONLY) >= 0);
+
+        assert_se(sd_journal_add_match(j, "_TRANSPORT=syslog", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "_UID=0", 0) >= 0);
+
+        SD_JOURNAL_FOREACH_BACKWARDS(j) {
+                const void *d;
+                size_t l;
+
+                assert_se(sd_journal_get_data(j, "MESSAGE", &d, &l) >= 0);
+
+                printf("%.*s\n", (int) l, (char*) d);
+
+                n++;
+                if (n >= 10)
+                        break;
+        }
+
+        return 0;
+}

src/libsystemd/sd-journal/test-journal-flush.c

@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "sd-journal.h"
+
+#include "alloc-util.h"
+#include "chattr-util.h"
+#include "journal-file.h"
+#include "journal-internal.h"
+#include "macro.h"
+#include "path-util.h"
+#include "string-util.h"
+
+int main(int argc, char *argv[]) {
+        _cleanup_free_ char *fn = NULL;
+        char dn[] = "/var/tmp/test-journal-flush.XXXXXX";
+        JournalFile *new_journal = NULL;
+        sd_journal *j = NULL;
+        unsigned n = 0;
+        int r;
+
+        assert_se(mkdtemp(dn));
+        (void) chattr_path(dn, FS_NOCOW_FL, FS_NOCOW_FL, NULL);
+
+        fn = path_join(dn, "test.journal");
+
+        r = journal_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, 0, false, NULL, NULL, NULL, NULL, &new_journal);
+        assert_se(r >= 0);
+
+        r = sd_journal_open(&j, 0);
+        assert_se(r >= 0);
+
+        sd_journal_set_data_threshold(j, 0);
+
+        SD_JOURNAL_FOREACH(j) {
+                Object *o;
+                JournalFile *f;
+
+                f = j->current_file;
+                assert_se(f && f->current_offset > 0);
+
+                r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
+                if (r < 0)
+                        log_error_errno(r, "journal_file_move_to_object failed: %m");
+                assert_se(r >= 0);
+
+                r = journal_file_copy_entry(f, new_journal, o, f->current_offset);
+                if (r < 0)
+                        log_error_errno(r, "journal_file_copy_entry failed: %m");
+                assert_se(r >= 0);
+
+                if (++n >= 10000)
+                        break;
+        }
+
+        sd_journal_close(j);
+
+        (void) journal_file_close(new_journal);
+
+        unlink(fn);
+        assert_se(rmdir(dn) == 0);
+
+        return 0;
+}

src/libsystemd/sd-journal/test-journal-init.c

@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include "sd-journal.h"
+
+#include "chattr-util.h"
+#include "log.h"
+#include "parse-util.h"
+#include "rm-rf.h"
+#include "tests.h"
+#include "util.h"
+
+int main(int argc, char *argv[]) {
+        sd_journal *j;
+        int r, i, I = 100;
+        char t[] = "/var/tmp/journal-stream-XXXXXX";
+
+        test_setup_logging(LOG_DEBUG);
+
+        if (argc >= 2) {
+                r = safe_atoi(argv[1], &I);
+                if (r < 0)
+                        log_info("Could not parse loop count argument. Using default.");
+        }
+
+        log_info("Running %d loops", I);
+
+        assert_se(mkdtemp(t));
+        (void) chattr_path(t, FS_NOCOW_FL, FS_NOCOW_FL, NULL);
+
+        for (i = 0; i < I; i++) {
+                r = sd_journal_open(&j, SD_JOURNAL_LOCAL_ONLY);
+                assert_se(r == 0);
+
+                sd_journal_close(j);
+
+                r = sd_journal_open_directory(&j, t, 0);
+                assert_se(r == 0);
+
+                sd_journal_close(j);
+
+                j = NULL;
+                r = sd_journal_open_directory(&j, t, SD_JOURNAL_LOCAL_ONLY);
+                assert_se(r == -EINVAL);
+                assert_se(j == NULL);
+        }
+
+        assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
+
+        return 0;
+}

src/libsystemd/sd-journal/test-journal-interleaving.c

@@ -0,0 +1,296 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "sd-journal.h"
+
+#include "alloc-util.h"
+#include "chattr-util.h"
+#include "io-util.h"
+#include "journal-file.h"
+#include "journal-vacuum.h"
+#include "log.h"
+#include "parse-util.h"
+#include "rm-rf.h"
+#include "tests.h"
+#include "util.h"
+
+/* This program tests skipping around in a multi-file journal. */
+
+static bool arg_keep = false;
+
+_noreturn_ static void log_assert_errno(const char *text, int error, const char *file, int line, const char *func) {
+        log_internal(LOG_CRIT, error, file, line, func,
+                     "'%s' failed at %s:%u (%s): %m", text, file, line, func);
+        abort();
+}
+
+#define assert_ret(expr)                                                \
+        do {                                                            \
+                int _r_ = (expr);                                       \
+                if (_unlikely_(_r_ < 0))                                \
+                        log_assert_errno(#expr, -_r_, PROJECT_FILE, __LINE__, __PRETTY_FUNCTION__); \
+        } while (false)
+
+static JournalFile *test_open(const char *name) {
+        JournalFile *f;
+        assert_ret(journal_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f));
+        return f;
+}
+
+static void test_close(JournalFile *f) {
+        (void) journal_file_close (f);
+}
+
+static void append_number(JournalFile *f, int n, uint64_t *seqnum) {
+        char *p;
+        dual_timestamp ts;
+        static dual_timestamp previous_ts = {};
+        struct iovec iovec[1];
+
+        dual_timestamp_get(&ts);
+
+        if (ts.monotonic <= previous_ts.monotonic)
+                ts.monotonic = previous_ts.monotonic + 1;
+
+        if (ts.realtime <= previous_ts.realtime)
+                ts.realtime = previous_ts.realtime + 1;
+
+        previous_ts = ts;
+
+        assert_se(asprintf(&p, "NUMBER=%d", n) >= 0);
+        iovec[0] = IOVEC_MAKE_STRING(p);
+        assert_ret(journal_file_append_entry(f, &ts, NULL, iovec, 1, seqnum, NULL, NULL));
+        free(p);
+}
+
+static void test_check_number (sd_journal *j, int n) {
+        const void *d;
+        _cleanup_free_ char *k;
+        size_t l;
+        int x;
+
+        assert_ret(sd_journal_get_data(j, "NUMBER", &d, &l));
+        assert_se(k = strndup(d, l));
+        printf("%s\n", k);
+
+        assert_se(safe_atoi(k + 7, &x) >= 0);
+        assert_se(n == x);
+}
+
+static void test_check_numbers_down (sd_journal *j, int count) {
+        int i;
+
+        for (i = 1; i <= count; i++) {
+                int r;
+                test_check_number(j, i);
+                assert_ret(r = sd_journal_next(j));
+                if (i == count)
+                        assert_se(r == 0);
+                else
+                        assert_se(r == 1);
+        }
+
+}
+
+static void test_check_numbers_up (sd_journal *j, int count) {
+        for (int i = count; i >= 1; i--) {
+                int r;
+                test_check_number(j, i);
+                assert_ret(r = sd_journal_previous(j));
+                if (i == 1)
+                        assert_se(r == 0);
+                else
+                        assert_se(r == 1);
+        }
+
+}
+
+static void setup_sequential(void) {
+        JournalFile *one, *two;
+        one = test_open("one.journal");
+        two = test_open("two.journal");
+        append_number(one, 1, NULL);
+        append_number(one, 2, NULL);
+        append_number(two, 3, NULL);
+        append_number(two, 4, NULL);
+        test_close(one);
+        test_close(two);
+}
+
+static void setup_interleaved(void) {
+        JournalFile *one, *two;
+        one = test_open("one.journal");
+        two = test_open("two.journal");
+        append_number(one, 1, NULL);
+        append_number(two, 2, NULL);
+        append_number(one, 3, NULL);
+        append_number(two, 4, NULL);
+        test_close(one);
+        test_close(two);
+}
+
+static void mkdtemp_chdir_chattr(char *path) {
+        assert_se(mkdtemp(path));
+        assert_se(chdir(path) >= 0);
+
+        /* Speed up things a bit on btrfs, ensuring that CoW is turned off for all files created in our
+         * directory during the test run */
+        (void) chattr_path(path, FS_NOCOW_FL, FS_NOCOW_FL, NULL);
+}
+
+static void test_skip(void (*setup)(void)) {
+        char t[] = "/var/tmp/journal-skip-XXXXXX";
+        sd_journal *j;
+        int r;
+
+        mkdtemp_chdir_chattr(t);
+
+        setup();
+
+        /* Seek to head, iterate down.
+         */
+        assert_ret(sd_journal_open_directory(&j, t, 0));
+        assert_ret(sd_journal_seek_head(j));
+        assert_ret(sd_journal_next(j));
+        test_check_numbers_down(j, 4);
+        sd_journal_close(j);
+
+        /* Seek to tail, iterate up.
+         */
+        assert_ret(sd_journal_open_directory(&j, t, 0));
+        assert_ret(sd_journal_seek_tail(j));
+        assert_ret(sd_journal_previous(j));
+        test_check_numbers_up(j, 4);
+        sd_journal_close(j);
+
+        /* Seek to tail, skip to head, iterate down.
+         */
+        assert_ret(sd_journal_open_directory(&j, t, 0));
+        assert_ret(sd_journal_seek_tail(j));
+        assert_ret(r = sd_journal_previous_skip(j, 4));
+        assert_se(r == 4);
+        test_check_numbers_down(j, 4);
+        sd_journal_close(j);
+
+        /* Seek to head, skip to tail, iterate up.
+         */
+        assert_ret(sd_journal_open_directory(&j, t, 0));
+        assert_ret(sd_journal_seek_head(j));
+        assert_ret(r = sd_journal_next_skip(j, 4));
+        assert_se(r == 4);
+        test_check_numbers_up(j, 4);
+        sd_journal_close(j);
+
+        log_info("Done...");
+
+        if (arg_keep)
+                log_info("Not removing %s", t);
+        else {
+                journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
+
+                assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
+        }
+
+        puts("------------------------------------------------------------");
+}
+
+static void test_sequence_numbers(void) {
+
+        char t[] = "/var/tmp/journal-seq-XXXXXX";
+        JournalFile *one, *two;
+        uint64_t seqnum = 0;
+        sd_id128_t seqnum_id;
+
+        mkdtemp_chdir_chattr(t);
+
+        assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0644,
+                                    true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &one) == 0);
+
+        append_number(one, 1, &seqnum);
+        printf("seqnum=%"PRIu64"\n", seqnum);
+        assert_se(seqnum == 1);
+        append_number(one, 2, &seqnum);
+        printf("seqnum=%"PRIu64"\n", seqnum);
+        assert_se(seqnum == 2);
+
+        assert_se(one->header->state == STATE_ONLINE);
+        assert_se(!sd_id128_equal(one->header->file_id, one->header->machine_id));
+        assert_se(!sd_id128_equal(one->header->file_id, one->header->boot_id));
+        assert_se(sd_id128_equal(one->header->file_id, one->header->seqnum_id));
+
+        memcpy(&seqnum_id, &one->header->seqnum_id, sizeof(sd_id128_t));
+
+        assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0644,
+                                    true, (uint64_t) -1, false, NULL, NULL, NULL, one, &two) == 0);
+
+        assert_se(two->header->state == STATE_ONLINE);
+        assert_se(!sd_id128_equal(two->header->file_id, one->header->file_id));
+        assert_se(sd_id128_equal(one->header->machine_id, one->header->machine_id));
+        assert_se(sd_id128_equal(one->header->boot_id, one->header->boot_id));
+        assert_se(sd_id128_equal(one->header->seqnum_id, one->header->seqnum_id));
+
+        append_number(two, 3, &seqnum);
+        printf("seqnum=%"PRIu64"\n", seqnum);
+        assert_se(seqnum == 3);
+        append_number(two, 4, &seqnum);
+        printf("seqnum=%"PRIu64"\n", seqnum);
+        assert_se(seqnum == 4);
+
+        test_close(two);
+
+        append_number(one, 5, &seqnum);
+        printf("seqnum=%"PRIu64"\n", seqnum);
+        assert_se(seqnum == 5);
+
+        append_number(one, 6, &seqnum);
+        printf("seqnum=%"PRIu64"\n", seqnum);
+        assert_se(seqnum == 6);
+
+        test_close(one);
+
+        /* restart server */
+        seqnum = 0;
+
+        assert_se(journal_file_open(-1, "two.journal", O_RDWR, 0,
+                                    true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &two) == 0);
+
+        assert_se(sd_id128_equal(two->header->seqnum_id, seqnum_id));
+
+        append_number(two, 7, &seqnum);
+        printf("seqnum=%"PRIu64"\n", seqnum);
+        assert_se(seqnum == 5);
+
+        /* So..., here we have the same seqnum in two files with the
+         * same seqnum_id. */
+
+        test_close(two);
+
+        log_info("Done...");
+
+        if (arg_keep)
+                log_info("Not removing %s", t);
+        else {
+                journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
+
+                assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
+        }
+}
+
+int main(int argc, char *argv[]) {
+        test_setup_logging(LOG_DEBUG);
+
+        /* journal_file_open requires a valid machine id */
+        if (access("/etc/machine-id", F_OK) != 0)
+                return log_tests_skipped("/etc/machine-id not found");
+
+        arg_keep = argc > 1;
+
+        test_skip(setup_sequential);
+        test_skip(setup_interleaved);
+
+        test_sequence_numbers();
+
+        return 0;
+}

src/libsystemd/sd-journal/test-journal-match.c

@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <stdio.h>
+
+#include "sd-journal.h"
+
+#include "alloc-util.h"
+#include "journal-internal.h"
+#include "log.h"
+#include "string-util.h"
+#include "tests.h"
+#include "util.h"
+
+int main(int argc, char *argv[]) {
+        _cleanup_(sd_journal_closep) sd_journal *j = NULL;
+        _cleanup_free_ char *t;
+
+        test_setup_logging(LOG_DEBUG);
+
+        assert_se(sd_journal_open(&j, 0) >= 0);
+
+        assert_se(sd_journal_add_match(j, "foobar", 0) < 0);
+        assert_se(sd_journal_add_match(j, "foobar=waldo", 0) < 0);
+        assert_se(sd_journal_add_match(j, "", 0) < 0);
+        assert_se(sd_journal_add_match(j, "=", 0) < 0);
+        assert_se(sd_journal_add_match(j, "=xxxxx", 0) < 0);
+        assert_se(sd_journal_add_match(j, (uint8_t[4]){'A', '=', '\1', '\2'}, 4) >= 0);
+        assert_se(sd_journal_add_match(j, (uint8_t[5]){'B', '=', 'C', '\0', 'D'}, 5) >= 0);
+        assert_se(sd_journal_add_match(j, "HALLO=WALDO", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "QUUX=mmmm", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "QUUX=xxxxx", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "HALLO=", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "QUUX=xxxxx", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "QUUX=yyyyy", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "PIFF=paff", 0) >= 0);
+
+        assert_se(sd_journal_add_disjunction(j) >= 0);
+
+        assert_se(sd_journal_add_match(j, "ONE=one", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "ONE=two", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "TWO=two", 0) >= 0);
+
+        assert_se(sd_journal_add_conjunction(j) >= 0);
+
+        assert_se(sd_journal_add_match(j, "L4_1=yes", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "L4_1=ok", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "L4_2=yes", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "L4_2=ok", 0) >= 0);
+
+        assert_se(sd_journal_add_disjunction(j) >= 0);
+
+        assert_se(sd_journal_add_match(j, "L3=yes", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "L3=ok", 0) >= 0);
+
+        assert_se(t = journal_make_match_string(j));
+
+        printf("resulting match expression is: %s\n", t);
+
+        assert_se(streq(t, "(((L3=ok OR L3=yes) OR ((L4_2=ok OR L4_2=yes) AND (L4_1=ok OR L4_1=yes))) AND ((TWO=two AND (ONE=two OR ONE=one)) OR (PIFF=paff AND (QUUX=yyyyy OR QUUX=xxxxx OR QUUX=mmmm) AND (HALLO= OR HALLO=WALDO) AND B=C\\000D AND A=\\001\\002)))"));
+
+        return 0;
+}

src/libsystemd/sd-journal/test-journal-send.c

@@ -0,0 +1,103 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <errno.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "sd-journal.h"
+#include "fileio.h"
+#include "macro.h"
+#include "memory-util.h"
+
+static void test_journal_print(void) {
+        assert_se(sd_journal_print(LOG_INFO, "XXX") == 0);
+        assert_se(sd_journal_print(LOG_INFO, "%s", "YYY") == 0);
+        assert_se(sd_journal_print(LOG_INFO, "X%4094sY", "ZZZ") == 0);
+        assert_se(sd_journal_print(LOG_INFO, "X%*sY", LONG_LINE_MAX - 8 - 3, "ZZZ") == 0);
+        assert_se(sd_journal_print(LOG_INFO, "X%*sY", LONG_LINE_MAX - 8 - 2, "ZZZ") == -ENOBUFS);
+}
+
+static void test_journal_send(void) {
+        _cleanup_free_ char *huge = NULL;
+
+#define HUGE_SIZE (4096*1024)
+        assert_se(huge = malloc(HUGE_SIZE));
+
+        /* utf-8 and non-utf-8, message-less and message-ful iovecs */
+        struct iovec graph1[] = {
+                {(char*) "GRAPH=graph", STRLEN("GRAPH=graph")}
+        };
+        struct iovec graph2[] = {
+                {(char*) "GRAPH=graph\n", STRLEN("GRAPH=graph\n")}
+        };
+        struct iovec message1[] = {
+                {(char*) "MESSAGE=graph", STRLEN("MESSAGE=graph")}
+        };
+        struct iovec message2[] = {
+                {(char*) "MESSAGE=graph\n", STRLEN("MESSAGE=graph\n")}
+        };
+
+        assert_se(sd_journal_print(LOG_INFO, "piepapo") == 0);
+
+        assert_se(sd_journal_send("MESSAGE=foobar",
+                                  "VALUE=%i", 7,
+                                  NULL) == 0);
+
+        errno = ENOENT;
+        assert_se(sd_journal_perror("Foobar") == 0);
+
+        assert_se(sd_journal_perror("") == 0);
+
+        memcpy(huge, "HUGE=", STRLEN("HUGE="));
+        memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
+        huge[HUGE_SIZE - 1] = '\0';
+
+        assert_se(sd_journal_send("MESSAGE=Huge field attached",
+                                  huge,
+                                  NULL) == 0);
+
+        assert_se(sd_journal_send("MESSAGE=uiui",
+                                  "VALUE=A",
+                                  "VALUE=B",
+                                  "VALUE=C",
+                                  "SINGLETON=1",
+                                  "OTHERVALUE=X",
+                                  "OTHERVALUE=Y",
+                                  "WITH_BINARY=this is a binary value \a",
+                                  NULL) == 0);
+
+        syslog(LOG_NOTICE, "Hello World!");
+
+        assert_se(sd_journal_print(LOG_NOTICE, "Hello World") == 0);
+
+        assert_se(sd_journal_send("MESSAGE=Hello World!",
+                                  "MESSAGE_ID=52fb62f99e2c49d89cfbf9d6de5e3555",
+                                  "PRIORITY=5",
+                                  "HOME=%s", getenv("HOME"),
+                                  "TERM=%s", getenv("TERM"),
+                                  "PAGE_SIZE=%li", sysconf(_SC_PAGESIZE),
+                                  "N_CPUS=%li", sysconf(_SC_NPROCESSORS_ONLN),
+                                  NULL) == 0);
+
+        assert_se(sd_journal_sendv(graph1, 1) == 0);
+        assert_se(sd_journal_sendv(graph2, 1) == 0);
+        assert_se(sd_journal_sendv(message1, 1) == 0);
+        assert_se(sd_journal_sendv(message2, 1) == 0);
+
+        /* test without location fields */
+#undef sd_journal_sendv
+        assert_se(sd_journal_sendv(graph1, 1) == 0);
+        assert_se(sd_journal_sendv(graph2, 1) == 0);
+        assert_se(sd_journal_sendv(message1, 1) == 0);
+        assert_se(sd_journal_sendv(message2, 1) == 0);
+}
+
+int main(int argc, char *argv[]) {
+        test_journal_print();
+        test_journal_send();
+
+        /* Sleep a bit to make it easy for journald to collect metadata. */
+        sleep(1);
+
+        return 0;
+}

src/libsystemd/sd-journal/test-journal-stream.c

@@ -0,0 +1,191 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "sd-journal.h"
+
+#include "alloc-util.h"
+#include "chattr-util.h"
+#include "io-util.h"
+#include "journal-file.h"
+#include "journal-internal.h"
+#include "log.h"
+#include "macro.h"
+#include "parse-util.h"
+#include "rm-rf.h"
+#include "tests.h"
+#include "util.h"
+
+#define N_ENTRIES 200
+
+static void verify_contents(sd_journal *j, unsigned skip) {
+        unsigned i;
+
+        assert_se(j);
+
+        i = 0;
+        SD_JOURNAL_FOREACH(j) {
+                const void *d;
+                char *k, *c;
+                size_t l;
+                unsigned u = 0;
+
+                assert_se(sd_journal_get_cursor(j, &k) >= 0);
+                printf("cursor: %s\n", k);
+                free(k);
+
+                assert_se(sd_journal_get_data(j, "MAGIC", &d, &l) >= 0);
+                printf("\t%.*s\n", (int) l, (const char*) d);
+
+                assert_se(sd_journal_get_data(j, "NUMBER", &d, &l) >= 0);
+                assert_se(k = strndup(d, l));
+                printf("\t%s\n", k);
+
+                if (skip > 0) {
+                        assert_se(safe_atou(k + 7, &u) >= 0);
+                        assert_se(i == u);
+                        i += skip;
+                }
+
+                free(k);
+
+                assert_se(sd_journal_get_cursor(j, &c) >= 0);
+                assert_se(sd_journal_test_cursor(j, c) > 0);
+                free(c);
+        }
+
+        if (skip > 0)
+                assert_se(i == N_ENTRIES);
+}
+
+static void run_test(void) {
+        JournalFile *one, *two, *three;
+        char t[] = "/var/tmp/journal-stream-XXXXXX";
+        unsigned i;
+        _cleanup_(sd_journal_closep) sd_journal *j = NULL;
+        char *z;
+        const void *data;
+        size_t l;
+        dual_timestamp previous_ts = DUAL_TIMESTAMP_NULL;
+
+        assert_se(mkdtemp(t));
+        assert_se(chdir(t) >= 0);
+        (void) chattr_path(t, FS_NOCOW_FL, FS_NOCOW_FL, NULL);
+
+        assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &one) == 0);
+        assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &two) == 0);
+        assert_se(journal_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &three) == 0);
+
+        for (i = 0; i < N_ENTRIES; i++) {
+                char *p, *q;
+                dual_timestamp ts;
+                struct iovec iovec[2];
+
+                dual_timestamp_get(&ts);
+
+                if (ts.monotonic <= previous_ts.monotonic)
+                        ts.monotonic = previous_ts.monotonic + 1;
+
+                if (ts.realtime <= previous_ts.realtime)
+                        ts.realtime = previous_ts.realtime + 1;
+
+                previous_ts = ts;
+
+                assert_se(asprintf(&p, "NUMBER=%u", i) >= 0);
+                iovec[0] = IOVEC_MAKE(p, strlen(p));
+
+                assert_se(asprintf(&q, "MAGIC=%s", i % 5 == 0 ? "quux" : "waldo") >= 0);
+
+                iovec[1] = IOVEC_MAKE(q, strlen(q));
+
+                if (i % 10 == 0)
+                        assert_se(journal_file_append_entry(three, &ts, NULL, iovec, 2, NULL, NULL, NULL) == 0);
+                else {
+                        if (i % 3 == 0)
+                                assert_se(journal_file_append_entry(two, &ts, NULL, iovec, 2, NULL, NULL, NULL) == 0);
+
+                        assert_se(journal_file_append_entry(one, &ts, NULL, iovec, 2, NULL, NULL, NULL) == 0);
+                }
+
+                free(p);
+                free(q);
+        }
+
+        (void) journal_file_close(one);
+        (void) journal_file_close(two);
+        (void) journal_file_close(three);
+
+        assert_se(sd_journal_open_directory(&j, t, 0) >= 0);
+
+        assert_se(sd_journal_add_match(j, "MAGIC=quux", 0) >= 0);
+        SD_JOURNAL_FOREACH_BACKWARDS(j) {
+                _cleanup_free_ char *c;
+
+                assert_se(sd_journal_get_data(j, "NUMBER", &data, &l) >= 0);
+                printf("\t%.*s\n", (int) l, (const char*) data);
+
+                assert_se(sd_journal_get_cursor(j, &c) >= 0);
+                assert_se(sd_journal_test_cursor(j, c) > 0);
+        }
+
+        SD_JOURNAL_FOREACH(j) {
+                _cleanup_free_ char *c;
+
+                assert_se(sd_journal_get_data(j, "NUMBER", &data, &l) >= 0);
+                printf("\t%.*s\n", (int) l, (const char*) data);
+
+                assert_se(sd_journal_get_cursor(j, &c) >= 0);
+                assert_se(sd_journal_test_cursor(j, c) > 0);
+        }
+
+        sd_journal_flush_matches(j);
+
+        verify_contents(j, 1);
+
+        printf("NEXT TEST\n");
+        assert_se(sd_journal_add_match(j, "MAGIC=quux", 0) >= 0);
+
+        assert_se(z = journal_make_match_string(j));
+        printf("resulting match expression is: %s\n", z);
+        free(z);
+
+        verify_contents(j, 5);
+
+        printf("NEXT TEST\n");
+        sd_journal_flush_matches(j);
+        assert_se(sd_journal_add_match(j, "MAGIC=waldo", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "NUMBER=10", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "NUMBER=11", 0) >= 0);
+        assert_se(sd_journal_add_match(j, "NUMBER=12", 0) >= 0);
+
+        assert_se(z = journal_make_match_string(j));
+        printf("resulting match expression is: %s\n", z);
+        free(z);
+
+        verify_contents(j, 0);
+
+        assert_se(sd_journal_query_unique(j, "NUMBER") >= 0);
+        SD_JOURNAL_FOREACH_UNIQUE(j, data, l)
+                printf("%.*s\n", (int) l, (const char*) data);
+
+        assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
+}
+
+int main(int argc, char *argv[]) {
+
+        /* journal_file_open requires a valid machine id */
+        if (access("/etc/machine-id", F_OK) != 0)
+                return log_tests_skipped("/etc/machine-id not found");
+
+        test_setup_logging(LOG_DEBUG);
+
+        /* Run this test twice. Once with old hashing and once with new hashing */
+        assert_se(setenv("SYSTEMD_JOURNAL_KEYED_HASH", "1", 1) >= 0);
+        run_test();
+
+        assert_se(setenv("SYSTEMD_JOURNAL_KEYED_HASH", "0", 1) >= 0);
+        run_test();
+
+        return 0;
+}

src/libsystemd/sd-journal/test-journal-verify.c

@@ -0,0 +1,136 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#include "chattr-util.h"
+#include "fd-util.h"
+#include "io-util.h"
+#include "journal-file.h"
+#include "journal-verify.h"
+#include "log.h"
+#include "rm-rf.h"
+#include "terminal-util.h"
+#include "tests.h"
+#include "util.h"
+
+#define N_ENTRIES 6000
+#define RANDOM_RANGE 77
+
+static void bit_toggle(const char *fn, uint64_t p) {
+        uint8_t b;
+        ssize_t r;
+        int fd;
+
+        fd = open(fn, O_RDWR|O_CLOEXEC);
+        assert_se(fd >= 0);
+
+        r = pread(fd, &b, 1, p/8);
+        assert_se(r == 1);
+
+        b ^= 1 << (p % 8);
+
+        r = pwrite(fd, &b, 1, p/8);
+        assert_se(r == 1);
+
+        safe_close(fd);
+}
+
+static int raw_verify(const char *fn, const char *verification_key) {
+        JournalFile *f;
+        int r;
+
+        r = journal_file_open(-1, fn, O_RDONLY, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f);
+        if (r < 0)
+                return r;
+
+        r = journal_file_verify(f, verification_key, NULL, NULL, NULL, false);
+        (void) journal_file_close(f);
+
+        return r;
+}
+
+int main(int argc, char *argv[]) {
+        char t[] = "/var/tmp/journal-XXXXXX";
+        unsigned n;
+        JournalFile *f;
+        const char *verification_key = argv[1];
+        usec_t from = 0, to = 0, total = 0;
+        char a[FORMAT_TIMESTAMP_MAX];
+        char b[FORMAT_TIMESTAMP_MAX];
+        char c[FORMAT_TIMESPAN_MAX];
+        struct stat st;
+        uint64_t p;
+
+        /* journal_file_open requires a valid machine id */
+        if (access("/etc/machine-id", F_OK) != 0)
+                return log_tests_skipped("/etc/machine-id not found");
+
+        test_setup_logging(LOG_DEBUG);
+
+        assert_se(mkdtemp(t));
+        assert_se(chdir(t) >= 0);
+        (void) chattr_path(t, FS_NOCOW_FL, FS_NOCOW_FL, NULL);
+
+        log_info("Generating...");
+
+        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
+
+        for (n = 0; n < N_ENTRIES; n++) {
+                struct iovec iovec;
+                struct dual_timestamp ts;
+                char *test;
+
+                dual_timestamp_get(&ts);
+
+                assert_se(asprintf(&test, "RANDOM=%lu", random() % RANDOM_RANGE));
+
+                iovec = IOVEC_MAKE_STRING(test);
+
+                assert_se(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0);
+
+                free(test);
+        }
+
+        (void) journal_file_close(f);
+
+        log_info("Verifying...");
+
+        assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
+        /* journal_file_print_header(f); */
+        journal_file_dump(f);
+
+        assert_se(journal_file_verify(f, verification_key, &from, &to, &total, true) >= 0);
+
+        if (verification_key && JOURNAL_HEADER_SEALED(f->header))
+                log_info("=> Validated from %s to %s, %s missing",
+                         format_timestamp(a, sizeof(a), from),
+                         format_timestamp(b, sizeof(b), to),
+                         format_timespan(c, sizeof(c), total > to ? total - to : 0, 0));
+
+        (void) journal_file_close(f);
+
+        if (verification_key) {
+                log_info("Toggling bits...");
+
+                assert_se(stat("test.journal", &st) >= 0);
+
+                for (p = 38448*8+0; p < ((uint64_t) st.st_size * 8); p ++) {
+                        bit_toggle("test.journal", p);
+
+                        log_info("[ %"PRIu64"+%"PRIu64"]", p / 8, p % 8);
+
+                        if (raw_verify("test.journal", verification_key) >= 0)
+                                log_notice(ANSI_HIGHLIGHT_RED ">>>> %"PRIu64" (bit %"PRIu64") can be toggled without detection." ANSI_NORMAL, p / 8, p % 8);
+
+                        bit_toggle("test.journal", p);
+                }
+        }
+
+        log_info("Exiting...");
+
+        assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
+
+        return 0;
+}

src/libsystemd/sd-journal/test-journal.c

@@ -0,0 +1,259 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "chattr-util.h"
+#include "io-util.h"
+#include "journal-authenticate.h"
+#include "journal-file.h"
+#include "journal-vacuum.h"
+#include "log.h"
+#include "rm-rf.h"
+#include "tests.h"
+
+static bool arg_keep = false;
+
+static void mkdtemp_chdir_chattr(char *path) {
+        assert_se(mkdtemp(path));
+        assert_se(chdir(path) >= 0);
+
+        /* Speed up things a bit on btrfs, ensuring that CoW is turned off for all files created in our
+         * directory during the test run */
+        (void) chattr_path(path, FS_NOCOW_FL, FS_NOCOW_FL, NULL);
+}
+
+static void test_non_empty(void) {
+        dual_timestamp ts;
+        JournalFile *f;
+        struct iovec iovec;
+        static const char test[] = "TEST1=1", test2[] = "TEST2=2";
+        Object *o;
+        uint64_t p;
+        sd_id128_t fake_boot_id;
+        char t[] = "/var/tmp/journal-XXXXXX";
+
+        test_setup_logging(LOG_DEBUG);
+
+        mkdtemp_chdir_chattr(t);
+
+        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f) == 0);
+
+        assert_se(dual_timestamp_get(&ts));
+        assert_se(sd_id128_randomize(&fake_boot_id) == 0);
+
+        iovec = IOVEC_MAKE_STRING(test);
+        assert_se(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0);
+
+        iovec = IOVEC_MAKE_STRING(test2);
+        assert_se(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0);
+
+        iovec = IOVEC_MAKE_STRING(test);
+        assert_se(journal_file_append_entry(f, &ts, &fake_boot_id, &iovec, 1, NULL, NULL, NULL) == 0);
+
+#if HAVE_GCRYPT
+        journal_file_append_tag(f);
+#endif
+        journal_file_dump(f);
+
+        assert_se(journal_file_next_entry(f, 0, DIRECTION_DOWN, &o, &p) == 1);
+        assert_se(le64toh(o->entry.seqnum) == 1);
+
+        assert_se(journal_file_next_entry(f, p, DIRECTION_DOWN, &o, &p) == 1);
+        assert_se(le64toh(o->entry.seqnum) == 2);
+
+        assert_se(journal_file_next_entry(f, p, DIRECTION_DOWN, &o, &p) == 1);
+        assert_se(le64toh(o->entry.seqnum) == 3);
+        assert_se(sd_id128_equal(o->entry.boot_id, fake_boot_id));
+
+        assert_se(journal_file_next_entry(f, p, DIRECTION_DOWN, &o, &p) == 0);
+
+        assert_se(journal_file_next_entry(f, 0, DIRECTION_DOWN, &o, &p) == 1);
+        assert_se(le64toh(o->entry.seqnum) == 1);
+
+        assert_se(journal_file_find_data_object(f, test, strlen(test), NULL, &p) == 1);
+        assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_DOWN, &o, NULL) == 1);
+        assert_se(le64toh(o->entry.seqnum) == 1);
+
+        assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_UP, &o, NULL) == 1);
+        assert_se(le64toh(o->entry.seqnum) == 3);
+
+        assert_se(journal_file_find_data_object(f, test2, strlen(test2), NULL, &p) == 1);
+        assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_UP, &o, NULL) == 1);
+        assert_se(le64toh(o->entry.seqnum) == 2);
+
+        assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_DOWN, &o, NULL) == 1);
+        assert_se(le64toh(o->entry.seqnum) == 2);
+
+        assert_se(journal_file_find_data_object(f, "quux", 4, NULL, &p) == 0);
+
+        assert_se(journal_file_move_to_entry_by_seqnum(f, 1, DIRECTION_DOWN, &o, NULL) == 1);
+        assert_se(le64toh(o->entry.seqnum) == 1);
+
+        assert_se(journal_file_move_to_entry_by_seqnum(f, 3, DIRECTION_DOWN, &o, NULL) == 1);
+        assert_se(le64toh(o->entry.seqnum) == 3);
+
+        assert_se(journal_file_move_to_entry_by_seqnum(f, 2, DIRECTION_DOWN, &o, NULL) == 1);
+        assert_se(le64toh(o->entry.seqnum) == 2);
+
+        assert_se(journal_file_move_to_entry_by_seqnum(f, 10, DIRECTION_DOWN, &o, NULL) == 0);
+
+        journal_file_rotate(&f, true, (uint64_t) -1, true, NULL);
+        journal_file_rotate(&f, true, (uint64_t) -1, true, NULL);
+
+        (void) journal_file_close(f);
+
+        log_info("Done...");
+
+        if (arg_keep)
+                log_info("Not removing %s", t);
+        else {
+                journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
+
+                assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
+        }
+
+        puts("------------------------------------------------------------");
+}
+
+static void test_empty(void) {
+        JournalFile *f1, *f2, *f3, *f4;
+        char t[] = "/var/tmp/journal-XXXXXX";
+
+        test_setup_logging(LOG_DEBUG);
+
+        mkdtemp_chdir_chattr(t);
+
+        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f1) == 0);
+
+        assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f2) == 0);
+
+        assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f3) == 0);
+
+        assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f4) == 0);
+
+        journal_file_print_header(f1);
+        puts("");
+        journal_file_print_header(f2);
+        puts("");
+        journal_file_print_header(f3);
+        puts("");
+        journal_file_print_header(f4);
+        puts("");
+
+        log_info("Done...");
+
+        if (arg_keep)
+                log_info("Not removing %s", t);
+        else {
+                journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
+
+                assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
+        }
+
+        (void) journal_file_close(f1);
+        (void) journal_file_close(f2);
+        (void) journal_file_close(f3);
+        (void) journal_file_close(f4);
+}
+
+#if HAVE_COMPRESSION
+static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) {
+        dual_timestamp ts;
+        JournalFile *f;
+        struct iovec iovec;
+        Object *o;
+        uint64_t p;
+        char t[] = "/var/tmp/journal-XXXXXX";
+        char data[2048] = {0};
+        bool is_compressed;
+        int r;
+
+        assert_se(data_size <= sizeof(data));
+
+        test_setup_logging(LOG_DEBUG);
+
+        mkdtemp_chdir_chattr(t);
+
+        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, compress_threshold, true, NULL, NULL, NULL, NULL, &f) == 0);
+
+        dual_timestamp_get(&ts);
+
+        iovec = IOVEC_MAKE(data, data_size);
+        assert_se(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0);
+
+#if HAVE_GCRYPT
+        journal_file_append_tag(f);
+#endif
+        journal_file_dump(f);
+
+        /* We have to partially reimplement some of the dump logic, because the normal next_entry does the
+         * decompression for us. */
+        p = le64toh(f->header->header_size);
+        for (;;) {
+                r = journal_file_move_to_object(f, OBJECT_UNUSED, p, &o);
+                assert_se(r == 0);
+                if (o->object.type == OBJECT_DATA)
+                        break;
+
+                assert_se(p < le64toh(f->header->tail_object_offset));
+                p = p + ALIGN64(le64toh(o->object.size));
+        }
+
+        is_compressed = (o->object.flags & OBJECT_COMPRESSION_MASK) != 0;
+
+        (void) journal_file_close(f);
+
+        log_info("Done...");
+
+        if (arg_keep)
+                log_info("Not removing %s", t);
+        else {
+                journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
+
+                assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
+        }
+
+        puts("------------------------------------------------------------");
+
+        return is_compressed;
+}
+
+static void test_min_compress_size(void) {
+        /* Note that XZ will actually fail to compress anything under 80 bytes, so you have to choose the limits
+         * carefully */
+
+        /* DEFAULT_MIN_COMPRESS_SIZE is 512 */
+        assert_se(!check_compressed((uint64_t) -1, 255));
+        assert_se(check_compressed((uint64_t) -1, 513));
+
+        /* compress everything */
+        assert_se(check_compressed(0, 96));
+        assert_se(check_compressed(8, 96));
+
+        /* Ensure we don't try to compress less than 8 bytes */
+        assert_se(!check_compressed(0, 7));
+
+        /* check boundary conditions */
+        assert_se(check_compressed(256, 256));
+        assert_se(!check_compressed(256, 255));
+}
+#endif
+
+int main(int argc, char *argv[]) {
+        arg_keep = argc > 1;
+
+        test_setup_logging(LOG_INFO);
+
+        /* journal_file_open requires a valid machine id */
+        if (access("/etc/machine-id", F_OK) != 0)
+                return log_tests_skipped("/etc/machine-id not found");
+
+        test_non_empty();
+        test_empty();
+#if HAVE_COMPRESSION
+        test_min_compress_size();
+#endif
+
+        return 0;
+}

src/libsystemd/sd-journal/test-mmap-cache.c

@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <fcntl.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include "fd-util.h"
+#include "macro.h"
+#include "mmap-cache.h"
+#include "tmpfile-util.h"
+#include "util.h"
+
+int main(int argc, char *argv[]) {
+        MMapFileDescriptor *fx;
+        int x, y, z, r;
+        char px[] = "/tmp/testmmapXXXXXXX", py[] = "/tmp/testmmapYXXXXXX", pz[] = "/tmp/testmmapZXXXXXX";
+        MMapCache *m;
+        void *p, *q;
+
+        assert_se(m = mmap_cache_new());
+
+        x = mkostemp_safe(px);
+        assert_se(x >= 0);
+        unlink(px);
+
+        assert_se(fx = mmap_cache_add_fd(m, x, PROT_READ));
+
+        y = mkostemp_safe(py);
+        assert_se(y >= 0);
+        unlink(py);
+
+        z = mkostemp_safe(pz);
+        assert_se(z >= 0);
+        unlink(pz);
+
+        r = mmap_cache_get(m, fx, 0, false, 1, 2, NULL, &p);
+        assert_se(r >= 0);
+
+        r = mmap_cache_get(m, fx, 0, false, 2, 2, NULL, &q);
+        assert_se(r >= 0);
+
+        assert_se((uint8_t*) p + 1 == (uint8_t*) q);
+
+        r = mmap_cache_get(m, fx, 1, false, 3, 2, NULL, &q);
+        assert_se(r >= 0);
+
+        assert_se((uint8_t*) p + 2 == (uint8_t*) q);
+
+        r = mmap_cache_get(m, fx, 0, false, 16ULL*1024ULL*1024ULL, 2, NULL, &p);
+        assert_se(r >= 0);
+
+        r = mmap_cache_get(m, fx, 1, false, 16ULL*1024ULL*1024ULL+1, 2, NULL, &q);
+        assert_se(r >= 0);
+
+        assert_se((uint8_t*) p + 1 == (uint8_t*) q);
+
+        mmap_cache_free_fd(m, fx);
+        mmap_cache_unref(m);
+
+        safe_close(x);
+        safe_close(y);
+        safe_close(z);
+
+        return 0;
+}