Merge pull request #21372 from DaanDeMeyer/journal-truncate

journal-file: truncate archived journals

src/libsystemd/sd-journal/journal-file.c

@@ -91,6 +91,144 @@
 #  pragma GCC diagnostic ignored "-Waddress-of-packed-member"
 #endif
 
+static int journal_file_tail_end(JournalFile *f, uint64_t *ret_offset) {
+        Object tail;
+        uint64_t p;
+        int r;
+
+        assert(f);
+        assert(f->header);
+        assert(ret_offset);
+
+        p = le64toh(f->header->tail_object_offset);
+        if (p == 0)
+                p = le64toh(f->header->header_size);
+        else {
+                uint64_t sz;
+
+                r = journal_file_read_object(f, OBJECT_UNUSED, p, &tail);
+                if (r < 0)
+                        return r;
+
+                sz = le64toh(tail.object.size);
+                if (sz > UINT64_MAX - sizeof(uint64_t) + 1)
+                        return -EBADMSG;
+
+                sz = ALIGN64(sz);
+                if (p > UINT64_MAX - sz)
+                        return -EBADMSG;
+
+                p += sz;
+        }
+
+        *ret_offset = p;
+
+        return 0;
+}
+
+static int journal_file_truncate(JournalFile *f) {
+        uint64_t p;
+        int r;
+
+        /* truncate excess from the end of archives */
+        r = journal_file_tail_end(f, &p);
+        if (r < 0)
+                return log_debug_errno(r, "Failed to determine end of tail object: %m");
+
+        /* arena_size can't exceed the file size, ensure it's updated before truncating */
+        f->header->arena_size = htole64(p - le64toh(f->header->header_size));
+
+        if (ftruncate(f->fd, p) < 0)
+                log_debug_errno(errno, "Failed to truncate %s: %m", f->path);
+
+        return 0;
+}
+
+static int journal_file_entry_array_punch_hole(JournalFile *f, uint64_t p, uint64_t n_entries) {
+        Object o;
+        uint64_t offset, sz, n_items = 0, n_unused;
+        int r;
+
+        if (n_entries == 0)
+                return 0;
+
+        for (uint64_t q = p; q != 0; q = le64toh(o.entry_array.next_entry_array_offset)) {
+                r = journal_file_read_object(f, OBJECT_ENTRY_ARRAY, q, &o);
+                if (r < 0)
+                        return r;
+
+                n_items += journal_file_entry_array_n_items(&o);
+                p = q;
+        }
+
+        if (p == 0)
+                return 0;
+
+        if (n_entries > n_items)
+                return -EBADMSG;
+
+        /* Amount of unused items in the final entry array. */
+        n_unused = n_items - n_entries;
+
+        if (n_unused == 0)
+                return 0;
+
+        offset = p + offsetof(Object, entry_array.items) +
+                (journal_file_entry_array_n_items(&o) - n_unused) * sizeof(le64_t);
+        sz = p + le64toh(o.object.size) - offset;
+
+        if (fallocate(f->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, offset, sz) < 0)
+                return log_debug_errno(errno, "Failed to punch hole in entry array of %s: %m", f->path);
+
+        return 0;
+}
+
+static int journal_file_punch_holes(JournalFile *f) {
+        HashItem items[4096 / sizeof(HashItem)];
+        uint64_t p, sz;
+        size_t to_read;
+        int r;
+
+        r = journal_file_entry_array_punch_hole(
+                f, le64toh(f->header->entry_array_offset), le64toh(f->header->n_entries));
+        if (r < 0)
+                return r;
+
+        p = le64toh(f->header->data_hash_table_offset);
+        sz = le64toh(f->header->data_hash_table_size);
+        to_read = MIN((size_t) f->last_stat.st_blksize, sizeof(items));
+
+        for (uint64_t i = p; i < p + sz; i += sizeof(items)) {
+                ssize_t n_read;
+
+                n_read = pread(f->fd, items, MIN(to_read, p + sz - i), i);
+                if (n_read < 0)
+                        return n_read;
+
+                for (size_t j = 0; j < (size_t) n_read / sizeof(HashItem); j++) {
+                        Object o;
+
+                        for (uint64_t q = le64toh(items[j].head_hash_offset); q != 0;
+                             q = le64toh(o.data.next_hash_offset)) {
+
+                                r = journal_file_read_object(f, OBJECT_DATA, q, &o);
+                                if (r < 0) {
+                                        log_debug_errno(r, "Invalid data object: %m, ignoring");
+                                        break;
+                                }
+
+                                if (le64toh(o.data.n_entries) == 0)
+                                        continue;
+
+                                (void) journal_file_entry_array_punch_hole(
+                                        f, le64toh(o.data.entry_array_offset), le64toh(o.data.n_entries) - 1);
+                        }
+                }
+        }
+
+        return 0;
+}
+
 /* This may be called from a separate thread to prevent blocking the caller for the duration of fsync().
  * As a result we use atomic operations on f->offline_state for inter-thread communications with
  * journal_file_set_offline() and journal_file_set_online(). */
@@ -117,6 +255,11 @@ static void journal_file_set_offline_internal(JournalFile *f) {
                         break;
 
                 case OFFLINE_SYNCING:
+                        if (f->archive) {
+                                (void) journal_file_truncate(f);
+                                (void) journal_file_punch_holes(f);
+                        }
+
                         (void) fsync(f->fd);
 
                         if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_SYNCING, OFFLINE_OFFLINING))
@@ -990,6 +1133,65 @@ int journal_file_move_to_object(JournalFile *f, ObjectType type, uint64_t offset
         return 0;
 }
 
+int journal_file_read_object(JournalFile *f, ObjectType type, uint64_t offset, Object *ret) {
+        int r;
+        Object o;
+        uint64_t s;
+
+        assert(f);
+        assert(ret);
+
+        /* Objects may only be located at multiple of 64 bit */
+        if (!VALID64(offset))
+                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
+                                       "Attempt to read object at non-64bit boundary: %" PRIu64,
+                                       offset);
+
+        /* Object may not be located in the file header */
+        if (offset < le64toh(f->header->header_size))
+                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
+                                       "Attempt to read object located in file header: %" PRIu64,
+                                       offset);
+
+        /* This will likely read too much data but it avoids having to call pread() twice. */
+        r = pread(f->fd, &o, sizeof(Object), offset);
+        if (r < 0)
+                return r;
+
+        s = le64toh(o.object.size);
+
+        if (s == 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
+                                       "Attempt to read uninitialized object: %" PRIu64,
+                                       offset);
+        if (s < sizeof(ObjectHeader))
+                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
+                                       "Attempt to read overly short object: %" PRIu64,
+                                       offset);
+
+        if (o.object.type <= OBJECT_UNUSED)
+                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
+                                       "Attempt to read object with invalid type: %" PRIu64,
+                                       offset);
+
+        if (s < minimum_header_size(&o))
+                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
+                                       "Attempt to read truncated object: %" PRIu64,
+                                       offset);
+
+        if (type > OBJECT_UNUSED && o.object.type != type)
+                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
+                                       "Attempt to read object of unexpected type: %" PRIu64,
+                                       offset);
+
+        r = journal_file_check_object(f, offset, &o);
+        if (r < 0)
+                return r;
+
+        *ret = o;
+        return 0;
+}
+
 static uint64_t journal_file_entry_seqnum(
                 JournalFile *f,
                 uint64_t *seqnum) {
@@ -1030,7 +1232,7 @@ int journal_file_append_object(
 
         int r;
         uint64_t p;
-        Object *tail, *o;
+        Object *o;
         void *t;
 
         assert(f);
@@ -1042,26 +1244,9 @@ int journal_file_append_object(
         if (r < 0)
                 return r;
 
-        p = le64toh(f->header->tail_object_offset);
-        if (p == 0)
-                p = le64toh(f->header->header_size);
-        else {
-                uint64_t sz;
-
-                r = journal_file_move_to_object(f, OBJECT_UNUSED, p, &tail);
-                if (r < 0)
-                        return r;
-
-                sz = le64toh(READ_NOW(tail->object.size));
-                if (sz > UINT64_MAX - sizeof(uint64_t) + 1)
-                        return -EBADMSG;
-
-                sz = ALIGN64(sz);
-                if (p > UINT64_MAX - sz)
-                        return -EBADMSG;
-
-                p += sz;
-        }
+        r = journal_file_tail_end(f, &p);
+        if (r < 0)
+                return r;
 
         r = journal_file_allocate(f, p, size);
         if (r < 0)

src/libsystemd/sd-journal/journal-file.h

@@ -202,6 +202,7 @@ static inline bool VALID_EPOCH(uint64_t u) {
         FLAGS_SET(le32toh((h)->incompatible_flags), HEADER_INCOMPATIBLE_KEYED_HASH)
 
 int journal_file_move_to_object(JournalFile *f, ObjectType type, uint64_t offset, Object **ret);
+int journal_file_read_object(JournalFile *f, ObjectType type, uint64_t offset, Object *ret);
 
 uint64_t journal_file_entry_n_items(Object *o) _pure_;
 uint64_t journal_file_entry_array_n_items(Object *o) _pure_;