Merge pull request #29456 from yuwata/journal-replace-truncating-with-punching-holes

journal: replace truncating with punching holes

src/libsystemd/sd-journal/test-journal-flush.c

@@ -9,19 +9,21 @@
 #include "chattr-util.h"
 #include "journal-file-util.h"
 #include "journal-internal.h"
+#include "logs-show.h"
 #include "macro.h"
 #include "path-util.h"
 #include "rm-rf.h"
 #include "string-util.h"
+#include "tests.h"
 #include "tmpfile-util.h"
 
-static void test_journal_flush(int argc, char *argv[]) {
+static void test_journal_flush_one(int argc, char *argv[]) {
         _cleanup_(mmap_cache_unrefp) MMapCache *m = NULL;
         _cleanup_free_ char *fn = NULL;
         _cleanup_(rm_rf_physical_and_freep) char *dn = NULL;
-        JournalFile *new_journal = NULL;
+        _cleanup_(journal_file_offline_closep) JournalFile *new_journal = NULL;
-        sd_journal *j = NULL;
+        _cleanup_(sd_journal_closep) sd_journal *j = NULL;
-        unsigned n = 0;
+        unsigned n, limit;
         int r;
 
         assert_se(m = mmap_cache_new());
@@ -41,6 +43,8 @@ static void test_journal_flush(int argc, char *argv[]) {
 
         sd_journal_set_data_threshold(j, 0);
 
+        n = 0;
+        limit = slow_tests_enabled() ? 10000 : 1000;
         SD_JOURNAL_FOREACH(j) {
                 Object *o;
                 JournalFile *f;
@@ -62,21 +66,53 @@ static void test_journal_flush(int argc, char *argv[]) {
                                     -EIO,             /* file rotated */
                                     -EREMCHG));       /* clock rollback */
 
-                if (++n >= 10000)
+                if (++n >= limit)
                         break;
         }
 
+        if (n == 0)
+                return (void) log_tests_skipped("No journal entry found");
+
+        /* Open the new journal before archiving and offlining the file. */
         sd_journal_close(j);
+        assert_se(sd_journal_open_directory(&j, dn, 0) >= 0);
 
-        (void) journal_file_offline_close(new_journal);
+        /* Read the online journal. */
+        assert_se(sd_journal_seek_tail(j) >= 0);
+        assert_se(sd_journal_step_one(j, 0) > 0);
+        printf("current_journal: %s (%i)\n", j->current_file->path, j->current_file->fd);
+        assert_se(show_journal_entry(stdout, j, OUTPUT_EXPORT, 0, 0, NULL, NULL, NULL, &(dual_timestamp) {}, &(sd_id128_t) {}) >= 0);
+
+        uint64_t p;
+        assert_se(journal_file_tail_end_by_mmap(j->current_file, &p) >= 0);
+        for (uint64_t q = ALIGN64(p + 1); q < (uint64_t) j->current_file->last_stat.st_size; q = ALIGN64(q + 1)) {
+                Object *o;
+
+                r = journal_file_move_to_object(j->current_file, OBJECT_UNUSED, q, &o);
+                assert_se(IN_SET(r, -EBADMSG, -EADDRNOTAVAIL));
+        }
+
+        /* Archive and offline file. */
+        assert_se(journal_file_archive(new_journal, NULL) >= 0);
+        assert_se(journal_file_set_offline(new_journal, /* wait = */ true) >= 0);
+
+        /* Read the archived and offline journal. */
+        for (uint64_t q = ALIGN64(p + 1); q < (uint64_t) j->current_file->last_stat.st_size; q = ALIGN64(q + 1)) {
+                Object *o;
+
+                r = journal_file_move_to_object(j->current_file, OBJECT_UNUSED, q, &o);
+                assert_se(IN_SET(r, -EBADMSG, -EADDRNOTAVAIL, -EIDRM));
+        }
 }
 
-int main(int argc, char *argv[]) {
+TEST(journal_flush) {
         assert_se(setenv("SYSTEMD_JOURNAL_COMPACT", "0", 1) >= 0);
-        test_journal_flush(argc, argv);
+        test_journal_flush_one(saved_argc, saved_argv);
-
-        assert_se(setenv("SYSTEMD_JOURNAL_COMPACT", "1", 1) >= 0);
-        test_journal_flush(argc, argv);
-
-        return 0;
 }
+
+TEST(journal_flush_compact) {
+        assert_se(setenv("SYSTEMD_JOURNAL_COMPACT", "1", 1) >= 0);
+        test_journal_flush_one(saved_argc, saved_argv);
+}
+
+DEFINE_TEST_MAIN(LOG_INFO);

src/shared/journal-file-util.c

@@ -19,22 +19,29 @@
 #define PAYLOAD_BUFFER_SIZE (16U * 1024U)
 #define MINIMUM_HOLE_SIZE (1U * 1024U * 1024U / 2U)
 
-static int journal_file_truncate(JournalFile *f) {
+static int journal_file_end_punch_hole(JournalFile *f) {
-        uint64_t p;
+        uint64_t p, sz;
         int r;
 
-        /* truncate excess from the end of archives */
         r = journal_file_tail_end_by_pread(f, &p);
         if (r < 0)
                 return log_debug_errno(r, "Failed to determine end of tail object: %m");
 
-        /* arena_size can't exceed the file size, ensure it's updated before truncating */
+        assert(p <= (uint64_t) f->last_stat.st_size);
-        f->header->arena_size = htole64(p - le64toh(f->header->header_size));
 
-        if (ftruncate(f->fd, p) < 0)
+        sz = ((uint64_t) f->last_stat.st_size) - p;
-                return log_debug_errno(errno, "Failed to truncate %s: %m", f->path);
+        if (sz < MINIMUM_HOLE_SIZE)
+                return 0;
 
-        return journal_file_fstat(f);
+        if (fallocate(f->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, p, sz) < 0) {
+                if (ERRNO_IS_NOT_SUPPORTED(errno))
+                        return log_debug_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), /* Make recognizable */
+                                               "Hole punching not supported by backing file system, skipping.");
+
+                return log_debug_errno(errno, "Failed to punch hole at end of journal file %s: %m", f->path);
+        }
+
+        return 0;
 }
 
 static int journal_file_entry_array_punch_hole(JournalFile *f, uint64_t p, uint64_t n_entries) {
@@ -73,25 +80,6 @@ static int journal_file_entry_array_punch_hole(JournalFile *f, uint64_t p, uint6
         if (sz < MINIMUM_HOLE_SIZE)
                 return 0;
 
-        if (p == le64toh(f->header->tail_object_offset) && !JOURNAL_HEADER_SEALED(f->header)) {
-                ssize_t n;
-
-                o.object.size = htole64(offset - p);
-
-                n = pwrite(f->fd, &o, sizeof(EntryArrayObject), p);
-                if (n < 0)
-                        return log_debug_errno(errno, "Failed to modify entry array object size: %m");
-                if ((size_t) n != sizeof(EntryArrayObject))
-                        return log_debug_errno(SYNTHETIC_ERRNO(EIO), "Short pwrite() while modifying entry array object size.");
-
-                f->header->arena_size = htole64(ALIGN64(offset) - le64toh(f->header->header_size));
-
-                if (ftruncate(f->fd, ALIGN64(offset)) < 0)
-                        return log_debug_errno(errno, "Failed to truncate %s: %m", f->path);
-
-                return 0;
-        }
-
         if (fallocate(f->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, offset, sz) < 0) {
                 if (ERRNO_IS_NOT_SUPPORTED(errno))
                         return log_debug_errno(SYNTHETIC_ERRNO(EOPNOTSUPP), /* Make recognizable */
@@ -192,7 +180,7 @@ static void journal_file_set_offline_internal(JournalFile *f) {
 
                 case OFFLINE_SYNCING:
                         if (f->archive) {
-                                (void) journal_file_truncate(f);
+                                (void) journal_file_end_punch_hole(f);
                                 (void) journal_file_punch_holes(f);
                         }
 
@@ -346,9 +334,14 @@ int journal_file_set_offline(JournalFile *f, bool wait) {
         /* Initiate a new offline. */
         f->offline_state = OFFLINE_SYNCING;
 
-        if (wait) /* Without using a thread if waiting. */
+        if (wait) {
+                /* Without using a thread if waiting. */
                 journal_file_set_offline_internal(f);
-        else {
+
+                assert(f->offline_state == OFFLINE_DONE);
+                f->offline_state = OFFLINE_JOINED;
+
+        } else {
                 sigset_t ss, saved_ss;
                 int k;