From 1d8d483f59ffa62974772fb58a8ef4abe88550ec Mon Sep 17 00:00:00 2001 From: msizanoen1 Date: Sat, 8 Oct 2022 14:37:02 +0700 Subject: [PATCH 1/3] journal: enforce strict consistency for realtime timestamps on write Ensure all realtime timestamps in a journal file are strictly ordered on write as a defense-in-depth measure. All known callers of journal_file_append_entry and journal_file_copy_entry, which call this function, should be able to handle the error by rotating the journal. This is especially helpful for systems with RTC local time enabled, where all log entries from initramfs might be recorded as several hours later than it actually is, which won't get caught by journald during log flushing. In those cases, the resulting inconsistency can cause libsystemd to loop infinitely through journal files as observed in `abrt-dump-journal-oops`. --- src/journal/journald-server.c | 4 ++++ src/libsystemd/sd-journal/journal-file.c | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 9e2d239abd..b59f42c4b8 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -839,6 +839,10 @@ static bool shall_try_append_again(JournalFile *f, int r) { log_ratelimit_warning(JOURNAL_LOG_RATELIMIT, "%s: Journal file is from the future, rotating.", f->path); return true; + case -EREMCHG: /* Time jumped backwards relative to last journal entry */ + log_ratelimit_warning(JOURNAL_LOG_RATELIMIT, "%s: Time jumped backwards relative to last journal entry, rotating.", f->path); + return true; + case -EAFNOSUPPORT: log_ratelimit_warning(JOURNAL_LOG_RATELIMIT, "%s: underlying file system does not support memory mapping or another required file system feature.", diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index ab518fea80..805773474c 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -2092,6 +2092,12 @@ static int journal_file_append_entry_internal( assert(ts); assert(items || n_items == 0); + if (ts->realtime < le64toh(f->header->tail_entry_realtime)) + return log_debug_errno(SYNTHETIC_ERRNO(EREMCHG), + "Realtime timestamp %" PRIu64 " smaller than previous realtime " + "timestamp %" PRIu64 ", refusing entry.", + ts->realtime, le64toh(f->header->tail_entry_realtime)); + osize = offsetof(Object, entry.items) + (n_items * journal_file_entry_item_size(f)); r = journal_file_append_object(f, OBJECT_ENTRY, osize, &o, &np); From 9cd80d8a580d63320efb245fc9c30c1365629ba1 Mon Sep 17 00:00:00 2001 From: msizanoen1 Date: Wed, 12 Oct 2022 15:38:25 +0700 Subject: [PATCH 2/3] test-journal-flush: don't fail on -EREMCHG -EREMCHG on write is now used to denote clock rollback conditions detected during journal write. --- src/journal/test-journal-flush.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/journal/test-journal-flush.c b/src/journal/test-journal-flush.c index c734aa02ca..b78e073ddd 100644 --- a/src/journal/test-journal-flush.c +++ b/src/journal/test-journal-flush.c @@ -58,7 +58,8 @@ static void test_journal_flush(int argc, char *argv[]) { assert_se(r >= 0 || IN_SET(r, -EBADMSG, /* corrupted file */ -EPROTONOSUPPORT, /* unsupported compression */ - -EIO)); /* file rotated */ + -EIO, /* file rotated */ + -EREMCHG)); /* clock rollback */ if (++n >= 10000) break; From e18722375b712f1277d1b44860f184752b586312 Mon Sep 17 00:00:00 2001 From: msizanoen1 Date: Mon, 9 Jan 2023 23:05:19 +0700 Subject: [PATCH 3/3] journal: handle clock rollback error in managed_journal_file_open_reliably -EREMCHG was introduced to denote a clock rollback detection error. Handle this in managed_journal_file_open_reliably. --- src/journal/managed-journal-file.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/journal/managed-journal-file.c b/src/journal/managed-journal-file.c index 810167772a..6e313f2a05 100644 --- a/src/journal/managed-journal-file.c +++ b/src/journal/managed-journal-file.c @@ -524,7 +524,8 @@ int managed_journal_file_open_reliably( -ESHUTDOWN, /* Already archived */ -EIO, /* IO error, including SIGBUS on mmap */ -EIDRM, /* File has been deleted */ - -ETXTBSY)) /* File is from the future */ + -ETXTBSY, /* File is from the future */ + -EREMCHG)) /* Clock rolled back */ return r; if ((open_flags & O_ACCMODE) == O_RDONLY)