From 035b0f8fe8c1883b17d864f15f99846ab206099d Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Tue, 30 Nov 2021 16:18:56 +0100 Subject: [PATCH 1/7] journal: Introduce journald-file.c for journal file write related logic Currently, all the logic related to writing journal files lives in journal-file.c which is part of libsystemd (sd-journal). Because it's part of libsystemd, we can't depend on any code from src/shared. To allow using code from src/shared when writing journal files, let's gradually move the write related logic from journal-file.c to journald-file.c in src/journal. This directory is not part of libsystemd and as such can use code from src/shared. We can safely remove any journal write related logic from libsystemd as it's not used by any public APIs in libsystemd. This commit introduces the new file along with the JournaldFile struct which wraps an instance of JournalFile. The goal is to gradually move more functions from journal-file.c and fields from JournalFile to journald-file.c and JournaldFile respectively. This commit also modifies all call sites that write journal files to use JournaldFile instead of JournalFile. All sd-journal tests that write journal files are moved to src/journal so they can make use of journald-file.c. Because the deferred closes logic is only used by journald, we move it out of journal-file.c as well. In journal_file_open(), we would wait for any remaining deferred closes for the file we're about to open to complete before continuing if the file was not newly created. In journald_file_open(), we call this logic unconditionally since it stands that if a file is newly created, it can't have any outstanding deferred closes. No changes in behavior are introduced aside from the earlier execution of waiting for any deferred closes to complete when opening a new journal file. --- meson.build | 4 +- src/journal-remote/journal-remote-write.c | 22 +-- src/journal-remote/journal-remote-write.h | 4 +- src/journal-remote/journal-remote.c | 16 +- src/journal-remote/meson.build | 5 +- src/journal/journald-file.c | 157 ++++++++++++++++++ src/journal/journald-file.h | 41 +++++ src/journal/journald-server.c | 131 ++++++++------- src/journal/journald-server.h | 6 +- src/journal/journald.c | 2 +- src/journal/meson.build | 22 +++ .../test-journal-flush.c | 10 +- .../test-journal-interleaving.c | 58 +++---- .../test-journal-stream.c | 24 +-- .../test-journal-verify.c | 15 +- .../sd-journal => journal}/test-journal.c | 103 ++++++------ src/libsystemd/meson.build | 10 -- src/libsystemd/sd-journal/journal-file.c | 111 ------------- src/libsystemd/sd-journal/journal-file.h | 16 -- src/libsystemd/sd-journal/sd-journal.c | 2 +- 20 files changed, 423 insertions(+), 336 deletions(-) create mode 100644 src/journal/journald-file.c create mode 100644 src/journal/journald-file.h rename src/{libsystemd/sd-journal => journal}/test-journal-flush.c (85%) rename src/{libsystemd/sd-journal => journal}/test-journal-interleaving.c (77%) rename src/{libsystemd/sd-journal => journal}/test-journal-stream.c (85%) rename src/{libsystemd/sd-journal => journal}/test-journal-verify.c (87%) rename src/{libsystemd/sd-journal => journal}/test-journal.c (56%) diff --git a/meson.build b/meson.build index 43f5bf2c5a..519de53345 100644 --- a/meson.build +++ b/meson.build @@ -2814,7 +2814,7 @@ if conf.get('ENABLE_REMOTE') == 1 and conf.get('HAVE_MICROHTTPD') == 1 public_programs += executable( 'systemd-journal-remote', systemd_journal_remote_sources, - include_directories : includes, + include_directories : journal_includes, link_with : [libshared, libsystemd_journal_remote], dependencies : [threads, @@ -2830,7 +2830,7 @@ if conf.get('ENABLE_REMOTE') == 1 and conf.get('HAVE_MICROHTTPD') == 1 public_programs += executable( 'systemd-journal-gatewayd', systemd_journal_gatewayd_sources, - include_directories : includes, + include_directories : journal_includes, link_with : [libshared], dependencies : [threads, libmicrohttpd, diff --git a/src/journal-remote/journal-remote-write.c b/src/journal-remote/journal-remote-write.c index c953a5e93e..fd7cb91f2c 100644 --- a/src/journal-remote/journal-remote-write.c +++ b/src/journal-remote/journal-remote-write.c @@ -3,11 +3,11 @@ #include "alloc-util.h" #include "journal-remote.h" -static int do_rotate(JournalFile **f, bool compress, bool seal) { - int r = journal_file_rotate(f, compress, UINT64_MAX, seal, NULL); +static int do_rotate(JournaldFile **f, bool compress, bool seal) { + int r = journald_file_rotate(f, compress, UINT64_MAX, seal, NULL); if (r < 0) { if (*f) - log_error_errno(r, "Failed to rotate %s: %m", (*f)->path); + log_error_errno(r, "Failed to rotate %s: %m", (*f)->file->path); else log_error_errno(r, "Failed to create rotated journal: %m"); } @@ -39,8 +39,8 @@ static Writer* writer_free(Writer *w) { return NULL; if (w->journal) { - log_debug("Closing journal file %s.", w->journal->path); - journal_file_close(w->journal); + log_debug("Closing journal file %s.", w->journal->file->path); + journald_file_close(w->journal); } if (w->server && w->hashmap_key) @@ -68,15 +68,15 @@ int writer_write(Writer *w, assert(iovw); assert(iovw->count > 0); - if (journal_file_rotate_suggested(w->journal, 0, LOG_DEBUG)) { + if (journal_file_rotate_suggested(w->journal->file, 0, LOG_DEBUG)) { log_info("%s: Journal header limits reached or header out-of-date, rotating", - w->journal->path); + w->journal->file->path); r = do_rotate(&w->journal, compress, seal); if (r < 0) return r; } - r = journal_file_append_entry(w->journal, ts, boot_id, + r = journal_file_append_entry(w->journal->file, ts, boot_id, iovw->iovec, iovw->count, &w->seqnum, NULL, NULL); if (r >= 0) { @@ -86,15 +86,15 @@ int writer_write(Writer *w, } else if (r == -EBADMSG) return r; - log_debug_errno(r, "%s: Write failed, rotating: %m", w->journal->path); + log_debug_errno(r, "%s: Write failed, rotating: %m", w->journal->file->path); r = do_rotate(&w->journal, compress, seal); if (r < 0) return r; else - log_debug("%s: Successfully rotated journal", w->journal->path); + log_debug("%s: Successfully rotated journal", w->journal->file->path); log_debug("Retrying write."); - r = journal_file_append_entry(w->journal, ts, boot_id, + r = journal_file_append_entry(w->journal->file, ts, boot_id, iovw->iovec, iovw->count, &w->seqnum, NULL, NULL); if (r < 0) diff --git a/src/journal-remote/journal-remote-write.h b/src/journal-remote/journal-remote-write.h index 123015b844..d97f6c674b 100644 --- a/src/journal-remote/journal-remote-write.h +++ b/src/journal-remote/journal-remote-write.h @@ -1,13 +1,13 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ #pragma once -#include "journal-file.h" +#include "journald-file.h" #include "journal-importer.h" typedef struct RemoteServer RemoteServer; typedef struct Writer { - JournalFile *journal; + JournaldFile *journal; JournalMetrics metrics; MMapCache *mmap; diff --git a/src/journal-remote/journal-remote.c b/src/journal-remote/journal-remote.c index 9fd6a23076..e91b46473b 100644 --- a/src/journal-remote/journal-remote.c +++ b/src/journal-remote/journal-remote.c @@ -14,7 +14,7 @@ #include "errno-util.h" #include "escape.h" #include "fd-util.h" -#include "journal-file.h" +#include "journald-file.h" #include "journal-remote-write.h" #include "journal-remote.h" #include "journald-native.h" @@ -61,16 +61,16 @@ static int open_output(RemoteServer *s, Writer *w, const char* host) { assert_not_reached(); } - r = journal_file_open_reliably(filename, - O_RDWR|O_CREAT, 0640, - s->compress, UINT64_MAX, s->seal, - &w->metrics, - w->mmap, NULL, - NULL, &w->journal); + r = journald_file_open_reliably(filename, + O_RDWR|O_CREAT, 0640, + s->compress, UINT64_MAX, s->seal, + &w->metrics, + w->mmap, NULL, + NULL, &w->journal); if (r < 0) return log_error_errno(r, "Failed to open output journal %s: %m", filename); - log_debug("Opened output file %s", w->journal->path); + log_debug("Opened output file %s", w->journal->file->path); return 0; } diff --git a/src/journal-remote/meson.build b/src/journal-remote/meson.build index 54b314552b..b9f8b472cb 100644 --- a/src/journal-remote/meson.build +++ b/src/journal-remote/meson.build @@ -25,6 +25,7 @@ libsystemd_journal_remote = static_library( 'systemd-journal-remote', libsystemd_journal_remote_sources, include_directories : journal_includes, + link_with : libjournal_core, dependencies : [threads, libmicrohttpd, libgnutls, @@ -77,5 +78,7 @@ endif fuzzers += [ [['src/journal-remote/fuzz-journal-remote.c'], [libsystemd_journal_remote, - libshared]], + libshared], + [], + [journal_includes]], ] diff --git a/src/journal/journald-file.c b/src/journal/journald-file.c new file mode 100644 index 0000000000..b12f40599b --- /dev/null +++ b/src/journal/journald-file.c @@ -0,0 +1,157 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include "chattr-util.h" +#include "fd-util.h" +#include "format-util.h" +#include "journal-authenticate.h" +#include "journald-file.h" +#include "path-util.h" +#include "random-util.h" +#include "set.h" +#include "sync-util.h" + +JournaldFile* journald_file_close(JournaldFile *f) { + if (!f) + return NULL; + + journal_file_close(f->file); + + return mfree(f); +} + +int journald_file_open( + int fd, + const char *fname, + int flags, + mode_t mode, + bool compress, + uint64_t compress_threshold_bytes, + bool seal, + JournalMetrics *metrics, + MMapCache *mmap_cache, + Set *deferred_closes, + JournaldFile *template, + JournaldFile **ret) { + _cleanup_free_ JournaldFile *f = NULL; + int r; + + set_clear_with_destructor(deferred_closes, journald_file_close); + + f = new0(JournaldFile, 1); + if (!f) + return -ENOMEM; + + r = journal_file_open(fd, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, + mmap_cache, template ? template->file : NULL, &f->file); + if (r < 0) + return r; + + *ret = TAKE_PTR(f); + + return 0; +} + + +JournaldFile* journald_file_initiate_close(JournaldFile *f, Set *deferred_closes) { + int r; + + assert(f); + + if (deferred_closes) { + r = set_put(deferred_closes, f); + if (r < 0) + log_debug_errno(r, "Failed to add file to deferred close set, closing immediately."); + else { + (void) journal_file_set_offline(f->file, false); + return NULL; + } + } + + return journald_file_close(f); +} + +int journald_file_rotate( + JournaldFile **f, + bool compress, + uint64_t compress_threshold_bytes, + bool seal, + Set *deferred_closes) { + + JournaldFile *new_file = NULL; + int r; + + assert(f); + assert(*f); + + r = journal_file_archive((*f)->file); + if (r < 0) + return r; + + r = journald_file_open( + -1, + (*f)->file->path, + (*f)->file->flags, + (*f)->file->mode, + compress, + compress_threshold_bytes, + seal, + NULL, /* metrics */ + (*f)->file->mmap, + deferred_closes, + *f, /* template */ + &new_file); + + journald_file_initiate_close(*f, deferred_closes); + *f = new_file; + + return r; +} + +int journald_file_open_reliably( + const char *fname, + int flags, + mode_t mode, + bool compress, + uint64_t compress_threshold_bytes, + bool seal, + JournalMetrics *metrics, + MMapCache *mmap_cache, + Set *deferred_closes, + JournaldFile *template, + JournaldFile **ret) { + + int r; + + r = journald_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, + mmap_cache, deferred_closes, template, ret); + if (!IN_SET(r, + -EBADMSG, /* Corrupted */ + -ENODATA, /* Truncated */ + -EHOSTDOWN, /* Other machine */ + -EPROTONOSUPPORT, /* Incompatible feature */ + -EBUSY, /* Unclean shutdown */ + -ESHUTDOWN, /* Already archived */ + -EIO, /* IO error, including SIGBUS on mmap */ + -EIDRM, /* File has been deleted */ + -ETXTBSY)) /* File is from the future */ + return r; + + if ((flags & O_ACCMODE) == O_RDONLY) + return r; + + if (!(flags & O_CREAT)) + return r; + + if (!endswith(fname, ".journal")) + return r; + + /* The file is corrupted. Rotate it away and try it again (but only once) */ + log_warning_errno(r, "File %s corrupted or uncleanly shut down, renaming and replacing.", fname); + + r = journal_file_dispose(AT_FDCWD, fname); + if (r < 0) + return r; + + return journald_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, + mmap_cache, deferred_closes, template, ret); +} diff --git a/src/journal/journald-file.h b/src/journal/journald-file.h new file mode 100644 index 0000000000..7a299bd975 --- /dev/null +++ b/src/journal/journald-file.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include "journal-file.h" + +typedef struct { + JournalFile *file; +} JournaldFile; + +int journald_file_open( + int fd, + const char *fname, + int flags, + mode_t mode, + bool compress, + uint64_t compress_threshold_bytes, + bool seal, + JournalMetrics *metrics, + MMapCache *mmap_cache, + Set *deferred_closes, + JournaldFile *template, + JournaldFile **ret); + +JournaldFile* journald_file_close(JournaldFile *f); +DEFINE_TRIVIAL_CLEANUP_FUNC(JournaldFile*, journald_file_close); + +int journald_file_open_reliably( + const char *fname, + int flags, + mode_t mode, + bool compress, + uint64_t compress_threshold_bytes, + bool seal, + JournalMetrics *metrics, + MMapCache *mmap_cache, + Set *deferred_closes, + JournaldFile *template, + JournaldFile **ret); + +JournaldFile* journald_file_initiate_close(JournaldFile *f, Set *deferred_closes); +int journald_file_rotate(JournaldFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes); diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 5ba9b3765b..3d40c3822e 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -29,7 +29,7 @@ #include "id128-util.h" #include "io-util.h" #include "journal-authenticate.h" -#include "journal-file.h" +#include "journald-file.h" #include "journal-internal.h" #include "journal-vacuum.h" #include "journald-audit.h" @@ -243,7 +243,7 @@ static bool uid_for_system_journal(uid_t uid) { return uid_is_system(uid) || uid_is_dynamic(uid) || uid == UID_NOBODY; } -static void server_add_acls(JournalFile *f, uid_t uid) { +static void server_add_acls(JournaldFile *f, uid_t uid) { assert(f); #if HAVE_ACL @@ -252,9 +252,9 @@ static void server_add_acls(JournalFile *f, uid_t uid) { if (uid_for_system_journal(uid)) return; - r = fd_add_uid_acl_permission(f->fd, uid, ACL_READ); + r = fd_add_uid_acl_permission(f->file->fd, uid, ACL_READ); if (r < 0) - log_warning_errno(r, "Failed to set ACL on %s, ignoring: %m", f->path); + log_warning_errno(r, "Failed to set ACL on %s, ignoring: %m", f->file->path); #endif } @@ -265,9 +265,9 @@ static int open_journal( int flags, bool seal, JournalMetrics *metrics, - JournalFile **ret) { + JournaldFile **ret) { - _cleanup_(journal_file_closep) JournalFile *f = NULL; + _cleanup_(journald_file_closep) JournaldFile *f = NULL; int r; assert(s); @@ -275,16 +275,18 @@ static int open_journal( assert(ret); if (reliably) - r = journal_file_open_reliably(fname, flags, 0640, s->compress.enabled, s->compress.threshold_bytes, - seal, metrics, s->mmap, s->deferred_closes, NULL, &f); + r = journald_file_open_reliably(fname, flags, 0640, s->compress.enabled, + s->compress.threshold_bytes, seal, metrics, s->mmap, + s->deferred_closes, NULL, &f); else - r = journal_file_open(-1, fname, flags, 0640, s->compress.enabled, s->compress.threshold_bytes, seal, - metrics, s->mmap, s->deferred_closes, NULL, &f); + r = journald_file_open(-1, fname, flags, 0640, s->compress.enabled, + s->compress.threshold_bytes, seal, metrics, s->mmap, + s->deferred_closes, NULL, &f); if (r < 0) return r; - r = journal_file_enable_post_change_timer(f, s->event, POST_CHANGE_TIMER_INTERVAL_USEC); + r = journal_file_enable_post_change_timer(f->file, s->event, POST_CHANGE_TIMER_INTERVAL_USEC); if (r < 0) return r; @@ -388,9 +390,9 @@ static int system_journal_open(Server *s, bool flush_requested, bool relinquish_ return r; } -static JournalFile* find_journal(Server *s, uid_t uid) { +static JournaldFile* find_journal(Server *s, uid_t uid) { _cleanup_free_ char *p = NULL; - JournalFile *f; + JournaldFile *f; int r; assert(s); @@ -433,7 +435,7 @@ static JournalFile* find_journal(Server *s, uid_t uid) { /* Too many open? Then let's close one (or more) */ while (ordered_hashmap_size(s->user_journals) >= USER_JOURNALS_MAX) { assert_se(f = ordered_hashmap_steal_first(s->user_journals)); - (void) journal_file_close(f); + (void) journald_file_close(f); } r = open_journal(s, true, p, O_RDWR|O_CREAT, s->seal, &s->system_storage.metrics, &f); @@ -442,7 +444,7 @@ static JournalFile* find_journal(Server *s, uid_t uid) { r = ordered_hashmap_put(s->user_journals, UID_TO_PTR(uid), f); if (r < 0) { - (void) journal_file_close(f); + (void) journald_file_close(f); return s->system_journal; } @@ -452,7 +454,7 @@ static JournalFile* find_journal(Server *s, uid_t uid) { static int do_rotate( Server *s, - JournalFile **f, + JournaldFile **f, const char* name, bool seal, uint32_t uid) { @@ -463,10 +465,10 @@ static int do_rotate( if (!*f) return -EINVAL; - r = journal_file_rotate(f, s->compress.enabled, s->compress.threshold_bytes, seal, s->deferred_closes); + r = journald_file_rotate(f, s->compress.enabled, s->compress.threshold_bytes, seal, s->deferred_closes); if (r < 0) { if (*f) - return log_error_errno(r, "Failed to rotate %s: %m", (*f)->path); + return log_error_errno(r, "Failed to rotate %s: %m", (*f)->file->path); else return log_error_errno(r, "Failed to create new %s journal: %m", name); } @@ -476,15 +478,15 @@ static int do_rotate( } static void server_process_deferred_closes(Server *s) { - JournalFile *f; + JournaldFile *f; /* Perform any deferred closes which aren't still offlining. */ SET_FOREACH(f, s->deferred_closes) { - if (journal_file_is_offlining(f)) + if (journal_file_is_offlining(f->file)) continue; (void) set_remove(s->deferred_closes, f); - (void) journal_file_close(f); + (void) journald_file_close(f); } } @@ -500,10 +502,10 @@ static void server_vacuum_deferred_closes(Server *s) { /* And now, let's close some more until we reach the limit again. */ while (set_size(s->deferred_closes) >= DEFERRED_CLOSES_MAX) { - JournalFile *f; + JournaldFile *f; assert_se(f = set_steal_first(s->deferred_closes)); - journal_file_close(f); + journald_file_close(f); } } @@ -526,7 +528,7 @@ static int vacuum_offline_user_journals(Server *s) { _cleanup_close_ int fd = -1; const char *a, *b; struct dirent *de; - JournalFile *f; + JournaldFile *f; uid_t uid; errno = 0; @@ -574,18 +576,18 @@ static int vacuum_offline_user_journals(Server *s) { server_vacuum_deferred_closes(s); /* Open the file briefly, so that we can archive it */ - r = journal_file_open(fd, - full, - O_RDWR, - 0640, - s->compress.enabled, - s->compress.threshold_bytes, - s->seal, - &s->system_storage.metrics, - s->mmap, - s->deferred_closes, - NULL, - &f); + r = journald_file_open(fd, + full, + O_RDWR, + 0640, + s->compress.enabled, + s->compress.threshold_bytes, + s->seal, + &s->system_storage.metrics, + s->mmap, + s->deferred_closes, + NULL, + &f); if (r < 0) { log_warning_errno(r, "Failed to read journal file %s for rotation, trying to move it out of the way: %m", full); @@ -598,20 +600,21 @@ static int vacuum_offline_user_journals(Server *s) { continue; } - TAKE_FD(fd); /* Donated to journal_file_open() */ + TAKE_FD(fd); /* Donated to journald_file_open() */ - r = journal_file_archive(f); + r = journal_file_archive(f->file); if (r < 0) log_debug_errno(r, "Failed to archive journal file '%s', ignoring: %m", full); - f = journal_initiate_close(f, s->deferred_closes); + journald_file_initiate_close(f, s->deferred_closes); + f = NULL; } return 0; } void server_rotate(Server *s) { - JournalFile *f; + JournaldFile *f; void *k; int r; @@ -640,17 +643,17 @@ void server_rotate(Server *s) { } void server_sync(Server *s) { - JournalFile *f; + JournaldFile *f; int r; if (s->system_journal) { - r = journal_file_set_offline(s->system_journal, false); + r = journal_file_set_offline(s->system_journal->file, false); if (r < 0) log_warning_errno(r, "Failed to sync system journal, ignoring: %m"); } ORDERED_HASHMAP_FOREACH(f, s->user_journals) { - r = journal_file_set_offline(f, false); + r = journal_file_set_offline(f->file, false); if (r < 0) log_warning_errno(r, "Failed to sync user journal, ignoring: %m"); } @@ -795,7 +798,7 @@ static bool shall_try_append_again(JournalFile *f, int r) { static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, size_t n, int priority) { bool vacuumed = false, rotate = false; struct dual_timestamp ts; - JournalFile *f; + JournaldFile *f; int r; assert(s); @@ -822,8 +825,8 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, size_t n if (!f) return; - if (journal_file_rotate_suggested(f, s->max_file_usec, LOG_INFO)) { - log_info("%s: Journal header limits reached or header out-of-date, rotating.", f->path); + if (journal_file_rotate_suggested(f->file, s->max_file_usec, LOG_INFO)) { + log_info("%s: Journal header limits reached or header out-of-date, rotating.", f->file->path); rotate = true; } } @@ -840,13 +843,13 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, size_t n s->last_realtime_clock = ts.realtime; - r = journal_file_append_entry(f, &ts, NULL, iovec, n, &s->seqnum, NULL, NULL); + r = journal_file_append_entry(f->file, &ts, NULL, iovec, n, &s->seqnum, NULL, NULL); if (r >= 0) { server_schedule_sync(s, priority); return; } - if (vacuumed || !shall_try_append_again(f, r)) { + if (vacuumed || !shall_try_append_again(f->file, r)) { log_error_errno(r, "Failed to write entry (%zu items, %zu bytes), ignoring: %m", n, IOVEC_TOTAL_SIZE(iovec, n)); return; } @@ -861,7 +864,7 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, size_t n return; log_debug("Retrying write."); - r = journal_file_append_entry(f, &ts, NULL, iovec, n, &s->seqnum, NULL, NULL); + r = journal_file_append_entry(f->file, &ts, NULL, iovec, n, &s->seqnum, NULL, NULL); if (r < 0) log_error_errno(r, "Failed to write entry (%zu items, %zu bytes) despite vacuuming, ignoring: %m", n, IOVEC_TOTAL_SIZE(iovec, n)); else @@ -1168,11 +1171,11 @@ int server_flush_to_var(Server *s, bool require_flag_file) { goto finish; } - r = journal_file_copy_entry(f, s->system_journal, o, f->current_offset); + r = journal_file_copy_entry(f, s->system_journal->file, o, f->current_offset); if (r >= 0) continue; - if (!shall_try_append_again(s->system_journal, r)) { + if (!shall_try_append_again(s->system_journal->file, r)) { log_error_errno(r, "Can't write entry: %m"); goto finish; } @@ -1189,7 +1192,7 @@ int server_flush_to_var(Server *s, bool require_flag_file) { } log_debug("Retrying write."); - r = journal_file_copy_entry(f, s->system_journal, o, f->current_offset); + r = journal_file_copy_entry(f, s->system_journal->file, o, f->current_offset); if (r < 0) { log_error_errno(r, "Can't write entry: %m"); goto finish; @@ -1200,9 +1203,9 @@ int server_flush_to_var(Server *s, bool require_flag_file) { finish: if (s->system_journal) - journal_file_post_change(s->system_journal); + journal_file_post_change(s->system_journal->file); - s->runtime_journal = journal_file_close(s->runtime_journal); + s->runtime_journal = journald_file_close(s->runtime_journal); if (r >= 0) (void) rm_rf(s->runtime_storage.path, REMOVE_ROOT); @@ -1242,9 +1245,9 @@ static int server_relinquish_var(Server *s) { (void) system_journal_open(s, false, true); - s->system_journal = journal_file_close(s->system_journal); - ordered_hashmap_clear_with_destructor(s->user_journals, journal_file_close); - set_clear_with_destructor(s->deferred_closes, journal_file_close); + s->system_journal = journald_file_close(s->system_journal); + ordered_hashmap_clear_with_destructor(s->user_journals, journald_file_close); + set_clear_with_destructor(s->deferred_closes, journald_file_close); fn = strjoina(s->runtime_directory, "/flushed"); if (unlink(fn) < 0 && errno != ENOENT) @@ -2439,16 +2442,16 @@ int server_init(Server *s, const char *namespace) { void server_maybe_append_tags(Server *s) { #if HAVE_GCRYPT - JournalFile *f; + JournaldFile *f; usec_t n; n = now(CLOCK_REALTIME); if (s->system_journal) - journal_file_maybe_append_tag(s->system_journal, n); + journal_file_maybe_append_tag(s->system_journal->file, n); ORDERED_HASHMAP_FOREACH(f, s->user_journals) - journal_file_maybe_append_tag(f, n); + journal_file_maybe_append_tag(f->file, n); #endif } @@ -2458,17 +2461,17 @@ void server_done(Server *s) { free(s->namespace); free(s->namespace_field); - set_free_with_destructor(s->deferred_closes, journal_file_close); + set_free_with_destructor(s->deferred_closes, journald_file_close); while (s->stdout_streams) stdout_stream_free(s->stdout_streams); client_context_flush_all(s); - (void) journal_file_close(s->system_journal); - (void) journal_file_close(s->runtime_journal); + (void) journald_file_close(s->system_journal); + (void) journald_file_close(s->runtime_journal); - ordered_hashmap_free_with_destructor(s->user_journals, journal_file_close); + ordered_hashmap_free_with_destructor(s->user_journals, journald_file_close); varlink_server_unref(s->varlink_server); diff --git a/src/journal/journald-server.h b/src/journal/journald-server.h index 5b7e59cada..92c78a0d74 100644 --- a/src/journal/journald-server.h +++ b/src/journal/journald-server.h @@ -10,7 +10,7 @@ typedef struct Server Server; #include "conf-parser.h" #include "hashmap.h" -#include "journal-file.h" +#include "journald-file.h" #include "journald-context.h" #include "journald-rate-limit.h" #include "journald-stream.h" @@ -89,8 +89,8 @@ struct Server { sd_event_source *watchdog_event_source; sd_event_source *idle_event_source; - JournalFile *runtime_journal; - JournalFile *system_journal; + JournaldFile *runtime_journal; + JournaldFile *system_journal; OrderedHashmap *user_journals; uint64_t seqnum; diff --git a/src/journal/journald.c b/src/journal/journald.c index 94aad05de9..3d4044295e 100644 --- a/src/journal/journald.c +++ b/src/journal/journald.c @@ -102,7 +102,7 @@ int main(int argc, char *argv[]) { if (server.system_journal) { usec_t u; - if (journal_file_next_evolve_usec(server.system_journal, &u)) { + if (journal_file_next_evolve_usec(server.system_journal->file, &u)) { if (n >= u) t = 0; else diff --git a/src/journal/meson.build b/src/journal/meson.build index 171e276736..3eeed772cf 100644 --- a/src/journal/meson.build +++ b/src/journal/meson.build @@ -7,6 +7,8 @@ sources = files(''' journald-console.h journald-context.c journald-context.h + journald-file.c + journald-file.h journald-kmsg.c journald-kmsg.h journald-native.c @@ -92,6 +94,26 @@ tests += [ [libxz, liblz4, libselinux]], + + [['src/journal/test-journal.c'], + [libjournal_core, + libshared]], + + [['src/journal/test-journal-stream.c'], + [libjournal_core, + libshared]], + + [['src/journal/test-journal-flush.c'], + [libjournal_core, + libshared]], + + [['src/journal/test-journal-verify.c'], + [libjournal_core, + libshared]], + + [['src/journal/test-journal-interleaving.c'], + [libjournal_core, + libshared]], ] fuzzers += [ diff --git a/src/libsystemd/sd-journal/test-journal-flush.c b/src/journal/test-journal-flush.c similarity index 85% rename from src/libsystemd/sd-journal/test-journal-flush.c rename to src/journal/test-journal-flush.c index c6fb65791e..f0a7024002 100644 --- a/src/libsystemd/sd-journal/test-journal-flush.c +++ b/src/journal/test-journal-flush.c @@ -7,7 +7,7 @@ #include "alloc-util.h" #include "chattr-util.h" -#include "journal-file.h" +#include "journald-file.h" #include "journal-internal.h" #include "macro.h" #include "path-util.h" @@ -16,7 +16,7 @@ int main(int argc, char *argv[]) { _cleanup_free_ char *fn = NULL; char dn[] = "/var/tmp/test-journal-flush.XXXXXX"; - JournalFile *new_journal = NULL; + JournaldFile *new_journal = NULL; sd_journal *j = NULL; unsigned n = 0; int r; @@ -26,7 +26,7 @@ int main(int argc, char *argv[]) { fn = path_join(dn, "test.journal"); - r = journal_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, 0, false, NULL, NULL, NULL, NULL, &new_journal); + r = journald_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, 0, false, NULL, NULL, NULL, NULL, &new_journal); assert_se(r >= 0); if (argc > 1) @@ -49,7 +49,7 @@ int main(int argc, char *argv[]) { log_error_errno(r, "journal_file_move_to_object failed: %m"); assert_se(r >= 0); - r = journal_file_copy_entry(f, new_journal, o, f->current_offset); + r = journal_file_copy_entry(f, new_journal->file, o, f->current_offset); if (r < 0) log_warning_errno(r, "journal_file_copy_entry failed: %m"); assert_se(r >= 0 || @@ -63,7 +63,7 @@ int main(int argc, char *argv[]) { sd_journal_close(j); - (void) journal_file_close(new_journal); + (void) journald_file_close(new_journal); unlink(fn); assert_se(rmdir(dn) == 0); diff --git a/src/libsystemd/sd-journal/test-journal-interleaving.c b/src/journal/test-journal-interleaving.c similarity index 77% rename from src/libsystemd/sd-journal/test-journal-interleaving.c rename to src/journal/test-journal-interleaving.c index 62db2266c7..48be6a14bc 100644 --- a/src/libsystemd/sd-journal/test-journal-interleaving.c +++ b/src/journal/test-journal-interleaving.c @@ -8,7 +8,7 @@ #include "alloc-util.h" #include "chattr-util.h" #include "io-util.h" -#include "journal-file.h" +#include "journald-file.h" #include "journal-vacuum.h" #include "log.h" #include "parse-util.h" @@ -33,17 +33,17 @@ _noreturn_ static void log_assert_errno(const char *text, int error, const char log_assert_errno(#expr, -_r_, PROJECT_FILE, __LINE__, __PRETTY_FUNCTION__); \ } while (false) -static JournalFile *test_open(const char *name) { - JournalFile *f; - assert_ret(journal_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &f)); +static JournaldFile *test_open(const char *name) { + JournaldFile *f; + assert_ret(journald_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &f)); return f; } -static void test_close(JournalFile *f) { - (void) journal_file_close (f); +static void test_close(JournaldFile *f) { + (void) journald_file_close(f); } -static void append_number(JournalFile *f, int n, uint64_t *seqnum) { +static void append_number(JournaldFile *f, int n, uint64_t *seqnum) { char *p; dual_timestamp ts; static dual_timestamp previous_ts = {}; @@ -61,7 +61,7 @@ static void append_number(JournalFile *f, int n, uint64_t *seqnum) { assert_se(asprintf(&p, "NUMBER=%d", n) >= 0); iovec[0] = IOVEC_MAKE_STRING(p); - assert_ret(journal_file_append_entry(f, &ts, NULL, iovec, 1, seqnum, NULL, NULL)); + assert_ret(journal_file_append_entry(f->file, &ts, NULL, iovec, 1, seqnum, NULL, NULL)); free(p); } @@ -108,7 +108,7 @@ static void test_check_numbers_up (sd_journal *j, int count) { } static void setup_sequential(void) { - JournalFile *one, *two; + JournaldFile *one, *two; one = test_open("one.journal"); two = test_open("two.journal"); append_number(one, 1, NULL); @@ -120,7 +120,7 @@ static void setup_sequential(void) { } static void setup_interleaved(void) { - JournalFile *one, *two; + JournaldFile *one, *two; one = test_open("one.journal"); two = test_open("two.journal"); append_number(one, 1, NULL); @@ -199,14 +199,14 @@ static void test_skip(void (*setup)(void)) { static void test_sequence_numbers(void) { char t[] = "/var/tmp/journal-seq-XXXXXX"; - JournalFile *one, *two; + JournaldFile *one, *two; uint64_t seqnum = 0; sd_id128_t seqnum_id; mkdtemp_chdir_chattr(t); - assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0644, - true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &one) == 0); + assert_se(journald_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0644, + true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &one) == 0); append_number(one, 1, &seqnum); printf("seqnum=%"PRIu64"\n", seqnum); @@ -215,21 +215,21 @@ static void test_sequence_numbers(void) { printf("seqnum=%"PRIu64"\n", seqnum); assert_se(seqnum == 2); - assert_se(one->header->state == STATE_ONLINE); - assert_se(!sd_id128_equal(one->header->file_id, one->header->machine_id)); - assert_se(!sd_id128_equal(one->header->file_id, one->header->boot_id)); - assert_se(sd_id128_equal(one->header->file_id, one->header->seqnum_id)); + assert_se(one->file->header->state == STATE_ONLINE); + assert_se(!sd_id128_equal(one->file->header->file_id, one->file->header->machine_id)); + assert_se(!sd_id128_equal(one->file->header->file_id, one->file->header->boot_id)); + assert_se(sd_id128_equal(one->file->header->file_id, one->file->header->seqnum_id)); - memcpy(&seqnum_id, &one->header->seqnum_id, sizeof(sd_id128_t)); + memcpy(&seqnum_id, &one->file->header->seqnum_id, sizeof(sd_id128_t)); - assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0644, - true, UINT64_MAX, false, NULL, NULL, NULL, one, &two) == 0); + assert_se(journald_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0644, + true, UINT64_MAX, false, NULL, NULL, NULL, one, &two) == 0); - assert_se(two->header->state == STATE_ONLINE); - assert_se(!sd_id128_equal(two->header->file_id, one->header->file_id)); - assert_se(sd_id128_equal(one->header->machine_id, one->header->machine_id)); - assert_se(sd_id128_equal(one->header->boot_id, one->header->boot_id)); - assert_se(sd_id128_equal(one->header->seqnum_id, one->header->seqnum_id)); + assert_se(two->file->header->state == STATE_ONLINE); + assert_se(!sd_id128_equal(two->file->header->file_id, one->file->header->file_id)); + assert_se(sd_id128_equal(one->file->header->machine_id, one->file->header->machine_id)); + assert_se(sd_id128_equal(one->file->header->boot_id, one->file->header->boot_id)); + assert_se(sd_id128_equal(one->file->header->seqnum_id, one->file->header->seqnum_id)); append_number(two, 3, &seqnum); printf("seqnum=%"PRIu64"\n", seqnum); @@ -253,10 +253,10 @@ static void test_sequence_numbers(void) { /* restart server */ seqnum = 0; - assert_se(journal_file_open(-1, "two.journal", O_RDWR, 0, - true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &two) == 0); + assert_se(journald_file_open(-1, "two.journal", O_RDWR, 0, + true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &two) == 0); - assert_se(sd_id128_equal(two->header->seqnum_id, seqnum_id)); + assert_se(sd_id128_equal(two->file->header->seqnum_id, seqnum_id)); append_number(two, 7, &seqnum); printf("seqnum=%"PRIu64"\n", seqnum); @@ -281,7 +281,7 @@ static void test_sequence_numbers(void) { int main(int argc, char *argv[]) { test_setup_logging(LOG_DEBUG); - /* journal_file_open requires a valid machine id */ + /* journald_file_open requires a valid machine id */ if (access("/etc/machine-id", F_OK) != 0) return log_tests_skipped("/etc/machine-id not found"); diff --git a/src/libsystemd/sd-journal/test-journal-stream.c b/src/journal/test-journal-stream.c similarity index 85% rename from src/libsystemd/sd-journal/test-journal-stream.c rename to src/journal/test-journal-stream.c index ca1eaaebfc..f7fc4332a9 100644 --- a/src/libsystemd/sd-journal/test-journal-stream.c +++ b/src/journal/test-journal-stream.c @@ -8,7 +8,7 @@ #include "alloc-util.h" #include "chattr-util.h" #include "io-util.h" -#include "journal-file.h" +#include "journald-file.h" #include "journal-internal.h" #include "log.h" #include "macro.h" @@ -60,7 +60,7 @@ static void verify_contents(sd_journal *j, unsigned skip) { } static void run_test(void) { - JournalFile *one, *two, *three; + JournaldFile *one, *two, *three; char t[] = "/var/tmp/journal-stream-XXXXXX"; unsigned i; _cleanup_(sd_journal_closep) sd_journal *j = NULL; @@ -73,9 +73,9 @@ static void run_test(void) { assert_se(chdir(t) >= 0); (void) chattr_path(t, FS_NOCOW_FL, FS_NOCOW_FL, NULL); - assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &one) == 0); - assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &two) == 0); - assert_se(journal_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &three) == 0); + assert_se(journald_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &one) == 0); + assert_se(journald_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &two) == 0); + assert_se(journald_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &three) == 0); for (i = 0; i < N_ENTRIES; i++) { char *p, *q; @@ -100,21 +100,21 @@ static void run_test(void) { iovec[1] = IOVEC_MAKE(q, strlen(q)); if (i % 10 == 0) - assert_se(journal_file_append_entry(three, &ts, NULL, iovec, 2, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(three->file, &ts, NULL, iovec, 2, NULL, NULL, NULL) == 0); else { if (i % 3 == 0) - assert_se(journal_file_append_entry(two, &ts, NULL, iovec, 2, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(two->file, &ts, NULL, iovec, 2, NULL, NULL, NULL) == 0); - assert_se(journal_file_append_entry(one, &ts, NULL, iovec, 2, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(one->file, &ts, NULL, iovec, 2, NULL, NULL, NULL) == 0); } free(p); free(q); } - (void) journal_file_close(one); - (void) journal_file_close(two); - (void) journal_file_close(three); + (void) journald_file_close(one); + (void) journald_file_close(two); + (void) journald_file_close(three); assert_se(sd_journal_open_directory(&j, t, 0) >= 0); @@ -174,7 +174,7 @@ static void run_test(void) { int main(int argc, char *argv[]) { - /* journal_file_open requires a valid machine id */ + /* journald_file_open requires a valid machine id */ if (access("/etc/machine-id", F_OK) != 0) return log_tests_skipped("/etc/machine-id not found"); diff --git a/src/libsystemd/sd-journal/test-journal-verify.c b/src/journal/test-journal-verify.c similarity index 87% rename from src/libsystemd/sd-journal/test-journal-verify.c rename to src/journal/test-journal-verify.c index 8eb9ca88f7..cf9692b3d2 100644 --- a/src/libsystemd/sd-journal/test-journal-verify.c +++ b/src/journal/test-journal-verify.c @@ -7,7 +7,7 @@ #include "chattr-util.h" #include "fd-util.h" #include "io-util.h" -#include "journal-file.h" +#include "journald-file.h" #include "journal-verify.h" #include "log.h" #include "rm-rf.h" @@ -41,7 +41,7 @@ static int raw_verify(const char *fn, const char *verification_key) { JournalFile *f; int r; - r = journal_file_open(-1, fn, O_RDONLY, 0666, true, UINT64_MAX, !!verification_key, NULL, NULL, NULL, NULL, &f); + r = journal_file_open(-1, fn, O_RDONLY, 0666, true, UINT64_MAX, !!verification_key, NULL, NULL, NULL, &f); if (r < 0) return r; @@ -55,12 +55,13 @@ int main(int argc, char *argv[]) { char t[] = "/var/tmp/journal-XXXXXX"; unsigned n; JournalFile *f; + JournaldFile *df; const char *verification_key = argv[1]; usec_t from = 0, to = 0, total = 0; struct stat st; uint64_t p; - /* journal_file_open requires a valid machine id */ + /* journald_file_open requires a valid machine id */ if (access("/etc/machine-id", F_OK) != 0) return log_tests_skipped("/etc/machine-id not found"); @@ -72,7 +73,7 @@ int main(int argc, char *argv[]) { log_info("Generating..."); - assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0); + assert_se(journald_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, !!verification_key, NULL, NULL, NULL, NULL, &df) == 0); for (n = 0; n < N_ENTRIES; n++) { struct iovec iovec; @@ -85,16 +86,16 @@ int main(int argc, char *argv[]) { iovec = IOVEC_MAKE_STRING(test); - assert_se(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(df->file, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); free(test); } - (void) journal_file_close(f); + (void) journald_file_close(df); log_info("Verifying..."); - assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, UINT64_MAX, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0); + assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, UINT64_MAX, !!verification_key, NULL, NULL, NULL, &f) == 0); /* journal_file_print_header(f); */ journal_file_dump(f); diff --git a/src/libsystemd/sd-journal/test-journal.c b/src/journal/test-journal.c similarity index 56% rename from src/libsystemd/sd-journal/test-journal.c rename to src/journal/test-journal.c index bf69111d44..c2c4a75fef 100644 --- a/src/libsystemd/sd-journal/test-journal.c +++ b/src/journal/test-journal.c @@ -6,7 +6,7 @@ #include "chattr-util.h" #include "io-util.h" #include "journal-authenticate.h" -#include "journal-file.h" +#include "journald-file.h" #include "journal-vacuum.h" #include "log.h" #include "rm-rf.h" @@ -25,7 +25,7 @@ static void mkdtemp_chdir_chattr(char *path) { static void test_non_empty(void) { dual_timestamp ts; - JournalFile *f; + JournaldFile *f; struct iovec iovec; static const char test[] = "TEST1=1", test2[] = "TEST2=2"; Object *o; @@ -37,71 +37,71 @@ static void test_non_empty(void) { mkdtemp_chdir_chattr(t); - assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, true, NULL, NULL, NULL, NULL, &f) == 0); + assert_se(journald_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, true, NULL, NULL, NULL, NULL, &f) == 0); assert_se(dual_timestamp_get(&ts)); assert_se(sd_id128_randomize(&fake_boot_id) == 0); iovec = IOVEC_MAKE_STRING(test); - assert_se(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(f->file, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); iovec = IOVEC_MAKE_STRING(test2); - assert_se(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(f->file, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); iovec = IOVEC_MAKE_STRING(test); - assert_se(journal_file_append_entry(f, &ts, &fake_boot_id, &iovec, 1, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(f->file, &ts, &fake_boot_id, &iovec, 1, NULL, NULL, NULL) == 0); #if HAVE_GCRYPT - journal_file_append_tag(f); + journal_file_append_tag(f->file); #endif - journal_file_dump(f); + journal_file_dump(f->file); - assert_se(journal_file_next_entry(f, 0, DIRECTION_DOWN, &o, &p) == 1); + assert_se(journal_file_next_entry(f->file, 0, DIRECTION_DOWN, &o, &p) == 1); assert_se(le64toh(o->entry.seqnum) == 1); - assert_se(journal_file_next_entry(f, p, DIRECTION_DOWN, &o, &p) == 1); + assert_se(journal_file_next_entry(f->file, p, DIRECTION_DOWN, &o, &p) == 1); assert_se(le64toh(o->entry.seqnum) == 2); - assert_se(journal_file_next_entry(f, p, DIRECTION_DOWN, &o, &p) == 1); + assert_se(journal_file_next_entry(f->file, p, DIRECTION_DOWN, &o, &p) == 1); assert_se(le64toh(o->entry.seqnum) == 3); assert_se(sd_id128_equal(o->entry.boot_id, fake_boot_id)); - assert_se(journal_file_next_entry(f, p, DIRECTION_DOWN, &o, &p) == 0); + assert_se(journal_file_next_entry(f->file, p, DIRECTION_DOWN, &o, &p) == 0); - assert_se(journal_file_next_entry(f, 0, DIRECTION_DOWN, &o, &p) == 1); + assert_se(journal_file_next_entry(f->file, 0, DIRECTION_DOWN, &o, &p) == 1); assert_se(le64toh(o->entry.seqnum) == 1); - assert_se(journal_file_find_data_object(f, test, strlen(test), NULL, &p) == 1); - assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_DOWN, &o, NULL) == 1); + assert_se(journal_file_find_data_object(f->file, test, strlen(test), NULL, &p) == 1); + assert_se(journal_file_next_entry_for_data(f->file, NULL, 0, p, DIRECTION_DOWN, &o, NULL) == 1); assert_se(le64toh(o->entry.seqnum) == 1); - assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_UP, &o, NULL) == 1); + assert_se(journal_file_next_entry_for_data(f->file, NULL, 0, p, DIRECTION_UP, &o, NULL) == 1); assert_se(le64toh(o->entry.seqnum) == 3); - assert_se(journal_file_find_data_object(f, test2, strlen(test2), NULL, &p) == 1); - assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_UP, &o, NULL) == 1); + assert_se(journal_file_find_data_object(f->file, test2, strlen(test2), NULL, &p) == 1); + assert_se(journal_file_next_entry_for_data(f->file, NULL, 0, p, DIRECTION_UP, &o, NULL) == 1); assert_se(le64toh(o->entry.seqnum) == 2); - assert_se(journal_file_next_entry_for_data(f, NULL, 0, p, DIRECTION_DOWN, &o, NULL) == 1); + assert_se(journal_file_next_entry_for_data(f->file, NULL, 0, p, DIRECTION_DOWN, &o, NULL) == 1); assert_se(le64toh(o->entry.seqnum) == 2); - assert_se(journal_file_find_data_object(f, "quux", 4, NULL, &p) == 0); + assert_se(journal_file_find_data_object(f->file, "quux", 4, NULL, &p) == 0); - assert_se(journal_file_move_to_entry_by_seqnum(f, 1, DIRECTION_DOWN, &o, NULL) == 1); + assert_se(journal_file_move_to_entry_by_seqnum(f->file, 1, DIRECTION_DOWN, &o, NULL) == 1); assert_se(le64toh(o->entry.seqnum) == 1); - assert_se(journal_file_move_to_entry_by_seqnum(f, 3, DIRECTION_DOWN, &o, NULL) == 1); + assert_se(journal_file_move_to_entry_by_seqnum(f->file, 3, DIRECTION_DOWN, &o, NULL) == 1); assert_se(le64toh(o->entry.seqnum) == 3); - assert_se(journal_file_move_to_entry_by_seqnum(f, 2, DIRECTION_DOWN, &o, NULL) == 1); + assert_se(journal_file_move_to_entry_by_seqnum(f->file, 2, DIRECTION_DOWN, &o, NULL) == 1); assert_se(le64toh(o->entry.seqnum) == 2); - assert_se(journal_file_move_to_entry_by_seqnum(f, 10, DIRECTION_DOWN, &o, NULL) == 0); + assert_se(journal_file_move_to_entry_by_seqnum(f->file, 10, DIRECTION_DOWN, &o, NULL) == 0); - journal_file_rotate(&f, true, UINT64_MAX, true, NULL); - journal_file_rotate(&f, true, UINT64_MAX, true, NULL); + journald_file_rotate(&f, true, UINT64_MAX, true, NULL); + journald_file_rotate(&f, true, UINT64_MAX, true, NULL); - (void) journal_file_close(f); + (void) journald_file_close(f); log_info("Done..."); @@ -117,28 +117,25 @@ static void test_non_empty(void) { } static void test_empty(void) { - JournalFile *f1, *f2, *f3, *f4; + JournaldFile *f1, *f2, *f3, *f4; char t[] = "/var/tmp/journal-XXXXXX"; test_setup_logging(LOG_DEBUG); mkdtemp_chdir_chattr(t); - assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, UINT64_MAX, false, NULL, NULL, NULL, NULL, &f1) == 0); + assert_se(journald_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, UINT64_MAX, false, NULL, NULL, NULL, NULL, &f1) == 0); + assert_se(journald_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &f2) == 0); + assert_se(journald_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, UINT64_MAX, true, NULL, NULL, NULL, NULL, &f3) == 0); + assert_se(journald_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, true, NULL, NULL, NULL, NULL, &f4) == 0); - assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, false, NULL, NULL, NULL, NULL, &f2) == 0); - - assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, UINT64_MAX, true, NULL, NULL, NULL, NULL, &f3) == 0); - - assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, UINT64_MAX, true, NULL, NULL, NULL, NULL, &f4) == 0); - - journal_file_print_header(f1); + journal_file_print_header(f1->file); puts(""); - journal_file_print_header(f2); + journal_file_print_header(f2->file); puts(""); - journal_file_print_header(f3); + journal_file_print_header(f3->file); puts(""); - journal_file_print_header(f4); + journal_file_print_header(f4->file); puts(""); log_info("Done..."); @@ -151,16 +148,16 @@ static void test_empty(void) { assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0); } - (void) journal_file_close(f1); - (void) journal_file_close(f2); - (void) journal_file_close(f3); - (void) journal_file_close(f4); + (void) journald_file_close(f1); + (void) journald_file_close(f2); + (void) journald_file_close(f3); + (void) journald_file_close(f4); } #if HAVE_COMPRESSION static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) { dual_timestamp ts; - JournalFile *f; + JournaldFile *f; struct iovec iovec; Object *o; uint64_t p; @@ -175,34 +172,34 @@ static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) { mkdtemp_chdir_chattr(t); - assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, compress_threshold, true, NULL, NULL, NULL, NULL, &f) == 0); + assert_se(journald_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, compress_threshold, true, NULL, NULL, NULL, NULL, &f) == 0); dual_timestamp_get(&ts); iovec = IOVEC_MAKE(data, data_size); - assert_se(journal_file_append_entry(f, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); + assert_se(journal_file_append_entry(f->file, &ts, NULL, &iovec, 1, NULL, NULL, NULL) == 0); #if HAVE_GCRYPT - journal_file_append_tag(f); + journal_file_append_tag(f->file); #endif - journal_file_dump(f); + journal_file_dump(f->file); /* We have to partially reimplement some of the dump logic, because the normal next_entry does the * decompression for us. */ - p = le64toh(f->header->header_size); + p = le64toh(f->file->header->header_size); for (;;) { - r = journal_file_move_to_object(f, OBJECT_UNUSED, p, &o); + r = journal_file_move_to_object(f->file, OBJECT_UNUSED, p, &o); assert_se(r == 0); if (o->object.type == OBJECT_DATA) break; - assert_se(p < le64toh(f->header->tail_object_offset)); + assert_se(p < le64toh(f->file->header->tail_object_offset)); p = p + ALIGN64(le64toh(o->object.size)); } is_compressed = (o->object.flags & OBJECT_COMPRESSION_MASK) != 0; - (void) journal_file_close(f); + (void) journald_file_close(f); log_info("Done..."); @@ -245,7 +242,7 @@ int main(int argc, char *argv[]) { test_setup_logging(LOG_INFO); - /* journal_file_open requires a valid machine id */ + /* journald_file_open requires a valid machine id */ if (access("/etc/machine-id", F_OK) != 0) return log_tests_skipped("/etc/machine-id not found"); diff --git a/src/libsystemd/meson.build b/src/libsystemd/meson.build index 6ee78fd195..73c0228248 100644 --- a/src/libsystemd/meson.build +++ b/src/libsystemd/meson.build @@ -190,8 +190,6 @@ custom_target( ############################################################ tests += [ - [['src/libsystemd/sd-journal/test-journal.c']], - [['src/libsystemd/sd-journal/test-journal-send.c']], [['src/libsystemd/sd-journal/test-journal-match.c']], @@ -199,16 +197,8 @@ tests += [ [['src/libsystemd/sd-journal/test-journal-enum.c'], [], [], [], '', 'timeout=360'], - [['src/libsystemd/sd-journal/test-journal-stream.c']], - - [['src/libsystemd/sd-journal/test-journal-flush.c']], - [['src/libsystemd/sd-journal/test-journal-init.c']], - [['src/libsystemd/sd-journal/test-journal-verify.c']], - - [['src/libsystemd/sd-journal/test-journal-interleaving.c']], - [['src/libsystemd/sd-journal/test-mmap-cache.c']], [['src/libsystemd/sd-journal/test-catalog.c']], diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index 611c8c0d28..b2ec6cde54 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -3562,7 +3562,6 @@ int journal_file_open( bool seal, JournalMetrics *metrics, MMapCache *mmap_cache, - Set *deferred_closes, JournalFile *template, JournalFile **ret) { @@ -3747,8 +3746,6 @@ int journal_file_open( f->header = h; if (!newly_created) { - set_clear_with_destructor(deferred_closes, journal_file_close); - r = journal_file_verify_header(f); if (r < 0) goto fail; @@ -3871,65 +3868,6 @@ int journal_file_archive(JournalFile *f) { return 0; } -JournalFile* journal_initiate_close( - JournalFile *f, - Set *deferred_closes) { - - int r; - - assert(f); - - if (deferred_closes) { - - r = set_put(deferred_closes, f); - if (r < 0) - log_debug_errno(r, "Failed to add file to deferred close set, closing immediately."); - else { - (void) journal_file_set_offline(f, false); - return NULL; - } - } - - return journal_file_close(f); -} - -int journal_file_rotate( - JournalFile **f, - bool compress, - uint64_t compress_threshold_bytes, - bool seal, - Set *deferred_closes) { - - JournalFile *new_file = NULL; - int r; - - assert(f); - assert(*f); - - r = journal_file_archive(*f); - if (r < 0) - return r; - - r = journal_file_open( - -1, - (*f)->path, - (*f)->flags, - (*f)->mode, - compress, - compress_threshold_bytes, - seal, - NULL, /* metrics */ - (*f)->mmap, - deferred_closes, - *f, /* template */ - &new_file); - - journal_initiate_close(*f, deferred_closes); - *f = new_file; - - return r; -} - int journal_file_dispose(int dir_fd, const char *fname) { _cleanup_free_ char *p = NULL; _cleanup_close_ int fd = -1; @@ -3965,55 +3903,6 @@ int journal_file_dispose(int dir_fd, const char *fname) { return 0; } -int journal_file_open_reliably( - const char *fname, - int flags, - mode_t mode, - bool compress, - uint64_t compress_threshold_bytes, - bool seal, - JournalMetrics *metrics, - MMapCache *mmap_cache, - Set *deferred_closes, - JournalFile *template, - JournalFile **ret) { - - int r; - - r = journal_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, mmap_cache, - deferred_closes, template, ret); - if (!IN_SET(r, - -EBADMSG, /* Corrupted */ - -ENODATA, /* Truncated */ - -EHOSTDOWN, /* Other machine */ - -EPROTONOSUPPORT, /* Incompatible feature */ - -EBUSY, /* Unclean shutdown */ - -ESHUTDOWN, /* Already archived */ - -EIO, /* IO error, including SIGBUS on mmap */ - -EIDRM, /* File has been deleted */ - -ETXTBSY)) /* File is from the future */ - return r; - - if ((flags & O_ACCMODE) == O_RDONLY) - return r; - - if (!(flags & O_CREAT)) - return r; - - if (!endswith(fname, ".journal")) - return r; - - /* The file is corrupted. Rotate it away and try it again (but only once) */ - log_warning_errno(r, "File %s corrupted or uncleanly shut down, renaming and replacing.", fname); - - r = journal_file_dispose(AT_FDCWD, fname); - if (r < 0) - return r; - - return journal_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, mmap_cache, - deferred_closes, template, ret); -} - int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p) { uint64_t q, n, xor_hash = 0; const sd_id128_t *boot_id; diff --git a/src/libsystemd/sd-journal/journal-file.h b/src/libsystemd/sd-journal/journal-file.h index 81b2ff23e1..0fbd84e31c 100644 --- a/src/libsystemd/sd-journal/journal-file.h +++ b/src/libsystemd/sd-journal/journal-file.h @@ -138,7 +138,6 @@ int journal_file_open( bool seal, JournalMetrics *metrics, MMapCache *mmap_cache, - Set *deferred_closes, JournalFile *template, JournalFile **ret); @@ -148,19 +147,6 @@ JournalFile* journal_file_close(JournalFile *j); int journal_file_fstat(JournalFile *f); DEFINE_TRIVIAL_CLEANUP_FUNC(JournalFile*, journal_file_close); -int journal_file_open_reliably( - const char *fname, - int flags, - mode_t mode, - bool compress, - uint64_t compress_threshold_bytes, - bool seal, - JournalMetrics *metrics, - MMapCache *mmap_cache, - Set *deferred_closes, - JournalFile *template, - JournalFile **ret); - #define ALIGN64(x) (((x) + 7ULL) & ~7ULL) #define VALID64(x) (((x) & 7ULL) == 0ULL) @@ -246,8 +232,6 @@ void journal_file_dump(JournalFile *f); void journal_file_print_header(JournalFile *f); int journal_file_archive(JournalFile *f); -JournalFile* journal_initiate_close(JournalFile *f, Set *deferred_closes); -int journal_file_rotate(JournalFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes); int journal_file_dispose(int dir_fd, const char *fname); diff --git a/src/libsystemd/sd-journal/sd-journal.c b/src/libsystemd/sd-journal/sd-journal.c index 4861726673..b3f14cc548 100644 --- a/src/libsystemd/sd-journal/sd-journal.c +++ b/src/libsystemd/sd-journal/sd-journal.c @@ -1334,7 +1334,7 @@ static int add_any_file( goto finish; } - r = journal_file_open(fd, path, O_RDONLY, 0, false, 0, false, NULL, j->mmap, NULL, NULL, &f); + r = journal_file_open(fd, path, O_RDONLY, 0, false, 0, false, NULL, j->mmap, NULL, &f); if (r < 0) { log_debug_errno(r, "Failed to open journal file %s: %m", path); goto finish; From 764721cc07d76641e9e674416aa3c704f481a954 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Tue, 30 Nov 2021 16:39:35 +0100 Subject: [PATCH 2/7] journal: Move offlining logic to journald-file.c With this change, the logic to write the final tag, emit the final change notification and to offline the file moves from journal_file_close() to journald_file_close(). Since all this logic is only executed when the journal file is writable and all code that writes journal files already uses journald_file_close() instead of journal_file_close(), this change should not introduce any changes in behaviour. Moving the offline related logic to journald-file.c allows us to use code from src/shared in the offlining logic, more specifically, we can use the file copying logic from copy.h to fix BTRFS filesystem compression for journal files when archiving. --- src/journal/journald-file.c | 307 ++++++++++++++++++++++- src/journal/journald-file.h | 2 + src/journal/journald-server.c | 6 +- src/journal/meson.build | 1 + src/libsystemd/sd-journal/journal-file.c | 306 +--------------------- src/libsystemd/sd-journal/journal-file.h | 5 +- 6 files changed, 317 insertions(+), 310 deletions(-) diff --git a/src/journal/journald-file.c b/src/journal/journald-file.c index b12f40599b..5dba791517 100644 --- a/src/journal/journald-file.c +++ b/src/journal/journald-file.c @@ -1,5 +1,8 @@ /* SPDX-License-Identifier: LGPL-2.1-or-later */ +#include +#include + #include "chattr-util.h" #include "fd-util.h" #include "format-util.h" @@ -10,10 +13,312 @@ #include "set.h" #include "sync-util.h" +static int journald_file_truncate(JournalFile *f) { + uint64_t p; + int r; + + /* truncate excess from the end of archives */ + r = journal_file_tail_end(f, &p); + if (r < 0) + return log_debug_errno(r, "Failed to determine end of tail object: %m"); + + /* arena_size can't exceed the file size, ensure it's updated before truncating */ + f->header->arena_size = htole64(p - le64toh(f->header->header_size)); + + if (ftruncate(f->fd, p) < 0) + log_debug_errno(errno, "Failed to truncate %s: %m", f->path); + + return 0; +} + +static int journald_file_entry_array_punch_hole(JournalFile *f, uint64_t p, uint64_t n_entries) { + Object o; + uint64_t offset, sz, n_items = 0, n_unused; + int r; + + if (n_entries == 0) + return 0; + + for (uint64_t q = p; q != 0; q = le64toh(o.entry_array.next_entry_array_offset)) { + r = journal_file_read_object(f, OBJECT_ENTRY_ARRAY, q, &o); + if (r < 0) + return r; + + n_items += journal_file_entry_array_n_items(&o); + p = q; + } + + if (p == 0) + return 0; + + if (n_entries > n_items) + return -EBADMSG; + + /* Amount of unused items in the final entry array. */ + n_unused = n_items - n_entries; + + if (n_unused == 0) + return 0; + + offset = p + offsetof(Object, entry_array.items) + + (journal_file_entry_array_n_items(&o) - n_unused) * sizeof(le64_t); + sz = p + le64toh(o.object.size) - offset; + + if (fallocate(f->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, offset, sz) < 0) + return log_debug_errno(errno, "Failed to punch hole in entry array of %s: %m", f->path); + + return 0; +} + +static int journald_file_punch_holes(JournalFile *f) { + HashItem items[4096 / sizeof(HashItem)]; + uint64_t p, sz; + size_t to_read; + int r; + + r = journald_file_entry_array_punch_hole( + f, le64toh(f->header->entry_array_offset), le64toh(f->header->n_entries)); + if (r < 0) + return r; + + p = le64toh(f->header->data_hash_table_offset); + sz = le64toh(f->header->data_hash_table_size); + to_read = MIN((size_t) f->last_stat.st_blksize, sizeof(items)); + + for (uint64_t i = p; i < p + sz; i += sizeof(items)) { + ssize_t n_read; + + n_read = pread(f->fd, items, MIN(to_read, p + sz - i), i); + if (n_read < 0) + return n_read; + + for (size_t j = 0; j < (size_t) n_read / sizeof(HashItem); j++) { + Object o; + + for (uint64_t q = le64toh(items[j].head_hash_offset); q != 0; + q = le64toh(o.data.next_hash_offset)) { + + r = journal_file_read_object(f, OBJECT_DATA, q, &o); + if (r < 0) { + log_debug_errno(r, "Invalid data object: %m, ignoring"); + break; + } + + if (le64toh(o.data.n_entries) == 0) + continue; + + (void) journald_file_entry_array_punch_hole( + f, le64toh(o.data.entry_array_offset), le64toh(o.data.n_entries) - 1); + } + } + } + + return 0; +} + +/* This may be called from a separate thread to prevent blocking the caller for the duration of fsync(). + * As a result we use atomic operations on f->offline_state for inter-thread communications with + * journal_file_set_offline() and journal_file_set_online(). */ +static void journald_file_set_offline_internal(JournaldFile *f) { + assert(f); + assert(f->file->fd >= 0); + assert(f->file->header); + + for (;;) { + switch (f->file->offline_state) { + case OFFLINE_CANCEL: + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_CANCEL, OFFLINE_DONE)) + continue; + return; + + case OFFLINE_AGAIN_FROM_SYNCING: + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_AGAIN_FROM_SYNCING, OFFLINE_SYNCING)) + continue; + break; + + case OFFLINE_AGAIN_FROM_OFFLINING: + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_AGAIN_FROM_OFFLINING, OFFLINE_SYNCING)) + continue; + break; + + case OFFLINE_SYNCING: + if (f->file->archive) { + (void) journald_file_truncate(f->file); + (void) journald_file_punch_holes(f->file); + } + + (void) fsync(f->file->fd); + + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_SYNCING, OFFLINE_OFFLINING)) + continue; + + f->file->header->state = f->file->archive ? STATE_ARCHIVED : STATE_OFFLINE; + (void) fsync(f->file->fd); + break; + + case OFFLINE_OFFLINING: + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_OFFLINING, OFFLINE_DONE)) + continue; + _fallthrough_; + case OFFLINE_DONE: + return; + + case OFFLINE_JOINED: + log_debug("OFFLINE_JOINED unexpected offline state for journal_file_set_offline_internal()"); + return; + } + } +} + +static void * journald_file_set_offline_thread(void *arg) { + JournaldFile *f = arg; + + (void) pthread_setname_np(pthread_self(), "journal-offline"); + + journald_file_set_offline_internal(f); + + return NULL; +} + +/* Trigger a restart if the offline thread is mid-flight in a restartable state. */ +static bool journald_file_set_offline_try_restart(JournaldFile *f) { + for (;;) { + switch (f->file->offline_state) { + case OFFLINE_AGAIN_FROM_SYNCING: + case OFFLINE_AGAIN_FROM_OFFLINING: + return true; + + case OFFLINE_CANCEL: + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_CANCEL, OFFLINE_AGAIN_FROM_SYNCING)) + continue; + return true; + + case OFFLINE_SYNCING: + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_SYNCING, OFFLINE_AGAIN_FROM_SYNCING)) + continue; + return true; + + case OFFLINE_OFFLINING: + if (!__sync_bool_compare_and_swap(&f->file->offline_state, OFFLINE_OFFLINING, OFFLINE_AGAIN_FROM_OFFLINING)) + continue; + return true; + + default: + return false; + } + } +} + +/* Sets a journal offline. + * + * If wait is false then an offline is dispatched in a separate thread for a + * subsequent journal_file_set_offline() or journal_file_set_online() of the + * same journal to synchronize with. + * + * If wait is true, then either an existing offline thread will be restarted + * and joined, or if none exists the offline is simply performed in this + * context without involving another thread. + */ +int journald_file_set_offline(JournaldFile *f, bool wait) { + int target_state; + bool restarted; + int r; + + assert(f); + + if (!f->file->writable) + return -EPERM; + + if (f->file->fd < 0 || !f->file->header) + return -EINVAL; + + target_state = f->file->archive ? STATE_ARCHIVED : STATE_OFFLINE; + + /* An offlining journal is implicitly online and may modify f->header->state, + * we must also join any potentially lingering offline thread when already in + * the desired offline state. + */ + if (!journald_file_is_offlining(f) && f->file->header->state == target_state) + return journal_file_set_offline_thread_join(f->file); + + /* Restart an in-flight offline thread and wait if needed, or join a lingering done one. */ + restarted = journald_file_set_offline_try_restart(f); + if ((restarted && wait) || !restarted) { + r = journal_file_set_offline_thread_join(f->file); + if (r < 0) + return r; + } + + if (restarted) + return 0; + + /* Initiate a new offline. */ + f->file->offline_state = OFFLINE_SYNCING; + + if (wait) /* Without using a thread if waiting. */ + journald_file_set_offline_internal(f); + else { + sigset_t ss, saved_ss; + int k; + + assert_se(sigfillset(&ss) >= 0); + /* Don't block SIGBUS since the offlining thread accesses a memory mapped file. + * Asynchronous SIGBUS signals can safely be handled by either thread. */ + assert_se(sigdelset(&ss, SIGBUS) >= 0); + + r = pthread_sigmask(SIG_BLOCK, &ss, &saved_ss); + if (r > 0) + return -r; + + r = pthread_create(&f->file->offline_thread, NULL, journald_file_set_offline_thread, f); + + k = pthread_sigmask(SIG_SETMASK, &saved_ss, NULL); + if (r > 0) { + f->file->offline_state = OFFLINE_JOINED; + return -r; + } + if (k > 0) + return -k; + } + + return 0; +} + +bool journald_file_is_offlining(JournaldFile *f) { + assert(f); + + __sync_synchronize(); + + if (IN_SET(f->file->offline_state, OFFLINE_DONE, OFFLINE_JOINED)) + return false; + + return true; +} + JournaldFile* journald_file_close(JournaldFile *f) { if (!f) return NULL; +#if HAVE_GCRYPT + /* Write the final tag */ + if (f->file->seal && f->file->writable) { + int r; + + r = journal_file_append_tag(f->file); + if (r < 0) + log_error_errno(r, "Failed to append tag when closing journal: %m"); + } +#endif + + if (f->file->post_change_timer) { + if (sd_event_source_get_enabled(f->file->post_change_timer, NULL) > 0) + journal_file_post_change(f->file); + + sd_event_source_disable_unref(f->file->post_change_timer); + } + + journald_file_set_offline(f, true); + journal_file_close(f->file); return mfree(f); @@ -62,7 +367,7 @@ JournaldFile* journald_file_initiate_close(JournaldFile *f, Set *deferred_closes if (r < 0) log_debug_errno(r, "Failed to add file to deferred close set, closing immediately."); else { - (void) journal_file_set_offline(f->file, false); + (void) journald_file_set_offline(f, false); return NULL; } } diff --git a/src/journal/journald-file.h b/src/journal/journald-file.h index 7a299bd975..341043c836 100644 --- a/src/journal/journald-file.h +++ b/src/journal/journald-file.h @@ -21,6 +21,8 @@ int journald_file_open( JournaldFile *template, JournaldFile **ret); +int journald_file_set_offline(JournaldFile *f, bool wait); +bool journald_file_is_offlining(JournaldFile *f); JournaldFile* journald_file_close(JournaldFile *f); DEFINE_TRIVIAL_CLEANUP_FUNC(JournaldFile*, journald_file_close); diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 3d40c3822e..5e97cd41fd 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -482,7 +482,7 @@ static void server_process_deferred_closes(Server *s) { /* Perform any deferred closes which aren't still offlining. */ SET_FOREACH(f, s->deferred_closes) { - if (journal_file_is_offlining(f->file)) + if (journald_file_is_offlining(f)) continue; (void) set_remove(s->deferred_closes, f); @@ -647,13 +647,13 @@ void server_sync(Server *s) { int r; if (s->system_journal) { - r = journal_file_set_offline(s->system_journal->file, false); + r = journald_file_set_offline(s->system_journal, false); if (r < 0) log_warning_errno(r, "Failed to sync system journal, ignoring: %m"); } ORDERED_HASHMAP_FOREACH(f, s->user_journals) { - r = journal_file_set_offline(f->file, false); + r = journald_file_set_offline(f, false); if (r < 0) log_warning_errno(r, "Failed to sync user journal, ignoring: %m"); } diff --git a/src/journal/meson.build b/src/journal/meson.build index 3eeed772cf..383ed355fd 100644 --- a/src/journal/meson.build +++ b/src/journal/meson.build @@ -35,6 +35,7 @@ libjournal_core = static_library( 'journal-core', sources, include_directories : includes, + dependencies: threads, install : false) journal_includes = [includes, include_directories('.')] diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index b2ec6cde54..9755bf14b9 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -91,7 +91,7 @@ # pragma GCC diagnostic ignored "-Waddress-of-packed-member" #endif -static int journal_file_tail_end(JournalFile *f, uint64_t *ret_offset) { +int journal_file_tail_end(JournalFile *f, uint64_t *ret_offset) { Object tail; uint64_t p; int r; @@ -126,174 +126,7 @@ static int journal_file_tail_end(JournalFile *f, uint64_t *ret_offset) { return 0; } -static int journal_file_truncate(JournalFile *f) { - uint64_t p; - int r; - - /* truncate excess from the end of archives */ - r = journal_file_tail_end(f, &p); - if (r < 0) - return log_debug_errno(r, "Failed to determine end of tail object: %m"); - - /* arena_size can't exceed the file size, ensure it's updated before truncating */ - f->header->arena_size = htole64(p - le64toh(f->header->header_size)); - - if (ftruncate(f->fd, p) < 0) - log_debug_errno(errno, "Failed to truncate %s: %m", f->path); - - return 0; -} - -static int journal_file_entry_array_punch_hole(JournalFile *f, uint64_t p, uint64_t n_entries) { - Object o; - uint64_t offset, sz, n_items = 0, n_unused; - int r; - - if (n_entries == 0) - return 0; - - for (uint64_t q = p; q != 0; q = le64toh(o.entry_array.next_entry_array_offset)) { - r = journal_file_read_object(f, OBJECT_ENTRY_ARRAY, q, &o); - if (r < 0) - return r; - - n_items += journal_file_entry_array_n_items(&o); - p = q; - } - - if (p == 0) - return 0; - - if (n_entries > n_items) - return -EBADMSG; - - /* Amount of unused items in the final entry array. */ - n_unused = n_items - n_entries; - - if (n_unused == 0) - return 0; - - offset = p + offsetof(Object, entry_array.items) + - (journal_file_entry_array_n_items(&o) - n_unused) * sizeof(le64_t); - sz = p + le64toh(o.object.size) - offset; - - if (fallocate(f->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, offset, sz) < 0) - return log_debug_errno(errno, "Failed to punch hole in entry array of %s: %m", f->path); - - return 0; -} - -static int journal_file_punch_holes(JournalFile *f) { - HashItem items[4096 / sizeof(HashItem)]; - uint64_t p, sz; - size_t to_read; - int r; - - r = journal_file_entry_array_punch_hole( - f, le64toh(f->header->entry_array_offset), le64toh(f->header->n_entries)); - if (r < 0) - return r; - - p = le64toh(f->header->data_hash_table_offset); - sz = le64toh(f->header->data_hash_table_size); - to_read = MIN((size_t) f->last_stat.st_blksize, sizeof(items)); - - for (uint64_t i = p; i < p + sz; i += sizeof(items)) { - ssize_t n_read; - - n_read = pread(f->fd, items, MIN(to_read, p + sz - i), i); - if (n_read < 0) - return n_read; - - for (size_t j = 0; j < (size_t) n_read / sizeof(HashItem); j++) { - Object o; - - for (uint64_t q = le64toh(items[j].head_hash_offset); q != 0; - q = le64toh(o.data.next_hash_offset)) { - - r = journal_file_read_object(f, OBJECT_DATA, q, &o); - if (r < 0) { - log_debug_errno(r, "Invalid data object: %m, ignoring"); - break; - } - - if (le64toh(o.data.n_entries) == 0) - continue; - - (void) journal_file_entry_array_punch_hole( - f, le64toh(o.data.entry_array_offset), le64toh(o.data.n_entries) - 1); - } - } - } - - return 0; -} - -/* This may be called from a separate thread to prevent blocking the caller for the duration of fsync(). - * As a result we use atomic operations on f->offline_state for inter-thread communications with - * journal_file_set_offline() and journal_file_set_online(). */ -static void journal_file_set_offline_internal(JournalFile *f) { - assert(f); - assert(f->fd >= 0); - assert(f->header); - - for (;;) { - switch (f->offline_state) { - case OFFLINE_CANCEL: - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_CANCEL, OFFLINE_DONE)) - continue; - return; - - case OFFLINE_AGAIN_FROM_SYNCING: - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_AGAIN_FROM_SYNCING, OFFLINE_SYNCING)) - continue; - break; - - case OFFLINE_AGAIN_FROM_OFFLINING: - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_AGAIN_FROM_OFFLINING, OFFLINE_SYNCING)) - continue; - break; - - case OFFLINE_SYNCING: - if (f->archive) { - (void) journal_file_truncate(f); - (void) journal_file_punch_holes(f); - } - - (void) fsync(f->fd); - - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_SYNCING, OFFLINE_OFFLINING)) - continue; - - f->header->state = f->archive ? STATE_ARCHIVED : STATE_OFFLINE; - (void) fsync(f->fd); - break; - - case OFFLINE_OFFLINING: - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_OFFLINING, OFFLINE_DONE)) - continue; - _fallthrough_; - case OFFLINE_DONE: - return; - - case OFFLINE_JOINED: - log_debug("OFFLINE_JOINED unexpected offline state for journal_file_set_offline_internal()"); - return; - } - } -} - -static void * journal_file_set_offline_thread(void *arg) { - JournalFile *f = arg; - - (void) pthread_setname_np(pthread_self(), "journal-offline"); - - journal_file_set_offline_internal(f); - - return NULL; -} - -static int journal_file_set_offline_thread_join(JournalFile *f) { +int journal_file_set_offline_thread_join(JournalFile *f) { int r; assert(f); @@ -313,110 +146,6 @@ static int journal_file_set_offline_thread_join(JournalFile *f) { return 0; } -/* Trigger a restart if the offline thread is mid-flight in a restartable state. */ -static bool journal_file_set_offline_try_restart(JournalFile *f) { - for (;;) { - switch (f->offline_state) { - case OFFLINE_AGAIN_FROM_SYNCING: - case OFFLINE_AGAIN_FROM_OFFLINING: - return true; - - case OFFLINE_CANCEL: - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_CANCEL, OFFLINE_AGAIN_FROM_SYNCING)) - continue; - return true; - - case OFFLINE_SYNCING: - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_SYNCING, OFFLINE_AGAIN_FROM_SYNCING)) - continue; - return true; - - case OFFLINE_OFFLINING: - if (!__sync_bool_compare_and_swap(&f->offline_state, OFFLINE_OFFLINING, OFFLINE_AGAIN_FROM_OFFLINING)) - continue; - return true; - - default: - return false; - } - } -} - -/* Sets a journal offline. - * - * If wait is false then an offline is dispatched in a separate thread for a - * subsequent journal_file_set_offline() or journal_file_set_online() of the - * same journal to synchronize with. - * - * If wait is true, then either an existing offline thread will be restarted - * and joined, or if none exists the offline is simply performed in this - * context without involving another thread. - */ -int journal_file_set_offline(JournalFile *f, bool wait) { - int target_state; - bool restarted; - int r; - - assert(f); - - if (!f->writable) - return -EPERM; - - if (f->fd < 0 || !f->header) - return -EINVAL; - - target_state = f->archive ? STATE_ARCHIVED : STATE_OFFLINE; - - /* An offlining journal is implicitly online and may modify f->header->state, - * we must also join any potentially lingering offline thread when already in - * the desired offline state. - */ - if (!journal_file_is_offlining(f) && f->header->state == target_state) - return journal_file_set_offline_thread_join(f); - - /* Restart an in-flight offline thread and wait if needed, or join a lingering done one. */ - restarted = journal_file_set_offline_try_restart(f); - if ((restarted && wait) || !restarted) { - r = journal_file_set_offline_thread_join(f); - if (r < 0) - return r; - } - - if (restarted) - return 0; - - /* Initiate a new offline. */ - f->offline_state = OFFLINE_SYNCING; - - if (wait) /* Without using a thread if waiting. */ - journal_file_set_offline_internal(f); - else { - sigset_t ss, saved_ss; - int k; - - assert_se(sigfillset(&ss) >= 0); - /* Don't block SIGBUS since the offlining thread accesses a memory mapped file. - * Asynchronous SIGBUS signals can safely be handled by either thread. */ - assert_se(sigdelset(&ss, SIGBUS) >= 0); - - r = pthread_sigmask(SIG_BLOCK, &ss, &saved_ss); - if (r > 0) - return -r; - - r = pthread_create(&f->offline_thread, NULL, journal_file_set_offline_thread, f); - - k = pthread_sigmask(SIG_SETMASK, &saved_ss, NULL); - if (r > 0) { - f->offline_state = OFFLINE_JOINED; - return -r; - } - if (k > 0) - return -k; - } - - return 0; -} - static int journal_file_set_online(JournalFile *f) { bool wait = true; @@ -484,41 +213,10 @@ static int journal_file_set_online(JournalFile *f) { } } -bool journal_file_is_offlining(JournalFile *f) { - assert(f); - - __sync_synchronize(); - - if (IN_SET(f->offline_state, OFFLINE_DONE, OFFLINE_JOINED)) - return false; - - return true; -} - JournalFile* journal_file_close(JournalFile *f) { if (!f) return NULL; -#if HAVE_GCRYPT - /* Write the final tag */ - if (f->seal && f->writable) { - int r; - - r = journal_file_append_tag(f); - if (r < 0) - log_error_errno(r, "Failed to append tag when closing journal: %m"); - } -#endif - - if (f->post_change_timer) { - if (sd_event_source_get_enabled(f->post_change_timer, NULL) > 0) - journal_file_post_change(f); - - sd_event_source_disable_unref(f->post_change_timer); - } - - journal_file_set_offline(f, true); - if (f->mmap && f->cache_fd) mmap_cache_fd_free(f->cache_fd); diff --git a/src/libsystemd/sd-journal/journal-file.h b/src/libsystemd/sd-journal/journal-file.h index 0fbd84e31c..6916cc4ac3 100644 --- a/src/libsystemd/sd-journal/journal-file.h +++ b/src/libsystemd/sd-journal/journal-file.h @@ -141,8 +141,7 @@ int journal_file_open( JournalFile *template, JournalFile **ret); -int journal_file_set_offline(JournalFile *f, bool wait); -bool journal_file_is_offlining(JournalFile *f); +int journal_file_set_offline_thread_join(JournalFile *f); JournalFile* journal_file_close(JournalFile *j); int journal_file_fstat(JournalFile *f); DEFINE_TRIVIAL_CLEANUP_FUNC(JournalFile*, journal_file_close); @@ -190,6 +189,8 @@ static inline bool VALID_EPOCH(uint64_t u) { int journal_file_move_to_object(JournalFile *f, ObjectType type, uint64_t offset, Object **ret); int journal_file_read_object(JournalFile *f, ObjectType type, uint64_t offset, Object *ret); +int journal_file_tail_end(JournalFile *f, uint64_t *ret_offset); + uint64_t journal_file_entry_n_items(Object *o) _pure_; uint64_t journal_file_entry_array_n_items(Object *o) _pure_; uint64_t journal_file_hash_table_n_items(Object *o) _pure_; From e5f14b1eb849bca6ef3a1b3357aecbf6b2e3208f Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Wed, 1 Dec 2021 09:31:00 +0100 Subject: [PATCH 3/7] mkosi: Add e2fsprogs to installed packages for Fedora lsattr is useful to check chattr flags on files. --- mkosi.default.d/fedora/10-mkosi.fedora | 1 + 1 file changed, 1 insertion(+) diff --git a/mkosi.default.d/fedora/10-mkosi.fedora b/mkosi.default.d/fedora/10-mkosi.fedora index c348d66c34..1bf860a5e1 100644 --- a/mkosi.default.d/fedora/10-mkosi.fedora +++ b/mkosi.default.d/fedora/10-mkosi.fedora @@ -73,3 +73,4 @@ Packages= tpm2-tss less netcat + e2fsprogs From bea29a4296cb869af861035ccd749ac4cb7e7e16 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Wed, 1 Dec 2021 09:32:59 +0100 Subject: [PATCH 4/7] mkosi: Add compsize to list of installed packages on Fedora compsize is useful to check the compression ratio of journal files on filesystems that support compression such as BTRFS. --- mkosi.default.d/fedora/10-mkosi.fedora | 1 + 1 file changed, 1 insertion(+) diff --git a/mkosi.default.d/fedora/10-mkosi.fedora b/mkosi.default.d/fedora/10-mkosi.fedora index 1bf860a5e1..78bc0ec342 100644 --- a/mkosi.default.d/fedora/10-mkosi.fedora +++ b/mkosi.default.d/fedora/10-mkosi.fedora @@ -74,3 +74,4 @@ Packages= less netcat e2fsprogs + compsize From 461955ef4c786755471d1f49e51dd374c62f55ea Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Wed, 1 Dec 2021 09:34:13 +0100 Subject: [PATCH 5/7] journal: Update the JournalFile path when archiving When we archive a path, we rename the file to indicate this. However, until now, we didn't actually update the path member of the corresponding JournalFile instance. Let's make sure we also update this to avoid misuse of the old path later on. This change also requires we save the previous path in journal_file_rotate() since we need to open a new file at the previous path. --- src/journal/journald-file.c | 5 +++-- src/journal/journald-server.c | 2 +- src/libsystemd/sd-journal/journal-file.c | 9 ++++++++- src/libsystemd/sd-journal/journal-file.h | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/journal/journald-file.c b/src/journal/journald-file.c index 5dba791517..fc46aa1963 100644 --- a/src/journal/journald-file.c +++ b/src/journal/journald-file.c @@ -382,19 +382,20 @@ int journald_file_rotate( bool seal, Set *deferred_closes) { + _cleanup_free_ char *path = NULL; JournaldFile *new_file = NULL; int r; assert(f); assert(*f); - r = journal_file_archive((*f)->file); + r = journal_file_archive((*f)->file, &path); if (r < 0) return r; r = journald_file_open( -1, - (*f)->file->path, + path, (*f)->file->flags, (*f)->file->mode, compress, diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c index 5e97cd41fd..a13bbeeee9 100644 --- a/src/journal/journald-server.c +++ b/src/journal/journald-server.c @@ -602,7 +602,7 @@ static int vacuum_offline_user_journals(Server *s) { TAKE_FD(fd); /* Donated to journald_file_open() */ - r = journal_file_archive(f->file); + r = journal_file_archive(f->file, NULL); if (r < 0) log_debug_errno(r, "Failed to archive journal file '%s', ignoring: %m", full); diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index 9755bf14b9..bc80a51524 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -3521,7 +3521,7 @@ fail: return r; } -int journal_file_archive(JournalFile *f) { +int journal_file_archive(JournalFile *f, char **ret_previous_path) { _cleanup_free_ char *p = NULL; assert(f); @@ -3552,6 +3552,13 @@ int journal_file_archive(JournalFile *f) { /* Sync the rename to disk */ (void) fsync_directory_of_file(f->fd); + if (ret_previous_path) + *ret_previous_path = f->path; + else + free(f->path); + + f->path = TAKE_PTR(p); + /* Set as archive so offlining commits w/state=STATE_ARCHIVED. Previously we would set old_file->header->state * to STATE_ARCHIVED directly here, but journal_file_set_offline() short-circuits when state != STATE_ONLINE, * which would result in the rotated journal never getting fsync() called before closing. Now we simply queue diff --git a/src/libsystemd/sd-journal/journal-file.h b/src/libsystemd/sd-journal/journal-file.h index 6916cc4ac3..5bcf591337 100644 --- a/src/libsystemd/sd-journal/journal-file.h +++ b/src/libsystemd/sd-journal/journal-file.h @@ -232,7 +232,7 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6 void journal_file_dump(JournalFile *f); void journal_file_print_header(JournalFile *f); -int journal_file_archive(JournalFile *f); +int journal_file_archive(JournalFile *f, char **ret_previous_path); int journal_file_dispose(int dir_fd, const char *fname); From 5a980196ab7a8a4291c18effc5054fa6198a24a2 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 3 Dec 2021 11:07:02 +0100 Subject: [PATCH 6/7] basic: Make sure we're extra paranoid in chattr_full On btrfs, trying to disable FS_NOCOW_FL on a file that has data already written will fail silently without reporting an error. To catch such cases, let's query the flags again if the IOC_SETFLAGS ioctl() succeeds to make sure the flags we tried to configure we're actually accepted by the kernel. --- src/basic/chattr-util.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/basic/chattr-util.c b/src/basic/chattr-util.c index 59707a79c0..807288a649 100644 --- a/src/basic/chattr-util.c +++ b/src/basic/chattr-util.c @@ -59,11 +59,25 @@ int chattr_full(const char *path, } if (ioctl(fd, FS_IOC_SETFLAGS, &new_attr) >= 0) { - if (ret_previous) - *ret_previous = old_attr; - if (ret_final) - *ret_final = new_attr; - return 1; + unsigned attr; + + /* Some filesystems (BTRFS) silently fail when a flag cannot be set. Let's make sure our + * changes actually went through by querying the flags again and verifying they're equal to + * the flags we tried to configure. */ + + if (ioctl(fd, FS_IOC_GETFLAGS, &attr) < 0) + return -errno; + + if (new_attr == attr) { + if (ret_previous) + *ret_previous = old_attr; + if (ret_final) + *ret_final = new_attr; + return 1; + } + + /* Trigger the fallback logic. */ + errno = EINVAL; } if ((errno != EINVAL && !ERRNO_IS_NOT_SUPPORTED(errno)) || From d71ece3f0b85c7a3decc50143b68ac07fc5831ae Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 3 Dec 2021 11:06:36 +0100 Subject: [PATCH 7/7] journal: Fix disabling NO_COW on btrfs filesystems Disabling NOCOW when data has been written to a file doesn't work. Instead, when we're done writing to a journal file (after archiving), let's rewrite the file with COW enabled. This also takes care of properly defragmenting the file. With zstd compression level 3, journal files are compressed to 12% of their original size with default journal settings. As rewriting the file might take a while since we also do an fsync() after the rewrite, this work is done in the offline thread to avoid blocking the journald event loop. --- src/journal/journald-file.c | 26 ++++++++++++++++++++++++ src/libsystemd/sd-journal/journal-file.c | 26 ------------------------ src/libsystemd/sd-journal/journal-file.h | 1 - 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/src/journal/journald-file.c b/src/journal/journald-file.c index fc46aa1963..7f607d4aa6 100644 --- a/src/journal/journald-file.c +++ b/src/journal/journald-file.c @@ -4,6 +4,7 @@ #include #include "chattr-util.h" +#include "copy.h" #include "fd-util.h" #include "format-util.h" #include "journal-authenticate.h" @@ -11,6 +12,7 @@ #include "path-util.h" #include "random-util.h" #include "set.h" +#include "stat-util.h" #include "sync-util.h" static int journald_file_truncate(JournalFile *f) { @@ -120,6 +122,8 @@ static int journald_file_punch_holes(JournalFile *f) { * As a result we use atomic operations on f->offline_state for inter-thread communications with * journal_file_set_offline() and journal_file_set_online(). */ static void journald_file_set_offline_internal(JournaldFile *f) { + int r; + assert(f); assert(f->file->fd >= 0); assert(f->file->header); @@ -154,6 +158,28 @@ static void journald_file_set_offline_internal(JournaldFile *f) { f->file->header->state = f->file->archive ? STATE_ARCHIVED : STATE_OFFLINE; (void) fsync(f->file->fd); + + /* If we've archived the journal file, first try to re-enable COW on the file. If the + * FS_NOCOW_FL flag was never set or we succesfully removed it, continue. If we fail + * to remove the flag on the archived file, rewrite the file without the NOCOW flag. + * We need this fallback because on some filesystems (BTRFS), the NOCOW flag cannot + * be removed after data has been written to a file. The only way to remove it is to + * copy all data to a new file without the NOCOW flag set. */ + + if (f->file->archive) { + r = chattr_fd(f->file->fd, 0, FS_NOCOW_FL, NULL); + if (r >= 0) + continue; + + log_debug_errno(r, "Failed to re-enable copy-on-write for %s: %m, rewriting file", f->file->path); + + r = copy_file_atomic(f->file->path, f->file->path, f->file->mode, 0, FS_NOCOW_FL, COPY_REPLACE | COPY_FSYNC); + if (r < 0) { + log_debug_errno(r, "Failed to rewrite %s: %m", f->file->path); + continue; + } + } + break; case OFFLINE_OFFLINING: diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index bc80a51524..11b9da1cb5 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -220,18 +220,6 @@ JournalFile* journal_file_close(JournalFile *f) { if (f->mmap && f->cache_fd) mmap_cache_fd_free(f->cache_fd); - if (f->fd >= 0 && f->defrag_on_close) { - - /* Be friendly to btrfs: turn COW back on again now, - * and defragment the file. We won't write to the file - * ever again, hence remove all fragmentation, and - * reenable all the good bits COW usually provides - * (such as data checksumming). */ - - (void) chattr_fd(f->fd, 0, FS_NOCOW_FL, NULL); - (void) btrfs_defrag_fd(f->fd); - } - if (f->close_fd) safe_close(f->fd); free(f->path); @@ -3566,16 +3554,11 @@ int journal_file_archive(JournalFile *f, char **ret_previous_path) { * occurs. */ f->archive = true; - /* Currently, btrfs is not very good with out write patterns and fragments heavily. Let's defrag our journal - * files when we archive them */ - f->defrag_on_close = true; - return 0; } int journal_file_dispose(int dir_fd, const char *fname) { _cleanup_free_ char *p = NULL; - _cleanup_close_ int fd = -1; assert(fname); @@ -3596,15 +3579,6 @@ int journal_file_dispose(int dir_fd, const char *fname) { if (renameat(dir_fd, fname, dir_fd, p) < 0) return -errno; - /* btrfs doesn't cope well with our write pattern and fragments heavily. Let's defrag all files we rotate */ - fd = openat(dir_fd, p, O_RDONLY|O_CLOEXEC|O_NOCTTY|O_NOFOLLOW); - if (fd < 0) - log_debug_errno(errno, "Failed to open file for defragmentation/FS_NOCOW_FL, ignoring: %m"); - else { - (void) chattr_fd(fd, 0, FS_NOCOW_FL, NULL); - (void) btrfs_defrag_fd(fd); - } - return 0; } diff --git a/src/libsystemd/sd-journal/journal-file.h b/src/libsystemd/sd-journal/journal-file.h index 5bcf591337..ecda2b3cc0 100644 --- a/src/libsystemd/sd-journal/journal-file.h +++ b/src/libsystemd/sd-journal/journal-file.h @@ -68,7 +68,6 @@ typedef struct JournalFile { bool compress_lz4:1; bool compress_zstd:1; bool seal:1; - bool defrag_on_close:1; bool close_fd:1; bool archive:1; bool keyed_hash:1;