From 58ecb72bcf5b2f25057d58fe52ce9717c0d07382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Wed, 30 Jul 2025 11:33:35 +0200 Subject: [PATCH 1/3] test-journal-dump: dump the headers of journal files We have journal_file_print_header(), but it's not exposed anywhere in a way that it is easy to call. --- src/libsystemd/meson.build | 4 ++ src/libsystemd/sd-journal/test-journal-dump.c | 46 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 src/libsystemd/sd-journal/test-journal-dump.c diff --git a/src/libsystemd/meson.build b/src/libsystemd/meson.build index fa483836fa..1cec325238 100644 --- a/src/libsystemd/meson.build +++ b/src/libsystemd/meson.build @@ -275,6 +275,10 @@ libsystemd_tests += [ 'sources' : files('sd-journal/test-journal-append.c'), 'type' : 'manual', }, + { + 'sources' : files('sd-journal/test-journal-dump.c'), + 'type' : 'manual', + }, { 'sources' : files('sd-journal/test-journal-verify.c'), 'timeout' : 90, diff --git a/src/libsystemd/sd-journal/test-journal-dump.c b/src/libsystemd/sd-journal/test-journal-dump.c new file mode 100644 index 0000000000..bac2fb9a66 --- /dev/null +++ b/src/libsystemd/sd-journal/test-journal-dump.c @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include "errno-util.h" +#include "journal-file.h" +#include "log.h" +#include "main-func.h" +#include "pager.h" +#include "strv.h" + +static int run(int argc, char *argv[]) { + int r = 0; + unsigned n = 0; + + _cleanup_(mmap_cache_unrefp) MMapCache *m = mmap_cache_new(); + assert_se(m); + + pager_open(/* flags= */ 0); + + STRV_FOREACH(s, strv_skip(argv, 1)) { + JournalFile *f = NULL; + + int k = journal_file_open( + /* fd= */ -EBADF, + *s, + O_RDONLY, + /* file_flags= */ 0, + 0666, + /* compress_threshold_bytes= */ UINT64_MAX, + /* metrics= */ NULL, + m, + /* template= */ NULL, + &f); + if (k < 0) + RET_GATHER(r, log_error_errno(k, "Failed to open %s, continuing: %m", *s)); + + if (n++ > 0) + puts(""); + + journal_file_print_header(f); + journal_file_close(f); + } + + return r; +} + +DEFINE_MAIN_FUNCTION(run); From fbbcd0edefd2921b117eebfeab8eefc3d26bd0c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Wed, 30 Jul 2025 11:39:35 +0200 Subject: [PATCH 2/3] journal: store counts, not byte sizes, in table size constants It's easier to think about the size in "objects", not bytes. Let's convert to bytes at the last moment. Also drop some of the pointless size suffixes. In general, it's the size of the variable that matters, not the constant that is written to it. No functional change. --- src/libsystemd/sd-journal/journal-file.c | 27 ++++++++++++------------ 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index f47e94fac8..3e9da297ba 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -46,11 +46,11 @@ #include "user-util.h" #include "xattr-util.h" -#define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*sizeof(HashItem)) -#define DEFAULT_FIELD_HASH_TABLE_SIZE (333ULL*sizeof(HashItem)) +#define DEFAULT_DATA_HASH_TABLE_SIZE 2047U +#define DEFAULT_FIELD_HASH_TABLE_SIZE 333U -#define DEFAULT_COMPRESS_THRESHOLD (512ULL) -#define MIN_COMPRESS_THRESHOLD (8ULL) +#define DEFAULT_COMPRESS_THRESHOLD 512U +#define MIN_COMPRESS_THRESHOLD 8U /* This is the minimum journal file size */ #define JOURNAL_FILE_SIZE_MIN (512 * U64_KB) /* 512 KiB */ @@ -1286,15 +1286,14 @@ static int journal_file_setup_data_hash_table(JournalFile *f) { beyond 75% fill level. Calculate the hash table size for the maximum file size based on these metrics. */ - s = (f->metrics.max_size * 4 / 768 / 3) * sizeof(HashItem); - if (s < DEFAULT_DATA_HASH_TABLE_SIZE) - s = DEFAULT_DATA_HASH_TABLE_SIZE; + s = MAX(f->metrics.max_size * 4 / 768 / 3, + DEFAULT_DATA_HASH_TABLE_SIZE); - log_debug("Reserving %"PRIu64" entries in data hash table.", s / sizeof(HashItem)); + log_debug("Reserving %"PRIu64" entries in data hash table.", s); r = journal_file_append_object(f, OBJECT_DATA_HASH_TABLE, - offsetof(Object, hash_table.items) + s, + offsetof(Object, hash_table.items) + s * sizeof(HashItem), &o, &p); if (r < 0) return r; @@ -1302,7 +1301,7 @@ static int journal_file_setup_data_hash_table(JournalFile *f) { memzero(o->hash_table.items, s); f->header->data_hash_table_offset = htole64(p + offsetof(Object, hash_table.items)); - f->header->data_hash_table_size = htole64(s); + f->header->data_hash_table_size = htole64(s * sizeof(HashItem)); return 0; } @@ -1319,19 +1318,19 @@ static int journal_file_setup_field_hash_table(JournalFile *f) { * number should grow very slowly only */ s = DEFAULT_FIELD_HASH_TABLE_SIZE; - log_debug("Reserving %"PRIu64" entries in field hash table.", s / sizeof(HashItem)); + log_debug("Reserving %"PRIu64" entries in field hash table.", s); r = journal_file_append_object(f, OBJECT_FIELD_HASH_TABLE, - offsetof(Object, hash_table.items) + s, + offsetof(Object, hash_table.items) + s * sizeof(HashItem), &o, &p); if (r < 0) return r; - memzero(o->hash_table.items, s); + memzero(o->hash_table.items, s * sizeof(HashItem)); f->header->field_hash_table_offset = htole64(p + offsetof(Object, hash_table.items)); - f->header->field_hash_table_size = htole64(s); + f->header->field_hash_table_size = htole64(s * sizeof(HashItem)); return 0; } From e8962d77ac5a10926d0216246cfb5ad5dc25a533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Wed, 30 Jul 2025 11:52:26 +0200 Subject: [PATCH 3/3] journal: treble field hash table size As discussed in https://github.com/systemd/systemd/issues/38399, "ordinary" systems can have the field table with a large number of values, causing journal rotation to occur early. For example, audit generates a log of fields: $ journalctl --fields | rg -c '^_?AUDIT' 114 It seems that the "structured log" capabilities of the journal are being use more than in the past. Looking at some journal files on my system, it seems the field hash table field is quite high in many cases: $ build/test-journal-dump /var/log/journal/*/* | rg 'table fill' Data hash table fill: 15.1% Field hash table fill: 69.1% Data hash table fill: 4.9% Field hash table fill: 32.4% Data hash table fill: 10.2% Field hash table fill: 34.2% Data hash table fill: 9.9% Field hash table fill: 37.2% Data hash table fill: 26.8% Field hash table fill: 21.9% Data hash table fill: 35.6% Field hash table fill: 22.8% Data hash table fill: 25.5% Field hash table fill: 54.1% Data hash table fill: 3.4% Field hash table fill: 43.8% Data hash table fill: 75.0% Field hash table fill: 70.3% Data hash table fill: 75.0% Field hash table fill: 63.1% Data hash table fill: 75.0% Field hash table fill: 74.2% Data hash table fill: 35.6% Field hash table fill: 43.2% Data hash table fill: 35.5% Field hash table fill: 75.4% Data hash table fill: 75.0% Field hash table fill: 59.8% Data hash table fill: 75.0% Field hash table fill: 56.5% Data hash table fill: 16.9% Field hash table fill: 76.3% Data hash table fill: 18.1% Field hash table fill: 76.9% Data hash table fill: 75.0% Field hash table fill: 42.0% Data hash table fill: 75.0% Field hash table fill: 22.8% Data hash table fill: 75.0% Field hash table fill: 22.8% Data hash table fill: 75.0% Field hash table fill: 22.8% Data hash table fill: 75.0% Field hash table fill: 22.8% Data hash table fill: 75.0% Field hash table fill: 32.1% Data hash table fill: 75.0% Field hash table fill: 21.9% Data hash table fill: 75.0% Field hash table fill: 21.9% Data hash table fill: 75.0% Field hash table fill: 21.9% Data hash table fill: 75.0% Field hash table fill: 22.8% Data hash table fill: 75.0% Field hash table fill: 22.8% Data hash table fill: 75.0% Field hash table fill: 21.9% Data hash table fill: 75.0% Field hash table fill: 22.5% Data hash table fill: 9.6% Field hash table fill: 53.8% Data hash table fill: 75.0% Field hash table fill: 22.2% Data hash table fill: 75.0% Field hash table fill: 22.2% Data hash table fill: 75.0% Field hash table fill: 22.2% Data hash table fill: 35.6% Field hash table fill: 75.1% Data hash table fill: 33.6% Field hash table fill: 50.2% Data hash table fill: 75.0% Field hash table fill: 26.7% Data hash table fill: 75.0% Field hash table fill: 25.8% Data hash table fill: 75.0% Field hash table fill: 29.1% Data hash table fill: 75.0% Field hash table fill: 25.8% Data hash table fill: 75.0% Field hash table fill: 31.8% Data hash table fill: 75.0% Field hash table fill: 18.9% Data hash table fill: 75.0% Field hash table fill: 22.2% Data hash table fill: 75.0% Field hash table fill: 20.1% Data hash table fill: 75.0% Field hash table fill: 29.1% Data hash table fill: 75.0% Field hash table fill: 30.9% Data hash table fill: 75.0% Field hash table fill: 28.5% Data hash table fill: 75.0% Field hash table fill: 28.5% Data hash table fill: 75.0% Field hash table fill: 25.8% Data hash table fill: 75.0% Field hash table fill: 25.2% Data hash table fill: 75.0% Field hash table fill: 39.3% Data hash table fill: 50.2% Field hash table fill: 75.1% Data hash table fill: 75.0% Field hash table fill: 61.9% Data hash table fill: 75.0% Field hash table fill: 56.5% Data hash table fill: 75.0% Field hash table fill: 58.6% Data hash table fill: 48.9% Field hash table fill: 79.6% Data hash table fill: 75.0% Field hash table fill: 71.5% Data hash table fill: 75.0% Field hash table fill: 60.1% Data hash table fill: 31.4% Field hash table fill: 75.7% Data hash table fill: 27.0% Field hash table fill: 69.4% Data hash table fill: 28.9% Field hash table fill: 76.6% Data hash table fill: 60.2% Field hash table fill: 79.9% Data hash table fill: 8.8% Field hash table fill: 78.7% Data hash table fill: 5.8% Field hash table fill: 61.3% Data hash table fill: 75.0% Field hash table fill: 64.0% Data hash table fill: 61.4% Field hash table fill: 63.4% Data hash table fill: 29.7% Field hash table fill: 61.9% Data hash table fill: 18.9% Field hash table fill: 30.9% Data hash table fill: 1.4% Field hash table fill: 22.2% Data hash table fill: 0.4% Field hash table fill: 13.5% Data hash table fill: 2.6% Field hash table fill: 37.5% Data hash table fill: 1.3% Field hash table fill: 23.4% Data hash table fill: 0.6% Field hash table fill: 15.3% Data hash table fill: 18.7% Field hash table fill: 33.9% Data hash table fill: 7.4% Field hash table fill: 37.5% Data hash table fill: 20.2% Field hash table fill: 44.1% Data hash table fill: 1.3% Field hash table fill: 33.0% Data hash table fill: 75.0% Field hash table fill: 19.2% Data hash table fill: 42.2% Field hash table fill: 23.4% Data hash table fill: 1.6% Field hash table fill: 87.1% Data hash table fill: 0.1% Field hash table fill: 98.8% Data hash table fill: 0.2% Field hash table fill: 128.8% Data hash table fill: 15.4% Field hash table fill: 31.2% Data hash table fill: 7.4% Field hash table fill: 22.5% Data hash table fill: 10.5% Field hash table fill: 38.7% Data hash table fill: 2.8% Field hash table fill: 18.0% Data hash table fill: 1.5% Field hash table fill: 15.9% Data hash table fill: 0.0% Field hash table fill: 7.5% Data hash table fill: 0.1% Field hash table fill: 12.0% Data hash table fill: 0.2% Field hash table fill: 10.8% Data hash table fill: 0.2% Field hash table fill: 15.6% Data hash table fill: 0.1% Field hash table fill: 11.7% Data hash table fill: 0.1% Field hash table fill: 12.0% Data hash table fill: 0.0% Field hash table fill: 6.6% Data hash table fill: 1.4% Field hash table fill: 18.0% Data hash table fill: 0.7% Field hash table fill: 16.8% Data hash table fill: 1.1% Field hash table fill: 18.0% Data hash table fill: 0.2% Field hash table fill: 10.8% Data hash table fill: 0.1% Field hash table fill: 10.8% Data hash table fill: 0.4% Field hash table fill: 11.1% Since filling of the field hash table to 75% normally causes file rotation, let's double the default to make rotation happen less often. We'll use 11kB more for the hash table, which should be fine, considering that journal files are usually at least 8 MB. Closes https://github.com/systemd/systemd/issues/38399. --- src/libsystemd/sd-journal/journal-file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsystemd/sd-journal/journal-file.c b/src/libsystemd/sd-journal/journal-file.c index 3e9da297ba..2a08f923d1 100644 --- a/src/libsystemd/sd-journal/journal-file.c +++ b/src/libsystemd/sd-journal/journal-file.c @@ -47,7 +47,7 @@ #include "xattr-util.h" #define DEFAULT_DATA_HASH_TABLE_SIZE 2047U -#define DEFAULT_FIELD_HASH_TABLE_SIZE 333U +#define DEFAULT_FIELD_HASH_TABLE_SIZE 1023U #define DEFAULT_COMPRESS_THRESHOLD 512U #define MIN_COMPRESS_THRESHOLD 8U