From 2779ecaca3d01a6391a429cf58cab02f77ac0262 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 27 Aug 2026 12:17:44 +0200 Subject: [PATCH] string-util: port strip_tab_ansi() to AnsiSeqParser So far strip_tab_ansi() was very superficial: it only focussed on OSC and CSI sequences, and left the other ANSI sequences in touch. Let's make it more powerful and port it to the more complete AnsiSeqParser logic, and strip *all* ANSI sequences. For the relevant usecases this seems like the better, safer choice, since it's good to strip too much rather than too little, when stripping ANSI sequences. --- src/basic/string-util.c | 175 +++++++++++++++--------------- src/test/test-strip-tab-ansi.c | 192 +++++++++++++++++++++++++++++++-- 2 files changed, 269 insertions(+), 98 deletions(-) diff --git a/src/basic/string-util.c b/src/basic/string-util.c index efd690b076..77ba8b05a0 100644 --- a/src/basic/string-util.c +++ b/src/basic/string-util.c @@ -3,6 +3,7 @@ #include #include "alloc-util.h" +#include "ansi-seq.h" #include "escape.h" #include "extract-word.h" #include "glyph-util.h" @@ -14,7 +15,6 @@ #include "path-util.h" #include "string-util.h" #include "strv.h" -#include "terminal-util.h" #include "utf8.h" char* first_word(const char *s, const char *word) { @@ -701,16 +701,32 @@ static void advance_offsets( shift[1] += size; } +static void retract_offsets( + size_t begin, + size_t end, + size_t offsets[2], /* note: we can't use [static 2] here, since this may be NULL */ + size_t shift[static 2]) { + + /* The input range [begin, end) was dropped from the output: offsets past it move back by its + * length, offsets inside it collapse onto its beginning. The shift is tracked in unsigned modular + * arithmetic, which is fine as the sum of all shifts applied to an offset is never negative: at most + * as many bytes can be dropped before an offset as there are bytes before it. */ + + if (!offsets) + return; + + assert(shift); + assert(begin <= end); + + for (size_t k = 0; k < 2; k++) + if (offsets[k] > begin) + shift[k] -= MIN(offsets[k], end) - begin; +} + char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { - const char *begin = NULL; - enum { - STATE_OTHER, - STATE_ESCAPE, - STATE_CSI, - STATE_OSC, - STATE_OSC_CLOSING, - } state = STATE_OTHER; + _cleanup_(ansi_seq_parser_done) AnsiSeqParser parser = {}; _cleanup_(memstream_done) MemStream m = {}; + const char *begin = NULL; size_t isz, shift[2] = {}, n_carriage_returns = 0; FILE *f; @@ -721,16 +737,14 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { /* This does three things: * * 1. Replaces TABs by 8 spaces - * 2. Strips ANSI color sequences (a subset of CSI), i.e. ESC '[' … 'm' sequences - * 3. Strips ANSI operating system sequences (OSC), i.e. ESC ']' … ST sequences - * 4. Strip trailing \r characters (since they would "move the cursor", but have no + * 2. Strips ANSI sequences (CSI, OSC and all other escape sequences) + * 3. Strips trailing \r characters (since they would "move the cursor", but have no * other effect). * - * Everything else will be left as it is. In particular other ANSI sequences are left as they are, as - * are any other special characters. Truncated ANSI sequences are left-as is too. This call is - * supposed to suppress the most basic formatting noise, but nothing else. - * - * Why care for OSC sequences? Well, to undo what terminal_urlify() and friends generate. */ + * Any other special characters are left as they are. Invalid or truncated ANSI sequences are + * dropped, too: when stripping ANSI sequences it's better to strip too much than too little. This + * call is supposed to suppress formatting noise (including what terminal_urlify() and friends + * generate), but nothing else. */ isz = _isz ? *_isz : strlen(*ibuf); @@ -740,28 +754,48 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { if (!f) return NULL; - for (const char *i = *ibuf; i < *ibuf + isz + 1; i++) { + for (const char *i = *ibuf; i < *ibuf + isz; i++) { + int e; - bool eot = i >= *ibuf + isz; - - switch (state) { - - case STATE_OTHER: - if (eot) + for (;;) { + e = ansi_seq_parser_feed(&parser, *i); + if (e < 0) + return NULL; + if (e != ANSI_SEQ_EVENT_ABORT) break; + /* Eat up bogus sequences. The aborted sequence is over (and dropped, up to but + * excluding the current character, which is fed again), hence forget about it, so + * that the next sequence flushes pending carriage returns again. */ + if (begin) { + retract_offsets(begin - *ibuf, i - *ibuf, highlight, shift); + begin = NULL; + } + } + + switch (e) { + + case ANSI_SEQ_EVENT_TEXT: if (*i == '\r') { n_carriage_returns++; - break; - } else if (*i == '\n') - /* Ignore carriage returns before new line */ + continue; + } + if (*i == '\n') { + /* Ignore carriage returns before new line. They are dropped from the output, + * hence the offsets need to account for them. (Note that pending carriage + * returns always immediately precede the current character, since anything + * else flushes them.) */ + retract_offsets( + (size_t) (i - *ibuf) - n_carriage_returns, + i - *ibuf, + highlight, + shift); n_carriage_returns = 0; + } for (; n_carriage_returns > 0; n_carriage_returns--) fputc('\r', f); - if (*i == '\x1B') - state = STATE_ESCAPE; - else if (*i == '\t') { + if (*i == '\t') { fputs(" ", f); advance_offsets(i - *ibuf, highlight, shift, 7); } else @@ -769,75 +803,36 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { break; - case STATE_ESCAPE: - assert(n_carriage_returns == 0); + case ANSI_SEQ_EVENT_SEQUENCE: + if (!begin) { + /* This starts a new sequence, flush any pending carriage returns first */ + for (; n_carriage_returns > 0; n_carriage_returns--) + fputc('\r', f); - if (eot) { - fputc('\x1B', f); - advance_offsets(i - *ibuf, highlight, shift, 1); - break; - } else if (*i == '[') { /* ANSI CSI */ - state = STATE_CSI; - begin = i + 1; - } else if (*i == ']') { /* ANSI OSC */ - state = STATE_OSC; - begin = i + 1; - } else { - fputc('\x1B', f); - fputc(*i, f); - advance_offsets(i - *ibuf, highlight, shift, 1); - state = STATE_OTHER; + begin = i; } - break; - case STATE_CSI: - assert(n_carriage_returns == 0); - - if (eot || !strchr(DIGITS ";:m", *i)) { /* EOT or invalid chars in sequence */ - fputc('\x1B', f); - fputc('[', f); - advance_offsets(i - *ibuf, highlight, shift, 2); - state = STATE_OTHER; - i = begin-1; - } else if (*i == 'm') - state = STATE_OTHER; - + case ANSI_SEQ_EVENT_END: + /* A complete sequence, drop it (the current character is its last one) */ + assert(begin); + retract_offsets(begin - *ibuf, i + 1 - *ibuf, highlight, shift); + begin = NULL; break; - case STATE_OSC: - assert(n_carriage_returns == 0); - - /* There are three kinds of OSC terminators: \x07, \x1b\x5c or \x9c. We only support - * the first two, because the last one is a valid UTF-8 codepoint and hence creates - * an ambiguity (many Terminal emulators refuse to support it as well). */ - if (eot || (!IN_SET(*i, '\x07', '\x1b') && !osc_char_is_valid(*i))) { /* EOT or invalid chars in sequence */ - fputc('\x1B', f); - fputc(']', f); - advance_offsets(i - *ibuf, highlight, shift, 2); - state = STATE_OTHER; - i = begin-1; - } else if (*i == '\x07') /* Single character ST */ - state = STATE_OTHER; - else if (*i == '\x1B') - state = STATE_OSC_CLOSING; - - break; - - case STATE_OSC_CLOSING: - if (eot || *i != '\x5c') { /* EOT or incomplete two-byte ST in sequence */ - fputc('\x1B', f); - fputc(']', f); - advance_offsets(i - *ibuf, highlight, shift, 2); - state = STATE_OTHER; - i = begin-1; - } else if (*i == '\x5c') - state = STATE_OTHER; - - break; + default: + assert_not_reached(); } } + /* If the input ends in the middle of a sequence the truncated sequence is dropped as well, hence + * the offsets need to account for it, too. */ + if (begin) + retract_offsets(begin - *ibuf, isz, highlight, shift); + + /* Trailing carriage returns are dropped, too */ + retract_offsets(isz - n_carriage_returns, isz, highlight, shift); + char *obuf; if (memstream_finalize(&m, &obuf, _isz) < 0) return NULL; diff --git a/src/test/test-strip-tab-ansi.c b/src/test/test-strip-tab-ansi.c index 00bd7cdaf1..b151cba0e9 100644 --- a/src/test/test-strip-tab-ansi.c +++ b/src/test/test-strip-tab-ansi.c @@ -9,7 +9,7 @@ #include "tests.h" TEST(strip_tab_ansi) { - _cleanup_free_ char *urlified = NULL, *q = NULL, *qq = NULL; + _cleanup_free_ char *urlified = NULL, *q = NULL; char *p, *z; assert_se(p = strdup("\tFoobar\tbar\twaldo\t")); @@ -24,14 +24,48 @@ TEST(strip_tab_ansi) { ASSERT_STREQ(p, "Hello world!"); free(p); + /* Aborted sequences and complete ones are dropped (note that "\x1B[H" below is + * a complete CSI sequence, hence "Hello" loses its first character) */ assert_se(p = strdup("\x1B[\x1B[\t\x1B[" ANSI_HIGHLIGHT "\x1B[" "Hello" ANSI_NORMAL ANSI_HIGHLIGHT_RED " world!" ANSI_NORMAL)); assert_se(strip_tab_ansi(&p, NULL, NULL)); - ASSERT_STREQ(p, "\x1B[\x1B[ \x1B[\x1B[Hello world!"); + ASSERT_STREQ(p, " ello world!"); free(p); + /* "\x1B[w" is a complete CSI sequence, too */ assert_se(p = strdup("\x1B[waldo")); assert_se(strip_tab_ansi(&p, NULL, NULL)); - ASSERT_STREQ(p, "\x1B[waldo"); + ASSERT_STREQ(p, "aldo"); + free(p); + + /* Non-SGR CSI sequences are dropped as well */ + ASSERT_NOT_NULL(p = strdup("foo\x1B[2Jbar\x1B[?25lbaz")); + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "foobarbaz"); + free(p); + + /* Two-character Fe sequences (7-bit C1 controls, such as RI) are dropped as a whole: the final + * character must not leak through as text */ + ASSERT_NOT_NULL(p = strdup("a\x1BMb\x1B" "Ec\x1B\\d")); + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "abcd"); + free(p); + + /* A carriage return followed by a (dropped) sequence is not trailing, and hence kept … */ + ASSERT_NOT_NULL(p = strdup("y\r\x1B[m")); + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "y\r"); + free(p); + + /* … also if a (dropped) two-character Fe sequence was encountered earlier … */ + ASSERT_NOT_NULL(p = strdup("\x1BZy\r\x1B[m")); + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "y\r"); + free(p); + + /* … or an aborted sequence (the aborting character is replayed as text) */ + ASSERT_NOT_NULL(p = strdup("\x1B\ny\r\x1B[m")); + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "\ny\r"); free(p); assert_se(p = strdup("\r\rwaldo")); @@ -58,14 +92,19 @@ TEST(strip_tab_ansi) { ASSERT_STREQ(p, "something i am a fabulous link something-else"); p = mfree(p); - /* Truncate the formatted string in the middle of an ANSI sequence (in which case we shouldn't touch the - * incomplete sequence) */ + /* Truncate the formatted string in the middle of an ANSI sequence */ z = strstr(q, "fstab"); if (z) { *z = 0; - assert_se(qq = strdup(q)); - assert_se(strip_tab_ansi(&q, NULL, NULL)); - ASSERT_STREQ(q, qq); + _cleanup_free_ char *d = ASSERT_PTR(strdup(z+5)); + + ASSERT_NOT_NULL(strip_tab_ansi(&q, NULL, NULL)); + ASSERT_NOT_NULL(strip_tab_ansi(&d, NULL, NULL)); + + ASSERT_STREQ(q, "something "); + /* NB: d starts with the tail of the truncated ST, i.e. a stray "ESC \", which is a complete + * Fe sequence, and hence dropped in its entirety */ + ASSERT_STREQ(d, "i am a fabulous link something-else"); } /* Test that both kinds of ST are recognized after OSC */ @@ -78,4 +117,141 @@ TEST(strip_tab_ansi) { free(p); } +TEST(strip_tab_ansi_highlight) { + /* The highlight offsets (as used by "journalctl --grep" to mark the matched range) refer to the + * input, and must be moved along with it: TAB expansion pushes them forward, dropped sequences pull + * them back. */ + + const char *in = "\tfoo" ANSI_HIGHLIGHT "bar" ANSI_NORMAL "\tbaz" ANSI_HIGHLIGHT_RED "qux"; + const char *out = " foobar bazqux"; + size_t bar = strlen("\tfoo" ANSI_HIGHLIGHT), + qux = strlen("\tfoo" ANSI_HIGHLIGHT "bar" ANSI_NORMAL "\tbaz" ANSI_HIGHLIGHT_RED); + + /* A range following a TAB and a dropped sequence, ending right where another sequence starts */ + _cleanup_free_ char *p = ASSERT_PTR(strdup(in)); + size_t highlight[2] = { bar, bar + 3 }; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, out); + ASSERT_EQ(highlight[0], 11U); + ASSERT_EQ(highlight[1], 14U); + ASSERT_EQ(memcmp(p + highlight[0], "bar", 3), 0); + + /* A range past two TABs and three dropped sequences */ + p = mfree(p); + p = ASSERT_PTR(strdup(in)); + highlight[0] = qux; + highlight[1] = qux + 3; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, out); + ASSERT_EQ(highlight[0], 25U); + ASSERT_EQ(highlight[1], 28U); + ASSERT_EQ(memcmp(p + highlight[0], "qux", 3), 0); + + /* A range that starts inside a dropped sequence collapses onto its beginning */ + p = mfree(p); + p = ASSERT_PTR(strdup(in)); + highlight[0] = bar - 2; + highlight[1] = bar + 3; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, out); + ASSERT_EQ(highlight[0], 11U); + ASSERT_EQ(highlight[1], 14U); + + /* A range spanning TABs and sequences: both ends move independently */ + p = mfree(p); + p = ASSERT_PTR(strdup(in)); + highlight[0] = 1; /* "foo" */ + highlight[1] = qux + 3; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, out); + ASSERT_EQ(highlight[0], 8U); + ASSERT_EQ(highlight[1], 28U); + + /* An aborted (and hence dropped) sequence before a TAB: only the ESC is dropped, the aborting + * character is replayed as text */ + p = mfree(p); + p = ASSERT_PTR(strdup("\x1B\x01\tab")); + highlight[0] = 3; + highlight[1] = 5; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "\x01 ab"); + ASSERT_EQ(highlight[0], 9U); + ASSERT_EQ(highlight[1], 11U); + + /* A dropped two-character Fe sequence before a TAB */ + p = mfree(p); + p = ASSERT_PTR(strdup("\x1BZ\tab")); + highlight[0] = 3; + highlight[1] = 5; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, " ab"); + ASSERT_EQ(highlight[0], 8U); + ASSERT_EQ(highlight[1], 10U); + + /* A truncated sequence at the very end is dropped, too: a range reaching into it is clamped to the + * end of the output, rather than pointing past it */ + p = mfree(p); + p = ASSERT_PTR(strdup("ab\x1B[31")); + highlight[0] = 1; + highlight[1] = 5; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "ab"); + ASSERT_EQ(highlight[0], 1U); + ASSERT_EQ(highlight[1], 2U); + + /* Carriage returns dropped before a newline are accounted for, too: a range following them moves + * back */ + p = mfree(p); + p = ASSERT_PTR(strdup("ab\r\r\ncd")); + highlight[0] = 5; /* "cd" */ + highlight[1] = 7; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "ab\ncd"); + ASSERT_EQ(highlight[0], 3U); + ASSERT_EQ(highlight[1], 5U); + ASSERT_EQ(memcmp(p + highlight[0], "cd", 2), 0); + + /* A range spanning them shrinks accordingly */ + p = mfree(p); + p = ASSERT_PTR(strdup("ab\r\r\ncd")); + highlight[0] = 1; /* "b\r\r\nc" */ + highlight[1] = 6; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "ab\ncd"); + ASSERT_EQ(highlight[0], 1U); + ASSERT_EQ(highlight[1], 4U); + ASSERT_EQ(memcmp(p + highlight[0], "b\nc", 3), 0); + + /* Trailing carriage returns are dropped, too: a range reaching into them is clamped to the end of + * the output */ + p = mfree(p); + p = ASSERT_PTR(strdup("ab\r\r")); + highlight[0] = 1; + highlight[1] = 4; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "ab"); + ASSERT_EQ(highlight[0], 1U); + ASSERT_EQ(highlight[1], 2U); + + /* Carriage returns that are kept do not move anything */ + p = mfree(p); + p = ASSERT_PTR(strdup("a\r\rb")); + highlight[0] = 3; + highlight[1] = 4; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "a\r\rb"); + ASSERT_EQ(highlight[0], 3U); + ASSERT_EQ(highlight[1], 4U); + + /* Nothing to shift: a range before all of it stays put */ + p = mfree(p); + p = ASSERT_PTR(strdup("ab\t" ANSI_HIGHLIGHT "c")); + highlight[0] = 0; + highlight[1] = 2; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "ab c"); + ASSERT_EQ(highlight[0], 0U); + ASSERT_EQ(highlight[1], 2U); +} + DEFINE_TEST_MAIN(LOG_INFO);