diff --git a/src/basic/string-util.c b/src/basic/string-util.c index efd690b076..77ba8b05a0 100644 --- a/src/basic/string-util.c +++ b/src/basic/string-util.c @@ -3,6 +3,7 @@ #include #include "alloc-util.h" +#include "ansi-seq.h" #include "escape.h" #include "extract-word.h" #include "glyph-util.h" @@ -14,7 +15,6 @@ #include "path-util.h" #include "string-util.h" #include "strv.h" -#include "terminal-util.h" #include "utf8.h" char* first_word(const char *s, const char *word) { @@ -701,16 +701,32 @@ static void advance_offsets( shift[1] += size; } +static void retract_offsets( + size_t begin, + size_t end, + size_t offsets[2], /* note: we can't use [static 2] here, since this may be NULL */ + size_t shift[static 2]) { + + /* The input range [begin, end) was dropped from the output: offsets past it move back by its + * length, offsets inside it collapse onto its beginning. The shift is tracked in unsigned modular + * arithmetic, which is fine as the sum of all shifts applied to an offset is never negative: at most + * as many bytes can be dropped before an offset as there are bytes before it. */ + + if (!offsets) + return; + + assert(shift); + assert(begin <= end); + + for (size_t k = 0; k < 2; k++) + if (offsets[k] > begin) + shift[k] -= MIN(offsets[k], end) - begin; +} + char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { - const char *begin = NULL; - enum { - STATE_OTHER, - STATE_ESCAPE, - STATE_CSI, - STATE_OSC, - STATE_OSC_CLOSING, - } state = STATE_OTHER; + _cleanup_(ansi_seq_parser_done) AnsiSeqParser parser = {}; _cleanup_(memstream_done) MemStream m = {}; + const char *begin = NULL; size_t isz, shift[2] = {}, n_carriage_returns = 0; FILE *f; @@ -721,16 +737,14 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { /* This does three things: * * 1. Replaces TABs by 8 spaces - * 2. Strips ANSI color sequences (a subset of CSI), i.e. ESC '[' … 'm' sequences - * 3. Strips ANSI operating system sequences (OSC), i.e. ESC ']' … ST sequences - * 4. Strip trailing \r characters (since they would "move the cursor", but have no + * 2. Strips ANSI sequences (CSI, OSC and all other escape sequences) + * 3. Strips trailing \r characters (since they would "move the cursor", but have no * other effect). * - * Everything else will be left as it is. In particular other ANSI sequences are left as they are, as - * are any other special characters. Truncated ANSI sequences are left-as is too. This call is - * supposed to suppress the most basic formatting noise, but nothing else. - * - * Why care for OSC sequences? Well, to undo what terminal_urlify() and friends generate. */ + * Any other special characters are left as they are. Invalid or truncated ANSI sequences are + * dropped, too: when stripping ANSI sequences it's better to strip too much than too little. This + * call is supposed to suppress formatting noise (including what terminal_urlify() and friends + * generate), but nothing else. */ isz = _isz ? *_isz : strlen(*ibuf); @@ -740,28 +754,48 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { if (!f) return NULL; - for (const char *i = *ibuf; i < *ibuf + isz + 1; i++) { + for (const char *i = *ibuf; i < *ibuf + isz; i++) { + int e; - bool eot = i >= *ibuf + isz; - - switch (state) { - - case STATE_OTHER: - if (eot) + for (;;) { + e = ansi_seq_parser_feed(&parser, *i); + if (e < 0) + return NULL; + if (e != ANSI_SEQ_EVENT_ABORT) break; + /* Eat up bogus sequences. The aborted sequence is over (and dropped, up to but + * excluding the current character, which is fed again), hence forget about it, so + * that the next sequence flushes pending carriage returns again. */ + if (begin) { + retract_offsets(begin - *ibuf, i - *ibuf, highlight, shift); + begin = NULL; + } + } + + switch (e) { + + case ANSI_SEQ_EVENT_TEXT: if (*i == '\r') { n_carriage_returns++; - break; - } else if (*i == '\n') - /* Ignore carriage returns before new line */ + continue; + } + if (*i == '\n') { + /* Ignore carriage returns before new line. They are dropped from the output, + * hence the offsets need to account for them. (Note that pending carriage + * returns always immediately precede the current character, since anything + * else flushes them.) */ + retract_offsets( + (size_t) (i - *ibuf) - n_carriage_returns, + i - *ibuf, + highlight, + shift); n_carriage_returns = 0; + } for (; n_carriage_returns > 0; n_carriage_returns--) fputc('\r', f); - if (*i == '\x1B') - state = STATE_ESCAPE; - else if (*i == '\t') { + if (*i == '\t') { fputs(" ", f); advance_offsets(i - *ibuf, highlight, shift, 7); } else @@ -769,75 +803,36 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { break; - case STATE_ESCAPE: - assert(n_carriage_returns == 0); + case ANSI_SEQ_EVENT_SEQUENCE: + if (!begin) { + /* This starts a new sequence, flush any pending carriage returns first */ + for (; n_carriage_returns > 0; n_carriage_returns--) + fputc('\r', f); - if (eot) { - fputc('\x1B', f); - advance_offsets(i - *ibuf, highlight, shift, 1); - break; - } else if (*i == '[') { /* ANSI CSI */ - state = STATE_CSI; - begin = i + 1; - } else if (*i == ']') { /* ANSI OSC */ - state = STATE_OSC; - begin = i + 1; - } else { - fputc('\x1B', f); - fputc(*i, f); - advance_offsets(i - *ibuf, highlight, shift, 1); - state = STATE_OTHER; + begin = i; } - break; - case STATE_CSI: - assert(n_carriage_returns == 0); - - if (eot || !strchr(DIGITS ";:m", *i)) { /* EOT or invalid chars in sequence */ - fputc('\x1B', f); - fputc('[', f); - advance_offsets(i - *ibuf, highlight, shift, 2); - state = STATE_OTHER; - i = begin-1; - } else if (*i == 'm') - state = STATE_OTHER; - + case ANSI_SEQ_EVENT_END: + /* A complete sequence, drop it (the current character is its last one) */ + assert(begin); + retract_offsets(begin - *ibuf, i + 1 - *ibuf, highlight, shift); + begin = NULL; break; - case STATE_OSC: - assert(n_carriage_returns == 0); - - /* There are three kinds of OSC terminators: \x07, \x1b\x5c or \x9c. We only support - * the first two, because the last one is a valid UTF-8 codepoint and hence creates - * an ambiguity (many Terminal emulators refuse to support it as well). */ - if (eot || (!IN_SET(*i, '\x07', '\x1b') && !osc_char_is_valid(*i))) { /* EOT or invalid chars in sequence */ - fputc('\x1B', f); - fputc(']', f); - advance_offsets(i - *ibuf, highlight, shift, 2); - state = STATE_OTHER; - i = begin-1; - } else if (*i == '\x07') /* Single character ST */ - state = STATE_OTHER; - else if (*i == '\x1B') - state = STATE_OSC_CLOSING; - - break; - - case STATE_OSC_CLOSING: - if (eot || *i != '\x5c') { /* EOT or incomplete two-byte ST in sequence */ - fputc('\x1B', f); - fputc(']', f); - advance_offsets(i - *ibuf, highlight, shift, 2); - state = STATE_OTHER; - i = begin-1; - } else if (*i == '\x5c') - state = STATE_OTHER; - - break; + default: + assert_not_reached(); } } + /* If the input ends in the middle of a sequence the truncated sequence is dropped as well, hence + * the offsets need to account for it, too. */ + if (begin) + retract_offsets(begin - *ibuf, isz, highlight, shift); + + /* Trailing carriage returns are dropped, too */ + retract_offsets(isz - n_carriage_returns, isz, highlight, shift); + char *obuf; if (memstream_finalize(&m, &obuf, _isz) < 0) return NULL; diff --git a/src/test/test-strip-tab-ansi.c b/src/test/test-strip-tab-ansi.c index 00bd7cdaf1..b151cba0e9 100644 --- a/src/test/test-strip-tab-ansi.c +++ b/src/test/test-strip-tab-ansi.c @@ -9,7 +9,7 @@ #include "tests.h" TEST(strip_tab_ansi) { - _cleanup_free_ char *urlified = NULL, *q = NULL, *qq = NULL; + _cleanup_free_ char *urlified = NULL, *q = NULL; char *p, *z; assert_se(p = strdup("\tFoobar\tbar\twaldo\t")); @@ -24,14 +24,48 @@ TEST(strip_tab_ansi) { ASSERT_STREQ(p, "Hello world!"); free(p); + /* Aborted sequences and complete ones are dropped (note that "\x1B[H" below is + * a complete CSI sequence, hence "Hello" loses its first character) */ assert_se(p = strdup("\x1B[\x1B[\t\x1B[" ANSI_HIGHLIGHT "\x1B[" "Hello" ANSI_NORMAL ANSI_HIGHLIGHT_RED " world!" ANSI_NORMAL)); assert_se(strip_tab_ansi(&p, NULL, NULL)); - ASSERT_STREQ(p, "\x1B[\x1B[ \x1B[\x1B[Hello world!"); + ASSERT_STREQ(p, " ello world!"); free(p); + /* "\x1B[w" is a complete CSI sequence, too */ assert_se(p = strdup("\x1B[waldo")); assert_se(strip_tab_ansi(&p, NULL, NULL)); - ASSERT_STREQ(p, "\x1B[waldo"); + ASSERT_STREQ(p, "aldo"); + free(p); + + /* Non-SGR CSI sequences are dropped as well */ + ASSERT_NOT_NULL(p = strdup("foo\x1B[2Jbar\x1B[?25lbaz")); + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "foobarbaz"); + free(p); + + /* Two-character Fe sequences (7-bit C1 controls, such as RI) are dropped as a whole: the final + * character must not leak through as text */ + ASSERT_NOT_NULL(p = strdup("a\x1BMb\x1B" "Ec\x1B\\d")); + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "abcd"); + free(p); + + /* A carriage return followed by a (dropped) sequence is not trailing, and hence kept … */ + ASSERT_NOT_NULL(p = strdup("y\r\x1B[m")); + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "y\r"); + free(p); + + /* … also if a (dropped) two-character Fe sequence was encountered earlier … */ + ASSERT_NOT_NULL(p = strdup("\x1BZy\r\x1B[m")); + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "y\r"); + free(p); + + /* … or an aborted sequence (the aborting character is replayed as text) */ + ASSERT_NOT_NULL(p = strdup("\x1B\ny\r\x1B[m")); + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "\ny\r"); free(p); assert_se(p = strdup("\r\rwaldo")); @@ -58,14 +92,19 @@ TEST(strip_tab_ansi) { ASSERT_STREQ(p, "something i am a fabulous link something-else"); p = mfree(p); - /* Truncate the formatted string in the middle of an ANSI sequence (in which case we shouldn't touch the - * incomplete sequence) */ + /* Truncate the formatted string in the middle of an ANSI sequence */ z = strstr(q, "fstab"); if (z) { *z = 0; - assert_se(qq = strdup(q)); - assert_se(strip_tab_ansi(&q, NULL, NULL)); - ASSERT_STREQ(q, qq); + _cleanup_free_ char *d = ASSERT_PTR(strdup(z+5)); + + ASSERT_NOT_NULL(strip_tab_ansi(&q, NULL, NULL)); + ASSERT_NOT_NULL(strip_tab_ansi(&d, NULL, NULL)); + + ASSERT_STREQ(q, "something "); + /* NB: d starts with the tail of the truncated ST, i.e. a stray "ESC \", which is a complete + * Fe sequence, and hence dropped in its entirety */ + ASSERT_STREQ(d, "i am a fabulous link something-else"); } /* Test that both kinds of ST are recognized after OSC */ @@ -78,4 +117,141 @@ TEST(strip_tab_ansi) { free(p); } +TEST(strip_tab_ansi_highlight) { + /* The highlight offsets (as used by "journalctl --grep" to mark the matched range) refer to the + * input, and must be moved along with it: TAB expansion pushes them forward, dropped sequences pull + * them back. */ + + const char *in = "\tfoo" ANSI_HIGHLIGHT "bar" ANSI_NORMAL "\tbaz" ANSI_HIGHLIGHT_RED "qux"; + const char *out = " foobar bazqux"; + size_t bar = strlen("\tfoo" ANSI_HIGHLIGHT), + qux = strlen("\tfoo" ANSI_HIGHLIGHT "bar" ANSI_NORMAL "\tbaz" ANSI_HIGHLIGHT_RED); + + /* A range following a TAB and a dropped sequence, ending right where another sequence starts */ + _cleanup_free_ char *p = ASSERT_PTR(strdup(in)); + size_t highlight[2] = { bar, bar + 3 }; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, out); + ASSERT_EQ(highlight[0], 11U); + ASSERT_EQ(highlight[1], 14U); + ASSERT_EQ(memcmp(p + highlight[0], "bar", 3), 0); + + /* A range past two TABs and three dropped sequences */ + p = mfree(p); + p = ASSERT_PTR(strdup(in)); + highlight[0] = qux; + highlight[1] = qux + 3; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, out); + ASSERT_EQ(highlight[0], 25U); + ASSERT_EQ(highlight[1], 28U); + ASSERT_EQ(memcmp(p + highlight[0], "qux", 3), 0); + + /* A range that starts inside a dropped sequence collapses onto its beginning */ + p = mfree(p); + p = ASSERT_PTR(strdup(in)); + highlight[0] = bar - 2; + highlight[1] = bar + 3; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, out); + ASSERT_EQ(highlight[0], 11U); + ASSERT_EQ(highlight[1], 14U); + + /* A range spanning TABs and sequences: both ends move independently */ + p = mfree(p); + p = ASSERT_PTR(strdup(in)); + highlight[0] = 1; /* "foo" */ + highlight[1] = qux + 3; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, out); + ASSERT_EQ(highlight[0], 8U); + ASSERT_EQ(highlight[1], 28U); + + /* An aborted (and hence dropped) sequence before a TAB: only the ESC is dropped, the aborting + * character is replayed as text */ + p = mfree(p); + p = ASSERT_PTR(strdup("\x1B\x01\tab")); + highlight[0] = 3; + highlight[1] = 5; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "\x01 ab"); + ASSERT_EQ(highlight[0], 9U); + ASSERT_EQ(highlight[1], 11U); + + /* A dropped two-character Fe sequence before a TAB */ + p = mfree(p); + p = ASSERT_PTR(strdup("\x1BZ\tab")); + highlight[0] = 3; + highlight[1] = 5; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, " ab"); + ASSERT_EQ(highlight[0], 8U); + ASSERT_EQ(highlight[1], 10U); + + /* A truncated sequence at the very end is dropped, too: a range reaching into it is clamped to the + * end of the output, rather than pointing past it */ + p = mfree(p); + p = ASSERT_PTR(strdup("ab\x1B[31")); + highlight[0] = 1; + highlight[1] = 5; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "ab"); + ASSERT_EQ(highlight[0], 1U); + ASSERT_EQ(highlight[1], 2U); + + /* Carriage returns dropped before a newline are accounted for, too: a range following them moves + * back */ + p = mfree(p); + p = ASSERT_PTR(strdup("ab\r\r\ncd")); + highlight[0] = 5; /* "cd" */ + highlight[1] = 7; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "ab\ncd"); + ASSERT_EQ(highlight[0], 3U); + ASSERT_EQ(highlight[1], 5U); + ASSERT_EQ(memcmp(p + highlight[0], "cd", 2), 0); + + /* A range spanning them shrinks accordingly */ + p = mfree(p); + p = ASSERT_PTR(strdup("ab\r\r\ncd")); + highlight[0] = 1; /* "b\r\r\nc" */ + highlight[1] = 6; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "ab\ncd"); + ASSERT_EQ(highlight[0], 1U); + ASSERT_EQ(highlight[1], 4U); + ASSERT_EQ(memcmp(p + highlight[0], "b\nc", 3), 0); + + /* Trailing carriage returns are dropped, too: a range reaching into them is clamped to the end of + * the output */ + p = mfree(p); + p = ASSERT_PTR(strdup("ab\r\r")); + highlight[0] = 1; + highlight[1] = 4; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "ab"); + ASSERT_EQ(highlight[0], 1U); + ASSERT_EQ(highlight[1], 2U); + + /* Carriage returns that are kept do not move anything */ + p = mfree(p); + p = ASSERT_PTR(strdup("a\r\rb")); + highlight[0] = 3; + highlight[1] = 4; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "a\r\rb"); + ASSERT_EQ(highlight[0], 3U); + ASSERT_EQ(highlight[1], 4U); + + /* Nothing to shift: a range before all of it stays put */ + p = mfree(p); + p = ASSERT_PTR(strdup("ab\t" ANSI_HIGHLIGHT "c")); + highlight[0] = 0; + highlight[1] = 2; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "ab c"); + ASSERT_EQ(highlight[0], 0U); + ASSERT_EQ(highlight[1], 2U); +} + DEFINE_TEST_MAIN(LOG_INFO);