From b8da2dc98e0e86cf496327cb3397fddfc3148150 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 27 Aug 2026 11:08:38 +0200 Subject: [PATCH 1/2] basic: add simple ansi sequence parser This tries to generalize ANSI sequence handling a bit, and will track and capture ANSI sequences in a more comprehensive fashion. --- src/basic/ansi-seq.c | 222 +++++++++++++++++++++++++++++++ src/basic/ansi-seq.h | 56 ++++++++ src/basic/meson.build | 1 + src/test/meson.build | 1 + src/test/test-ansi-seq.c | 276 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 556 insertions(+) create mode 100644 src/basic/ansi-seq.c create mode 100644 src/basic/ansi-seq.h create mode 100644 src/test/test-ansi-seq.c diff --git a/src/basic/ansi-seq.c b/src/basic/ansi-seq.c new file mode 100644 index 0000000000..4fc64ad062 --- /dev/null +++ b/src/basic/ansi-seq.c @@ -0,0 +1,222 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include "alloc-util.h" +#include "ansi-seq.h" +#include "string-util.h" + +void ansi_seq_parser_done(AnsiSeqParser *p) { + assert(p); + + p->string = mfree(p->string); + p->string_len = 0; +} + +static void ansi_seq_parser_begin_string(AnsiSeqParser *p, char introducer) { + assert(p); + + p->string_len = 0; + if (p->string) + p->string[0] = 0; + p->bad = false; + p->introducer = introducer; +} + +static int ansi_seq_parser_push_string(AnsiSeqParser *p, char c) { + assert(p); + assert(!p->bad); /* only ever set together with a reset to the ground state, hence never seen here */ + + /* Safety check: don't capture unbounded string sequences */ + if (p->string_len >= ANSI_SEQ_STRING_MAX) + return 0; /* too long */ + + if (p->capture) { + if (!GREEDY_REALLOC(p->string, p->string_len + 2)) + return -ENOMEM; + + p->string[p->string_len+0] = c; + p->string[p->string_len+1] = 0; + } + + p->string_len++; + + return 1; /* added */ +} + +static int ansi_seq_parser_bad(AnsiSeqParser *p) { + assert(p); + + /* A bad sequence has been received, reset state to ground, and remember this */ + + p->state = ANSI_SEQ_STATE_GROUND; + p->bad = true; + return ANSI_SEQ_EVENT_ABORT; +} + +int ansi_seq_parser_feed(AnsiSeqParser *p, char c) { + uint8_t u = (uint8_t) c; + int r; + + assert(p); + + switch (p->state) { + + case ANSI_SEQ_STATE_GROUND: + if (c == 0x1B) { /* ESC */ + p->state = ANSI_SEQ_STATE_ESC; + ansi_seq_parser_begin_string(p, /* introducer= */ 0); + return ANSI_SEQ_EVENT_SEQUENCE; + } + + return ANSI_SEQ_EVENT_TEXT; + + case ANSI_SEQ_STATE_ESC: + switch (c) { + + case '[': /* CSI sequence */ + p->state = ANSI_SEQ_STATE_CSI; + ansi_seq_parser_begin_string(p, c); + return ANSI_SEQ_EVENT_SEQUENCE; + + case ']': /* OSC */ + case 'P': /* DCS */ + case 'X': /* SOS */ + case '^': /* PM */ + case '_': /* APC */ + p->state = ANSI_SEQ_STATE_STRING; + ansi_seq_parser_begin_string(p, c); + return ANSI_SEQ_EVENT_SEQUENCE; + + case 0x40 ... 0x4f: /* Fe sequence, i.e. the 7-bit form of the C1 controls (e.g. ESC M = RI), + * except for those introducing CSI/string sequences, handled above */ + case 0x51 ... 0x57: + case 0x59 ... 0x5a: + case 0x5c: + case 0x60 ... 0x7e: /* Fs sequence */ + case 0x30 ... 0x3f: /* Fp sequence */ + p->state = ANSI_SEQ_STATE_GROUND; + p->introducer = c; + return ANSI_SEQ_EVENT_END; + + case 0x20 ... 0x2f: /* nF sequence */ + p->state = ANSI_SEQ_STATE_NF; + ansi_seq_parser_begin_string(p, c); + return ANSI_SEQ_EVENT_SEQUENCE; + + default: + /* Anything else is not a valid escape sequence, give up on it */ + return ansi_seq_parser_bad(p); + } + + case ANSI_SEQ_STATE_CSI: + if (u >= 0x40 && u <= 0x7E) { /* final byte, the sequence is complete */ + r = ansi_seq_parser_push_string(p, c); + if (r < 0) + return r; + if (r == 0) + return ansi_seq_parser_bad(p); + + p->state = ANSI_SEQ_STATE_GROUND; + return ANSI_SEQ_EVENT_END; + } + + if (u >= 0x20 && u <= 0x3F) { + /* parameter or intermediate byte, the sequence continues */ + r = ansi_seq_parser_push_string(p, c); + if (r < 0) + return r; + if (r > 0) + return ANSI_SEQ_EVENT_SEQUENCE; + + /* fall through on too long */ + } + + /* Anything else is not a valid escape sequence, give up on it */ + return ansi_seq_parser_bad(p); + + case ANSI_SEQ_STATE_STRING: + if (c == 0x07) { /* BEL, the single character ST */ + p->state = ANSI_SEQ_STATE_GROUND; + return ANSI_SEQ_EVENT_END; + } + + if (c == 0x1B) { /* Presumably the beginning of the two character ST. Note that we do not + * support the C1 encoding of ST (0x9C), since that's a valid UTF-8 + * codepoint and would hence be ambiguous. */ + p->state = ANSI_SEQ_STATE_STRING_ESC; + return ANSI_SEQ_EVENT_SEQUENCE; + } + + if (u >= 0x20U) { /* regular payload */ + r = ansi_seq_parser_push_string(p, c); + if (r < 0) + return r; + if (r > 0) + return ANSI_SEQ_EVENT_SEQUENCE; + + /* fall through on too long */ + } + + /* Any other control character: give up on this sequence */ + return ansi_seq_parser_bad(p); + + case ANSI_SEQ_STATE_STRING_ESC: + if (c == '\\') { /* ST is complete, and so is the string sequence */ + p->state = ANSI_SEQ_STATE_GROUND; + return ANSI_SEQ_EVENT_END; + } + + return ansi_seq_parser_bad(p); + + case ANSI_SEQ_STATE_NF: + if (u >= 0x30 && u <= 0x7E) { /* final byte, the sequence is complete */ + r = ansi_seq_parser_push_string(p, c); + if (r < 0) + return r; + if (r == 0) + return ansi_seq_parser_bad(p); + + p->state = ANSI_SEQ_STATE_GROUND; + return ANSI_SEQ_EVENT_END; + } + + if (u >= 0x20 && u <= 0x2F) { + /* parameter or intermediate byte, the sequence continues */ + r = ansi_seq_parser_push_string(p, c); + if (r < 0) + return r; + if (r > 0) + return ANSI_SEQ_EVENT_SEQUENCE; + + /* fall through on too long */ + } + + /* Anything else is not a valid escape sequence, give up on it */ + return ansi_seq_parser_bad(p); + + default: + assert_not_reached(); + } +} + +int ansi_seq_parser_feed_harder(AnsiSeqParser *p, char c) { + int r; + + assert(p); + + /* Just like ansi_seq_parser_feed(), but automatically eat up abort events, and resubmit the bad character. */ + + do { + r = ansi_seq_parser_feed(p, c); + } while (r == ANSI_SEQ_EVENT_ABORT); + + return r; +} + +const char* ansi_seq_parser_string(const AnsiSeqParser *p) { + assert(p); + + if (!p->capture || p->bad || p->introducer == 0) + return NULL; + + return strempty(p->string); +} diff --git a/src/basic/ansi-seq.h b/src/basic/ansi-seq.h new file mode 100644 index 0000000000..bd406eecb6 --- /dev/null +++ b/src/basic/ansi-seq.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +#include "forward.h" + +/* A minimal, generic state engine that tracks ANSI escape sequence boundaries in a byte stream. It does not + * interpret sequences, it only determines where they begin and end, so that callers can find "safe" + * positions in a stream at which generated sequences may be inserted without corrupting a sequence already + * in progress. Optionally, the payload of sequences is captured, so that callers may act on specific + * sequences: for CSI and nF sequences that's the parameter, intermediate and final bytes, for "string" + * sequences (OSC and friends) it's the string itself. */ + +typedef enum AnsiSeqState { + ANSI_SEQ_STATE_GROUND, /* regular characters */ + ANSI_SEQ_STATE_ESC, /* ESC seen, or within an escape sequence that is neither CSI nor a string sequence */ + ANSI_SEQ_STATE_CSI, /* within a CSI sequence */ + ANSI_SEQ_STATE_STRING, /* within a string sequence (OSC, DCS, SOS, PM, APC) */ + ANSI_SEQ_STATE_STRING_ESC, /* within a string sequence, ESC seen (presumably the beginning of ST) */ + ANSI_SEQ_STATE_NF, /* within an NF sequence */ + _ANSI_SEQ_STATE_MAX, + _ANSI_SEQ_STATE_INVALID = -EINVAL, +} AnsiSeqState; + +typedef enum AnsiSeqEvent { + ANSI_SEQ_EVENT_TEXT, /* a regular character, not part of any sequence */ + ANSI_SEQ_EVENT_SEQUENCE, /* a character belonging to a sequence that is not complete yet */ + ANSI_SEQ_EVENT_END, /* the character completes a sequence, its payload may be queried */ + ANSI_SEQ_EVENT_ABORT, /* the character aborted a sequence and was not consumed: feed it again! */ + _ANSI_SEQ_EVENT_MAX, + _ANSI_SEQ_EVENT_INVALID = -EINVAL, +} AnsiSeqEvent; + +/* Maximum length of a sequence payload we are willing to process. A sequence that grows beyond this is + * considered invalid and aborted. */ +#define ANSI_SEQ_STRING_MAX 192U + +typedef struct AnsiSeqParser { + AnsiSeqState state; + char introducer; /* for string sequences: the character that introduced it, e.g. ']' for OSC */ + char *string; /* the accumulated payload of the most recent string sequence */ + size_t string_len; + bool bad; /* the current string sequence grew beyond ANSI_SEQ_STRING_MAX or is otherwise invalid */ + bool capture; /* if true we'll capture the sequence */ +} AnsiSeqParser; + +void ansi_seq_parser_done(AnsiSeqParser *p); + +/* Feeds a single character, returns an AnsiSeqEvent, or a negative errno on capture allocation failure */ +int ansi_seq_parser_feed(AnsiSeqParser *p, char c); + +/* Just like this, but eat up errors, and immediately submit character again */ +int ansi_seq_parser_feed_harder(AnsiSeqParser *p, char c); + +/* After ANSI_SEQ_EVENT_END: the captured payload as NUL terminated string, or NULL if capturing is + * off or the payload overflowed */ +const char* ansi_seq_parser_string(const AnsiSeqParser *p); diff --git a/src/basic/meson.build b/src/basic/meson.build index 6f59596b06..61d9765373 100644 --- a/src/basic/meson.build +++ b/src/basic/meson.build @@ -5,6 +5,7 @@ basic_sources = files( 'af-list.c', 'alloc-util.c', 'ansi-color.c', + 'ansi-seq.c', 'architecture.c', 'argv-util.c', 'arphrd-util.c', diff --git a/src/test/meson.build b/src/test/meson.build index 81ff491b50..2a305ccddc 100644 --- a/src/test/meson.build +++ b/src/test/meson.build @@ -57,6 +57,7 @@ generated_sources += [test_libsystemd_sym_c, test_libudev_sym_c] simple_tests += files( 'test-alloc-util.c', + 'test-ansi-seq.c', 'test-architecture.c', 'test-argv-util.c', 'test-arphrd-util.c', diff --git a/src/test/test-ansi-seq.c b/src/test/test-ansi-seq.c new file mode 100644 index 0000000000..d604a3bce7 --- /dev/null +++ b/src/test/test-ansi-seq.c @@ -0,0 +1,276 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ + +#include "alloc-util.h" +#include "ansi-seq.h" +#include "escape.h" +#include "tests.h" + +static void feed_string(AnsiSeqParser *p, const char *s, const char *expected_events) { + size_t e = 0; + int r; + + _cleanup_free_ char *escaped = ASSERT_PTR(cescape(s)); + + log_info("looking at: %s", escaped); + + /* NB: an aborted character is fed twice and hence produces two events, i.e. the expected event + * string may be longer than the input string */ + + for (size_t i = 0; s[i]; i++) + for (;;) { + r = ansi_seq_parser_feed(p, s[i]); + ASSERT_OK(r); + + AnsiSeqEvent expected; + + switch (expected_events[e++]) { + case 't': + expected = ANSI_SEQ_EVENT_TEXT; + break; + case 's': + expected = ANSI_SEQ_EVENT_SEQUENCE; + break; + case 'e': + expected = ANSI_SEQ_EVENT_END; + break; + case 'a': + expected = ANSI_SEQ_EVENT_ABORT; + break; + default: + assert_not_reached(); + } + + ASSERT_EQ(r, (int) expected); + + if (r != ANSI_SEQ_EVENT_ABORT) /* aborted characters must be fed again */ + break; + } + + ASSERT_EQ(expected_events[e], '\0'); +} + +TEST(ansi_seq_parser) { + _cleanup_(ansi_seq_parser_done) AnsiSeqParser p = { .capture = true }; + + /* Regular text */ + feed_string(&p, "hello\n\r", "ttttttt"); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + + /* A CSI color sequence */ + feed_string(&p, "\x1B[0;31m", "sssssse"); + ASSERT_EQ(p.introducer, '['); + ASSERT_STREQ(ansi_seq_parser_string(&p), "0;31m"); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + + /* A two-character escape sequence */ + feed_string(&p, "\x1B" "c", "se"); + ASSERT_EQ(p.introducer, 'c'); + ASSERT_STREQ(ansi_seq_parser_string(&p), ""); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + + /* A CSI sequence with DEC private parameters */ + feed_string(&p, "\x1B[?25l", "ssssse"); + ASSERT_EQ(p.introducer, '['); + ASSERT_STREQ(ansi_seq_parser_string(&p), "?25l"); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + + /* An nF escape sequence */ + feed_string(&p, "\x1B(B", "sse"); + ASSERT_EQ(p.introducer, '('); + ASSERT_STREQ(ansi_seq_parser_string(&p), "B"); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + + /* An nF escape sequence with a 0x2F intermediate byte */ + feed_string(&p, "\x1B#/0", "ssse"); + ASSERT_EQ(p.introducer, '#'); + ASSERT_STREQ(ansi_seq_parser_string(&p), "/0"); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + + /* An OSC sequence terminated by BEL */ + feed_string(&p, "\x1B]0;title\a", "ssssssssse"); + ASSERT_EQ(p.introducer, ']'); + ASSERT_STREQ(ansi_seq_parser_string(&p), "0;title"); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + + /* An OSC sequence terminated by ST, interleaved feeding */ + feed_string(&p, "\x1B]28", "ssss"); + feed_string(&p, "11;width=80;height=24\x1B\\", "sssssssssssssssssssssse"); + ASSERT_EQ(p.introducer, ']'); + ASSERT_STREQ(ansi_seq_parser_string(&p), "2811;width=80;height=24"); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + + /* A DCS sequence: tracked, payload captured with 'P' introducer */ + feed_string(&p, "\x1BPxyz\x1B\\", "sssssse"); + ASSERT_EQ(p.introducer, 'P'); + ASSERT_STREQ(ansi_seq_parser_string(&p), "xyz"); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + + /* An aborted OSC sequence: the \n is replayed as text */ + feed_string(&p, "\x1B]123\n", "sssssat"); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + ASSERT_NULL(ansi_seq_parser_string(&p)); + + /* An overlong string sequence */ + feed_string(&p, "\x1B]", "ss"); + for (size_t i = 0; i < ANSI_SEQ_STRING_MAX; i++) + ASSERT_EQ(ansi_seq_parser_feed(&p, 'x'), ANSI_SEQ_EVENT_SEQUENCE); + ASSERT_EQ(ansi_seq_parser_feed(&p, 'x'), ANSI_SEQ_EVENT_ABORT); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + ASSERT_NULL(ansi_seq_parser_string(&p)); + + /* Capturing works again afterwards */ + feed_string(&p, "\x1B]foo\a", "ssssse"); + ASSERT_EQ(p.introducer, ']'); + ASSERT_STREQ(ansi_seq_parser_string(&p), "foo"); + + /* The other string sequence introducers: SOS, PM, APC */ + feed_string(&p, "\x1BXsos\x1B\\", "sssssse"); + ASSERT_EQ(p.introducer, 'X'); + ASSERT_STREQ(ansi_seq_parser_string(&p), "sos"); + + feed_string(&p, "\x1B^pm\a", "sssse"); + ASSERT_EQ(p.introducer, '^'); + ASSERT_STREQ(ansi_seq_parser_string(&p), "pm"); + + feed_string(&p, "\x1B_apc\a", "ssssse"); + ASSERT_EQ(p.introducer, '_'); + ASSERT_STREQ(ansi_seq_parser_string(&p), "apc"); + + /* A two-character Fp escape sequence (DECSC) */ + feed_string(&p, "\x1B" "7", "se"); + ASSERT_EQ(p.introducer, '7'); + ASSERT_STREQ(ansi_seq_parser_string(&p), ""); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + + /* Two-character Fe escape sequences, i.e. 7-bit encoded C1 controls (RI, NEL, a stray ST) */ + feed_string(&p, "\x1BM", "se"); + ASSERT_EQ(p.introducer, 'M'); + ASSERT_STREQ(ansi_seq_parser_string(&p), ""); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + + feed_string(&p, "\x1B" "E", "se"); + ASSERT_EQ(p.introducer, 'E'); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + + feed_string(&p, "\x1B\\", "se"); + ASSERT_EQ(p.introducer, '\\'); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + + /* ESC followed by something that cannot start a sequence: aborted, the character is replayed */ + feed_string(&p, "\x1B\n", "sat"); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + ASSERT_NULL(ansi_seq_parser_string(&p)); + + /* An ESC inside a string sequence that is not followed by '\' (i.e. not ST): aborted, the + * character following the ESC is replayed */ + feed_string(&p, "\x1B]foo\x1B" "x", "ssssssat"); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + ASSERT_NULL(ansi_seq_parser_string(&p)); + + /* An overlong CSI sequence: aborted, even at its final byte */ + feed_string(&p, "\x1B[", "ss"); + for (size_t i = 0; i < ANSI_SEQ_STRING_MAX; i++) + ASSERT_EQ(ansi_seq_parser_feed(&p, ';'), ANSI_SEQ_EVENT_SEQUENCE); + ASSERT_EQ(ansi_seq_parser_feed(&p, 'm'), ANSI_SEQ_EVENT_ABORT); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + ASSERT_NULL(ansi_seq_parser_string(&p)); + ASSERT_EQ(ansi_seq_parser_feed(&p, 'm'), ANSI_SEQ_EVENT_TEXT); + + /* Ditto for an overlong nF sequence */ + feed_string(&p, "\x1B(", "ss"); + for (size_t i = 0; i < ANSI_SEQ_STRING_MAX; i++) + ASSERT_EQ(ansi_seq_parser_feed(&p, ' '), ANSI_SEQ_EVENT_SEQUENCE); + ASSERT_EQ(ansi_seq_parser_feed(&p, 'B'), ANSI_SEQ_EVENT_ABORT); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + ASSERT_NULL(ansi_seq_parser_string(&p)); + ASSERT_EQ(ansi_seq_parser_feed(&p, 'B'), ANSI_SEQ_EVENT_TEXT); + + /* And an overlong CSI sequence that continues past the limit with a non-final byte */ + feed_string(&p, "\x1B[", "ss"); + for (size_t i = 0; i < ANSI_SEQ_STRING_MAX; i++) + ASSERT_EQ(ansi_seq_parser_feed(&p, '1'), ANSI_SEQ_EVENT_SEQUENCE); + ASSERT_EQ(ansi_seq_parser_feed(&p, '1'), ANSI_SEQ_EVENT_ABORT); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + ASSERT_EQ(ansi_seq_parser_feed(&p, '1'), ANSI_SEQ_EVENT_TEXT); +} + +static void feed_string_harder(AnsiSeqParser *p, const char *s, const char *expected_events) { + size_t e = 0; + + _cleanup_free_ char *escaped = ASSERT_PTR(cescape(s)); + + log_info("looking at (harder): %s", escaped); + + /* Unlike feed_string() this never sees abort events, as ansi_seq_parser_feed_harder() swallows + * them, hence the expected event string has exactly one event per input character */ + + for (size_t i = 0; s[i]; i++) { + int r = ansi_seq_parser_feed_harder(p, s[i]); + ASSERT_OK(r); + ASSERT_NE(r, (int) ANSI_SEQ_EVENT_ABORT); + + switch (expected_events[e++]) { + case 't': + ASSERT_EQ(r, (int) ANSI_SEQ_EVENT_TEXT); + break; + case 's': + ASSERT_EQ(r, (int) ANSI_SEQ_EVENT_SEQUENCE); + break; + case 'e': + ASSERT_EQ(r, (int) ANSI_SEQ_EVENT_END); + break; + default: + assert_not_reached(); + } + } + + ASSERT_EQ(expected_events[e], '\0'); +} + +TEST(ansi_seq_parser_feed_harder) { + _cleanup_(ansi_seq_parser_done) AnsiSeqParser p = { .capture = true }; + + /* Complete sequences work as with ansi_seq_parser_feed() */ + feed_string_harder(&p, "a\x1B[1mb", "tssset"); + ASSERT_STREQ(ansi_seq_parser_string(&p), "1m"); + + /* An aborted OSC sequence: the aborting character is resubmitted internally and comes out as text */ + feed_string_harder(&p, "\x1B]123\nx", "ssssstt"); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + ASSERT_NULL(ansi_seq_parser_string(&p)); + + /* An aborted CSI sequence whose aborting character (ESC) starts a new sequence */ + feed_string_harder(&p, "\x1B[1\x1B[m", "ssssse"); + ASSERT_STREQ(ansi_seq_parser_string(&p), "m"); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + + /* ESC ESC: the first ESC is dropped, the second one starts a sequence */ + feed_string_harder(&p, "\x1B\x1B[m", "ssse"); + ASSERT_STREQ(ansi_seq_parser_string(&p), "m"); +} + +TEST(ansi_seq_parser_no_capture) { + _cleanup_(ansi_seq_parser_done) AnsiSeqParser p = {}; /* capture off */ + + /* Boundaries are still tracked, but no payload is ever made available */ + feed_string(&p, "\x1B]0;title\a", "ssssssssse"); + ASSERT_EQ(p.introducer, ']'); + ASSERT_NULL(ansi_seq_parser_string(&p)); + ASSERT_NULL(p.string); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + + feed_string(&p, "\x1B[0;31m", "sssssse"); + ASSERT_EQ(p.introducer, '['); + ASSERT_NULL(ansi_seq_parser_string(&p)); + ASSERT_NULL(p.string); + + /* The length limit applies regardless */ + feed_string(&p, "\x1B]", "ss"); + for (size_t i = 0; i < ANSI_SEQ_STRING_MAX; i++) + ASSERT_EQ(ansi_seq_parser_feed(&p, 'x'), ANSI_SEQ_EVENT_SEQUENCE); + ASSERT_EQ(ansi_seq_parser_feed(&p, 'x'), ANSI_SEQ_EVENT_ABORT); + ASSERT_EQ(p.state, ANSI_SEQ_STATE_GROUND); + ASSERT_NULL(p.string); +} + +DEFINE_TEST_MAIN(LOG_INFO); From 2779ecaca3d01a6391a429cf58cab02f77ac0262 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 27 Aug 2026 12:17:44 +0200 Subject: [PATCH 2/2] string-util: port strip_tab_ansi() to AnsiSeqParser So far strip_tab_ansi() was very superficial: it only focussed on OSC and CSI sequences, and left the other ANSI sequences in touch. Let's make it more powerful and port it to the more complete AnsiSeqParser logic, and strip *all* ANSI sequences. For the relevant usecases this seems like the better, safer choice, since it's good to strip too much rather than too little, when stripping ANSI sequences. --- src/basic/string-util.c | 175 +++++++++++++++--------------- src/test/test-strip-tab-ansi.c | 192 +++++++++++++++++++++++++++++++-- 2 files changed, 269 insertions(+), 98 deletions(-) diff --git a/src/basic/string-util.c b/src/basic/string-util.c index efd690b076..77ba8b05a0 100644 --- a/src/basic/string-util.c +++ b/src/basic/string-util.c @@ -3,6 +3,7 @@ #include #include "alloc-util.h" +#include "ansi-seq.h" #include "escape.h" #include "extract-word.h" #include "glyph-util.h" @@ -14,7 +15,6 @@ #include "path-util.h" #include "string-util.h" #include "strv.h" -#include "terminal-util.h" #include "utf8.h" char* first_word(const char *s, const char *word) { @@ -701,16 +701,32 @@ static void advance_offsets( shift[1] += size; } +static void retract_offsets( + size_t begin, + size_t end, + size_t offsets[2], /* note: we can't use [static 2] here, since this may be NULL */ + size_t shift[static 2]) { + + /* The input range [begin, end) was dropped from the output: offsets past it move back by its + * length, offsets inside it collapse onto its beginning. The shift is tracked in unsigned modular + * arithmetic, which is fine as the sum of all shifts applied to an offset is never negative: at most + * as many bytes can be dropped before an offset as there are bytes before it. */ + + if (!offsets) + return; + + assert(shift); + assert(begin <= end); + + for (size_t k = 0; k < 2; k++) + if (offsets[k] > begin) + shift[k] -= MIN(offsets[k], end) - begin; +} + char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { - const char *begin = NULL; - enum { - STATE_OTHER, - STATE_ESCAPE, - STATE_CSI, - STATE_OSC, - STATE_OSC_CLOSING, - } state = STATE_OTHER; + _cleanup_(ansi_seq_parser_done) AnsiSeqParser parser = {}; _cleanup_(memstream_done) MemStream m = {}; + const char *begin = NULL; size_t isz, shift[2] = {}, n_carriage_returns = 0; FILE *f; @@ -721,16 +737,14 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { /* This does three things: * * 1. Replaces TABs by 8 spaces - * 2. Strips ANSI color sequences (a subset of CSI), i.e. ESC '[' … 'm' sequences - * 3. Strips ANSI operating system sequences (OSC), i.e. ESC ']' … ST sequences - * 4. Strip trailing \r characters (since they would "move the cursor", but have no + * 2. Strips ANSI sequences (CSI, OSC and all other escape sequences) + * 3. Strips trailing \r characters (since they would "move the cursor", but have no * other effect). * - * Everything else will be left as it is. In particular other ANSI sequences are left as they are, as - * are any other special characters. Truncated ANSI sequences are left-as is too. This call is - * supposed to suppress the most basic formatting noise, but nothing else. - * - * Why care for OSC sequences? Well, to undo what terminal_urlify() and friends generate. */ + * Any other special characters are left as they are. Invalid or truncated ANSI sequences are + * dropped, too: when stripping ANSI sequences it's better to strip too much than too little. This + * call is supposed to suppress formatting noise (including what terminal_urlify() and friends + * generate), but nothing else. */ isz = _isz ? *_isz : strlen(*ibuf); @@ -740,28 +754,48 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { if (!f) return NULL; - for (const char *i = *ibuf; i < *ibuf + isz + 1; i++) { + for (const char *i = *ibuf; i < *ibuf + isz; i++) { + int e; - bool eot = i >= *ibuf + isz; - - switch (state) { - - case STATE_OTHER: - if (eot) + for (;;) { + e = ansi_seq_parser_feed(&parser, *i); + if (e < 0) + return NULL; + if (e != ANSI_SEQ_EVENT_ABORT) break; + /* Eat up bogus sequences. The aborted sequence is over (and dropped, up to but + * excluding the current character, which is fed again), hence forget about it, so + * that the next sequence flushes pending carriage returns again. */ + if (begin) { + retract_offsets(begin - *ibuf, i - *ibuf, highlight, shift); + begin = NULL; + } + } + + switch (e) { + + case ANSI_SEQ_EVENT_TEXT: if (*i == '\r') { n_carriage_returns++; - break; - } else if (*i == '\n') - /* Ignore carriage returns before new line */ + continue; + } + if (*i == '\n') { + /* Ignore carriage returns before new line. They are dropped from the output, + * hence the offsets need to account for them. (Note that pending carriage + * returns always immediately precede the current character, since anything + * else flushes them.) */ + retract_offsets( + (size_t) (i - *ibuf) - n_carriage_returns, + i - *ibuf, + highlight, + shift); n_carriage_returns = 0; + } for (; n_carriage_returns > 0; n_carriage_returns--) fputc('\r', f); - if (*i == '\x1B') - state = STATE_ESCAPE; - else if (*i == '\t') { + if (*i == '\t') { fputs(" ", f); advance_offsets(i - *ibuf, highlight, shift, 7); } else @@ -769,75 +803,36 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { break; - case STATE_ESCAPE: - assert(n_carriage_returns == 0); + case ANSI_SEQ_EVENT_SEQUENCE: + if (!begin) { + /* This starts a new sequence, flush any pending carriage returns first */ + for (; n_carriage_returns > 0; n_carriage_returns--) + fputc('\r', f); - if (eot) { - fputc('\x1B', f); - advance_offsets(i - *ibuf, highlight, shift, 1); - break; - } else if (*i == '[') { /* ANSI CSI */ - state = STATE_CSI; - begin = i + 1; - } else if (*i == ']') { /* ANSI OSC */ - state = STATE_OSC; - begin = i + 1; - } else { - fputc('\x1B', f); - fputc(*i, f); - advance_offsets(i - *ibuf, highlight, shift, 1); - state = STATE_OTHER; + begin = i; } - break; - case STATE_CSI: - assert(n_carriage_returns == 0); - - if (eot || !strchr(DIGITS ";:m", *i)) { /* EOT or invalid chars in sequence */ - fputc('\x1B', f); - fputc('[', f); - advance_offsets(i - *ibuf, highlight, shift, 2); - state = STATE_OTHER; - i = begin-1; - } else if (*i == 'm') - state = STATE_OTHER; - + case ANSI_SEQ_EVENT_END: + /* A complete sequence, drop it (the current character is its last one) */ + assert(begin); + retract_offsets(begin - *ibuf, i + 1 - *ibuf, highlight, shift); + begin = NULL; break; - case STATE_OSC: - assert(n_carriage_returns == 0); - - /* There are three kinds of OSC terminators: \x07, \x1b\x5c or \x9c. We only support - * the first two, because the last one is a valid UTF-8 codepoint and hence creates - * an ambiguity (many Terminal emulators refuse to support it as well). */ - if (eot || (!IN_SET(*i, '\x07', '\x1b') && !osc_char_is_valid(*i))) { /* EOT or invalid chars in sequence */ - fputc('\x1B', f); - fputc(']', f); - advance_offsets(i - *ibuf, highlight, shift, 2); - state = STATE_OTHER; - i = begin-1; - } else if (*i == '\x07') /* Single character ST */ - state = STATE_OTHER; - else if (*i == '\x1B') - state = STATE_OSC_CLOSING; - - break; - - case STATE_OSC_CLOSING: - if (eot || *i != '\x5c') { /* EOT or incomplete two-byte ST in sequence */ - fputc('\x1B', f); - fputc(']', f); - advance_offsets(i - *ibuf, highlight, shift, 2); - state = STATE_OTHER; - i = begin-1; - } else if (*i == '\x5c') - state = STATE_OTHER; - - break; + default: + assert_not_reached(); } } + /* If the input ends in the middle of a sequence the truncated sequence is dropped as well, hence + * the offsets need to account for it, too. */ + if (begin) + retract_offsets(begin - *ibuf, isz, highlight, shift); + + /* Trailing carriage returns are dropped, too */ + retract_offsets(isz - n_carriage_returns, isz, highlight, shift); + char *obuf; if (memstream_finalize(&m, &obuf, _isz) < 0) return NULL; diff --git a/src/test/test-strip-tab-ansi.c b/src/test/test-strip-tab-ansi.c index 00bd7cdaf1..b151cba0e9 100644 --- a/src/test/test-strip-tab-ansi.c +++ b/src/test/test-strip-tab-ansi.c @@ -9,7 +9,7 @@ #include "tests.h" TEST(strip_tab_ansi) { - _cleanup_free_ char *urlified = NULL, *q = NULL, *qq = NULL; + _cleanup_free_ char *urlified = NULL, *q = NULL; char *p, *z; assert_se(p = strdup("\tFoobar\tbar\twaldo\t")); @@ -24,14 +24,48 @@ TEST(strip_tab_ansi) { ASSERT_STREQ(p, "Hello world!"); free(p); + /* Aborted sequences and complete ones are dropped (note that "\x1B[H" below is + * a complete CSI sequence, hence "Hello" loses its first character) */ assert_se(p = strdup("\x1B[\x1B[\t\x1B[" ANSI_HIGHLIGHT "\x1B[" "Hello" ANSI_NORMAL ANSI_HIGHLIGHT_RED " world!" ANSI_NORMAL)); assert_se(strip_tab_ansi(&p, NULL, NULL)); - ASSERT_STREQ(p, "\x1B[\x1B[ \x1B[\x1B[Hello world!"); + ASSERT_STREQ(p, " ello world!"); free(p); + /* "\x1B[w" is a complete CSI sequence, too */ assert_se(p = strdup("\x1B[waldo")); assert_se(strip_tab_ansi(&p, NULL, NULL)); - ASSERT_STREQ(p, "\x1B[waldo"); + ASSERT_STREQ(p, "aldo"); + free(p); + + /* Non-SGR CSI sequences are dropped as well */ + ASSERT_NOT_NULL(p = strdup("foo\x1B[2Jbar\x1B[?25lbaz")); + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "foobarbaz"); + free(p); + + /* Two-character Fe sequences (7-bit C1 controls, such as RI) are dropped as a whole: the final + * character must not leak through as text */ + ASSERT_NOT_NULL(p = strdup("a\x1BMb\x1B" "Ec\x1B\\d")); + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "abcd"); + free(p); + + /* A carriage return followed by a (dropped) sequence is not trailing, and hence kept … */ + ASSERT_NOT_NULL(p = strdup("y\r\x1B[m")); + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "y\r"); + free(p); + + /* … also if a (dropped) two-character Fe sequence was encountered earlier … */ + ASSERT_NOT_NULL(p = strdup("\x1BZy\r\x1B[m")); + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "y\r"); + free(p); + + /* … or an aborted sequence (the aborting character is replayed as text) */ + ASSERT_NOT_NULL(p = strdup("\x1B\ny\r\x1B[m")); + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "\ny\r"); free(p); assert_se(p = strdup("\r\rwaldo")); @@ -58,14 +92,19 @@ TEST(strip_tab_ansi) { ASSERT_STREQ(p, "something i am a fabulous link something-else"); p = mfree(p); - /* Truncate the formatted string in the middle of an ANSI sequence (in which case we shouldn't touch the - * incomplete sequence) */ + /* Truncate the formatted string in the middle of an ANSI sequence */ z = strstr(q, "fstab"); if (z) { *z = 0; - assert_se(qq = strdup(q)); - assert_se(strip_tab_ansi(&q, NULL, NULL)); - ASSERT_STREQ(q, qq); + _cleanup_free_ char *d = ASSERT_PTR(strdup(z+5)); + + ASSERT_NOT_NULL(strip_tab_ansi(&q, NULL, NULL)); + ASSERT_NOT_NULL(strip_tab_ansi(&d, NULL, NULL)); + + ASSERT_STREQ(q, "something "); + /* NB: d starts with the tail of the truncated ST, i.e. a stray "ESC \", which is a complete + * Fe sequence, and hence dropped in its entirety */ + ASSERT_STREQ(d, "i am a fabulous link something-else"); } /* Test that both kinds of ST are recognized after OSC */ @@ -78,4 +117,141 @@ TEST(strip_tab_ansi) { free(p); } +TEST(strip_tab_ansi_highlight) { + /* The highlight offsets (as used by "journalctl --grep" to mark the matched range) refer to the + * input, and must be moved along with it: TAB expansion pushes them forward, dropped sequences pull + * them back. */ + + const char *in = "\tfoo" ANSI_HIGHLIGHT "bar" ANSI_NORMAL "\tbaz" ANSI_HIGHLIGHT_RED "qux"; + const char *out = " foobar bazqux"; + size_t bar = strlen("\tfoo" ANSI_HIGHLIGHT), + qux = strlen("\tfoo" ANSI_HIGHLIGHT "bar" ANSI_NORMAL "\tbaz" ANSI_HIGHLIGHT_RED); + + /* A range following a TAB and a dropped sequence, ending right where another sequence starts */ + _cleanup_free_ char *p = ASSERT_PTR(strdup(in)); + size_t highlight[2] = { bar, bar + 3 }; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, out); + ASSERT_EQ(highlight[0], 11U); + ASSERT_EQ(highlight[1], 14U); + ASSERT_EQ(memcmp(p + highlight[0], "bar", 3), 0); + + /* A range past two TABs and three dropped sequences */ + p = mfree(p); + p = ASSERT_PTR(strdup(in)); + highlight[0] = qux; + highlight[1] = qux + 3; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, out); + ASSERT_EQ(highlight[0], 25U); + ASSERT_EQ(highlight[1], 28U); + ASSERT_EQ(memcmp(p + highlight[0], "qux", 3), 0); + + /* A range that starts inside a dropped sequence collapses onto its beginning */ + p = mfree(p); + p = ASSERT_PTR(strdup(in)); + highlight[0] = bar - 2; + highlight[1] = bar + 3; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, out); + ASSERT_EQ(highlight[0], 11U); + ASSERT_EQ(highlight[1], 14U); + + /* A range spanning TABs and sequences: both ends move independently */ + p = mfree(p); + p = ASSERT_PTR(strdup(in)); + highlight[0] = 1; /* "foo" */ + highlight[1] = qux + 3; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, out); + ASSERT_EQ(highlight[0], 8U); + ASSERT_EQ(highlight[1], 28U); + + /* An aborted (and hence dropped) sequence before a TAB: only the ESC is dropped, the aborting + * character is replayed as text */ + p = mfree(p); + p = ASSERT_PTR(strdup("\x1B\x01\tab")); + highlight[0] = 3; + highlight[1] = 5; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "\x01 ab"); + ASSERT_EQ(highlight[0], 9U); + ASSERT_EQ(highlight[1], 11U); + + /* A dropped two-character Fe sequence before a TAB */ + p = mfree(p); + p = ASSERT_PTR(strdup("\x1BZ\tab")); + highlight[0] = 3; + highlight[1] = 5; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, " ab"); + ASSERT_EQ(highlight[0], 8U); + ASSERT_EQ(highlight[1], 10U); + + /* A truncated sequence at the very end is dropped, too: a range reaching into it is clamped to the + * end of the output, rather than pointing past it */ + p = mfree(p); + p = ASSERT_PTR(strdup("ab\x1B[31")); + highlight[0] = 1; + highlight[1] = 5; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "ab"); + ASSERT_EQ(highlight[0], 1U); + ASSERT_EQ(highlight[1], 2U); + + /* Carriage returns dropped before a newline are accounted for, too: a range following them moves + * back */ + p = mfree(p); + p = ASSERT_PTR(strdup("ab\r\r\ncd")); + highlight[0] = 5; /* "cd" */ + highlight[1] = 7; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "ab\ncd"); + ASSERT_EQ(highlight[0], 3U); + ASSERT_EQ(highlight[1], 5U); + ASSERT_EQ(memcmp(p + highlight[0], "cd", 2), 0); + + /* A range spanning them shrinks accordingly */ + p = mfree(p); + p = ASSERT_PTR(strdup("ab\r\r\ncd")); + highlight[0] = 1; /* "b\r\r\nc" */ + highlight[1] = 6; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "ab\ncd"); + ASSERT_EQ(highlight[0], 1U); + ASSERT_EQ(highlight[1], 4U); + ASSERT_EQ(memcmp(p + highlight[0], "b\nc", 3), 0); + + /* Trailing carriage returns are dropped, too: a range reaching into them is clamped to the end of + * the output */ + p = mfree(p); + p = ASSERT_PTR(strdup("ab\r\r")); + highlight[0] = 1; + highlight[1] = 4; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "ab"); + ASSERT_EQ(highlight[0], 1U); + ASSERT_EQ(highlight[1], 2U); + + /* Carriage returns that are kept do not move anything */ + p = mfree(p); + p = ASSERT_PTR(strdup("a\r\rb")); + highlight[0] = 3; + highlight[1] = 4; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "a\r\rb"); + ASSERT_EQ(highlight[0], 3U); + ASSERT_EQ(highlight[1], 4U); + + /* Nothing to shift: a range before all of it stays put */ + p = mfree(p); + p = ASSERT_PTR(strdup("ab\t" ANSI_HIGHLIGHT "c")); + highlight[0] = 0; + highlight[1] = 2; + ASSERT_NOT_NULL(strip_tab_ansi(&p, NULL, highlight)); + ASSERT_STREQ(p, "ab c"); + ASSERT_EQ(highlight[0], 0U); + ASSERT_EQ(highlight[1], 2U); +} + DEFINE_TEST_MAIN(LOG_INFO);