From e65b0904a082e0be64f267527741285dbabcb31b Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 31 Oct 2024 09:51:32 +0100 Subject: [PATCH 1/5] string-util: it's called OSC sequence, not CSO sequence --- src/basic/string-util.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/basic/string-util.c b/src/basic/string-util.c index 1715d62a39..9526443e50 100644 --- a/src/basic/string-util.c +++ b/src/basic/string-util.c @@ -698,7 +698,7 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { STATE_OTHER, STATE_ESCAPE, STATE_CSI, - STATE_CSO, + STATE_OSC, } state = STATE_OTHER; _cleanup_(memstream_done) MemStream m = {}; size_t isz, shift[2] = {}, n_carriage_returns = 0; @@ -711,7 +711,7 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { * * 1. Replaces TABs by 8 spaces * 2. Strips ANSI color sequences (a subset of CSI), i.e. ESC '[' … 'm' sequences - * 3. Strips ANSI operating system sequences (CSO), i.e. ESC ']' … BEL sequences + * 3. Strips ANSI operating system sequences (OSC), i.e. ESC ']' … BEL sequences * 4. Strip trailing \r characters (since they would "move the cursor", but have no * other effect). * @@ -719,7 +719,7 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { * are any other special characters. Truncated ANSI sequences are left-as is too. This call is * supposed to suppress the most basic formatting noise, but nothing else. * - * Why care for CSO sequences? Well, to undo what terminal_urlify() and friends generate. */ + * Why care for OSC sequences? Well, to undo what terminal_urlify() and friends generate. */ isz = _isz ? *_isz : strlen(*ibuf); @@ -766,8 +766,8 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { } else if (*i == '[') { /* ANSI CSI */ state = STATE_CSI; begin = i + 1; - } else if (*i == ']') { /* ANSI CSO */ - state = STATE_CSO; + } else if (*i == ']') { /* ANSI OSC */ + state = STATE_OSC; begin = i + 1; } else { fputc('\x1B', f); @@ -793,7 +793,7 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { break; - case STATE_CSO: + case STATE_OSC: assert(n_carriage_returns == 0); if (i >= *ibuf + isz || /* EOT … */ From b8311af8101577aa4ad431d827191cb7526b3435 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 31 Oct 2024 10:05:13 +0100 Subject: [PATCH 2/5] tree-wide: prefer generating 0x1B 0x5C as ANSI sequence "ST" OSC sequences can be closed with one of three terminators: 1. ASCII code 7, aka BEL, aka ^G, aka \x07, aka \a 2. ASCII code 156, aka \x9c 2. Pair of ASCII code 27 followed by ASCII code 92, aka \x1b\x5c Of these, in some corner case scenarios BEL makes problem (see #34604). Hence switch away from that wherever we use it, and prefer the \x1b\x5c instead. That's preferable over \x9c, since the latter is also a valid UTF-8 codepoint. See discussion here for example: https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda#the-escape-sequence Fixes: #34604 --- src/basic/terminal-util.c | 4 ++-- src/basic/terminal-util.h | 7 +++++++ src/shared/pretty-print.c | 6 +++--- src/shared/ptyfwd.c | 4 ++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/basic/terminal-util.c b/src/basic/terminal-util.c index 8ded46f493..4172db12a5 100644 --- a/src/basic/terminal-util.c +++ b/src/basic/terminal-util.c @@ -1923,12 +1923,12 @@ int get_default_background_color(double *ret_red, double *ret_green, double *ret if (tcsetattr(STDIN_FILENO, TCSADRAIN, &new_termios) < 0) return -errno; - r = loop_write(STDOUT_FILENO, "\x1B]11;?\x07", SIZE_MAX); + r = loop_write(STDOUT_FILENO, "\x1B]11;?" ANSI_ST, SIZE_MAX); if (r < 0) goto finish; usec_t end = usec_add(now(CLOCK_MONOTONIC), 333 * USEC_PER_MSEC); - char buf[STRLEN("\x1B]11;rgb:0/0/0\x07")]; /* shortest possible reply */ + char buf[STRLEN("\x1B]11;rgb:0/0/0" ANSI_ST)]; /* shortest possible reply */ size_t buf_full = 0; BackgroundColorContext context = {}; diff --git a/src/basic/terminal-util.h b/src/basic/terminal-util.h index b446e547d6..3d02e92d0b 100644 --- a/src/basic/terminal-util.h +++ b/src/basic/terminal-util.h @@ -27,6 +27,13 @@ #define ANSI_WINDOW_TITLE_PUSH "\x1b[22;2t" #define ANSI_WINDOW_TITLE_POP "\x1b[23;2t" +/* ANSI "string terminator" character ("ST"). Terminal emulators typically allow three different ones: 0x07, + * 0x9c, and 0x1B 0x5C. We'll avoid 0x07 (BEL, aka ^G) since it might trigger unexpected TTY signal + * handling. And we'll avoid 0x9c since that's also valid regular codepoint in UTF-8 and elsewhere, and + * creates ambiguities. Because of that some terminal emulators explicitly choose not to support it. Hence we + * use 0x1B 0x5c */ +#define ANSI_ST "\e\\" + bool isatty_safe(int fd); int terminal_reset_defensive(int fd, bool switch_to_text); diff --git a/src/shared/pretty-print.c b/src/shared/pretty-print.c index df1c20fa24..1e9819ab09 100644 --- a/src/shared/pretty-print.c +++ b/src/shared/pretty-print.c @@ -88,7 +88,7 @@ int terminal_urlify(const char *url, const char *text, char **ret) { text = url; if (urlify_enabled()) - n = strjoin("\x1B]8;;", url, "\a", text, "\x1B]8;;\a"); + n = strjoin("\x1B]8;;", url, ANSI_ST, text, "\x1B]8;;" ANSI_ST); else n = strdup(text); if (!n) @@ -475,7 +475,7 @@ void draw_progress_bar_unbuffered(const char *prefix, double percentage) { * https://conemu.github.io/en/AnsiEscapeCodes.html#ConEmu_specific_OSC * https://github.com/microsoft/terminal/pull/8055 */ - fprintf(stderr, "\x1B]9;4;1;%u\a", (unsigned) ceil(percentage)); + fprintf(stderr, "\x1B]9;4;1;%u" ANSI_ST, (unsigned) ceil(percentage)); size_t cols = columns(); size_t prefix_width = utf8_console_width(prefix) + 1 /* space */; @@ -534,7 +534,7 @@ void clear_progress_bar_unbuffered(const char *prefix) { stderr); else /* Undo Windows Terminal progress indication again. */ - fputs("\x1B]9;4;0;;\a" + fputs("\x1B]9;4;0;;" ANSI_ST ANSI_ERASE_TO_END_OF_LINE, stderr); fputc('\r', stderr); diff --git a/src/shared/ptyfwd.c b/src/shared/ptyfwd.c index c06c279d87..11701f6768 100644 --- a/src/shared/ptyfwd.c +++ b/src/shared/ptyfwd.c @@ -397,7 +397,7 @@ static int insert_window_title_fix(PTYForward *f, size_t offset) { if (!t) return 0; - _cleanup_free_ char *joined = strjoin("\x1b]0;", f->title_prefix, t, "\a"); + _cleanup_free_ char *joined = strjoin("\x1b]0;", f->title_prefix, t, ANSI_ST); if (!joined) return -ENOMEM; @@ -567,7 +567,7 @@ static int do_shovel(PTYForward *f) { if (f->title) { if (!strextend(&f->out_buffer, ANSI_WINDOW_TITLE_PUSH - "\x1b]2;", f->title, "\a")) + "\x1b]2;", f->title, ANSI_ST)) return log_oom(); } From de0ebee637636400a73b0d966f18a8826796f87d Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 31 Oct 2024 10:10:37 +0100 Subject: [PATCH 3/5] ptyfwd: document why we only honour two of the three kinds of ST --- src/shared/ptyfwd.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/shared/ptyfwd.c b/src/shared/ptyfwd.c index 11701f6768..34b24837ef 100644 --- a/src/shared/ptyfwd.c +++ b/src/shared/ptyfwd.c @@ -507,11 +507,16 @@ static int pty_forward_ansi_process(PTYForward *f, size_t offset) { } else { /* Otherwise, the OSC sequence is over * - * There are two allowed ways to end an OSC sequence: - * BEL '\x07' - * String Terminator (ST): \ - "\x1b\x5c" - * since we cannot lookahead to see if the Esc is followed by a \ - * we cut a corner here and assume it will be \. */ + * There are three documented ways to end an OSC sequence: + * 1. BEL aka ^G aka \x07 + * 2. \x9c + * 3. \x1b\x5c + * since we cannot look ahead to see if the Esc is followed by a "\" + * we cut a corner here and assume it will be "\"e. + * + * Note that we do not support \x9c here, because that's also a valid UTF8 + * codepoint, and that would create ambiguity. Various terminal emulators + * similar do not support it. */ if (IN_SET(c, '\x07', '\x1b')) { r = insert_window_title_fix(f, i+1); From 0367424786e1ceac953e6336d5592c376d47d73a Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 31 Oct 2024 10:37:02 +0100 Subject: [PATCH 4/5] terminal-util: define ANSI_OSC as macro for the OSC terminal sequence prefix --- src/basic/terminal-util.c | 4 ++-- src/basic/terminal-util.h | 3 +++ src/shared/pretty-print.c | 8 +++++--- src/shared/ptyfwd.c | 4 ++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/basic/terminal-util.c b/src/basic/terminal-util.c index 4172db12a5..81c02d85a3 100644 --- a/src/basic/terminal-util.c +++ b/src/basic/terminal-util.c @@ -1923,12 +1923,12 @@ int get_default_background_color(double *ret_red, double *ret_green, double *ret if (tcsetattr(STDIN_FILENO, TCSADRAIN, &new_termios) < 0) return -errno; - r = loop_write(STDOUT_FILENO, "\x1B]11;?" ANSI_ST, SIZE_MAX); + r = loop_write(STDOUT_FILENO, ANSI_OSC "11;?" ANSI_ST, SIZE_MAX); if (r < 0) goto finish; usec_t end = usec_add(now(CLOCK_MONOTONIC), 333 * USEC_PER_MSEC); - char buf[STRLEN("\x1B]11;rgb:0/0/0" ANSI_ST)]; /* shortest possible reply */ + char buf[STRLEN(ANSI_OSC "11;rgb:0/0/0" ANSI_ST)]; /* shortest possible reply */ size_t buf_full = 0; BackgroundColorContext context = {}; diff --git a/src/basic/terminal-util.h b/src/basic/terminal-util.h index 3d02e92d0b..08d8cbac25 100644 --- a/src/basic/terminal-util.h +++ b/src/basic/terminal-util.h @@ -34,6 +34,9 @@ * use 0x1B 0x5c */ #define ANSI_ST "\e\\" +/* The "operating system command" ("OSC") start sequence */ +#define ANSI_OSC "\e]" + bool isatty_safe(int fd); int terminal_reset_defensive(int fd, bool switch_to_text); diff --git a/src/shared/pretty-print.c b/src/shared/pretty-print.c index 1e9819ab09..4e38e60775 100644 --- a/src/shared/pretty-print.c +++ b/src/shared/pretty-print.c @@ -88,7 +88,9 @@ int terminal_urlify(const char *url, const char *text, char **ret) { text = url; if (urlify_enabled()) - n = strjoin("\x1B]8;;", url, ANSI_ST, text, "\x1B]8;;" ANSI_ST); + n = strjoin(ANSI_OSC "8;;", url, ANSI_ST, + text, + ANSI_OSC "8;;" ANSI_ST); else n = strdup(text); if (!n) @@ -475,7 +477,7 @@ void draw_progress_bar_unbuffered(const char *prefix, double percentage) { * https://conemu.github.io/en/AnsiEscapeCodes.html#ConEmu_specific_OSC * https://github.com/microsoft/terminal/pull/8055 */ - fprintf(stderr, "\x1B]9;4;1;%u" ANSI_ST, (unsigned) ceil(percentage)); + fprintf(stderr, ANSI_OSC "9;4;1;%u" ANSI_ST, (unsigned) ceil(percentage)); size_t cols = columns(); size_t prefix_width = utf8_console_width(prefix) + 1 /* space */; @@ -534,7 +536,7 @@ void clear_progress_bar_unbuffered(const char *prefix) { stderr); else /* Undo Windows Terminal progress indication again. */ - fputs("\x1B]9;4;0;;" ANSI_ST + fputs(ANSI_OSC "9;4;0;;" ANSI_ST ANSI_ERASE_TO_END_OF_LINE, stderr); fputc('\r', stderr); diff --git a/src/shared/ptyfwd.c b/src/shared/ptyfwd.c index 34b24837ef..c1e3b7180b 100644 --- a/src/shared/ptyfwd.c +++ b/src/shared/ptyfwd.c @@ -397,7 +397,7 @@ static int insert_window_title_fix(PTYForward *f, size_t offset) { if (!t) return 0; - _cleanup_free_ char *joined = strjoin("\x1b]0;", f->title_prefix, t, ANSI_ST); + _cleanup_free_ char *joined = strjoin(ANSI_OSC "0;", f->title_prefix, t, ANSI_ST); if (!joined) return -ENOMEM; @@ -572,7 +572,7 @@ static int do_shovel(PTYForward *f) { if (f->title) { if (!strextend(&f->out_buffer, ANSI_WINDOW_TITLE_PUSH - "\x1b]2;", f->title, ANSI_ST)) + ANSI_OSC "2;", f->title, ANSI_ST)) return log_oom(); } From a39c51799b27531183e7393ea9c669f6148f257a Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Thu, 31 Oct 2024 11:22:39 +0100 Subject: [PATCH 5/5] string-util: also check for 0x1b 0x5c ST when stripping ANSI from strings --- src/basic/string-util.c | 25 ++++++++++++++++++++++--- src/test/test-strip-tab-ansi.c | 9 +++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/basic/string-util.c b/src/basic/string-util.c index 9526443e50..171a368059 100644 --- a/src/basic/string-util.c +++ b/src/basic/string-util.c @@ -699,6 +699,7 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { STATE_ESCAPE, STATE_CSI, STATE_OSC, + STATE_OSC_CLOSING, } state = STATE_OTHER; _cleanup_(memstream_done) MemStream m = {}; size_t isz, shift[2] = {}, n_carriage_returns = 0; @@ -711,7 +712,7 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { * * 1. Replaces TABs by 8 spaces * 2. Strips ANSI color sequences (a subset of CSI), i.e. ESC '[' … 'm' sequences - * 3. Strips ANSI operating system sequences (OSC), i.e. ESC ']' … BEL sequences + * 3. Strips ANSI operating system sequences (OSC), i.e. ESC ']' … ST sequences * 4. Strip trailing \r characters (since they would "move the cursor", but have no * other effect). * @@ -796,14 +797,32 @@ char* strip_tab_ansi(char **ibuf, size_t *_isz, size_t highlight[2]) { case STATE_OSC: assert(n_carriage_returns == 0); + /* There are three kinds of OSC terminators: \x07, \x1b\x5c or \x9c. We only support + * the first two, because the last one is a valid UTF-8 codepoint and hence creates + * an ambiguity (many Terminal emulators refuse to support it as well). */ if (i >= *ibuf + isz || /* EOT … */ - (*i != '\a' && (uint8_t) *i < 32U) || (uint8_t) *i > 126U) { /* … or invalid chars in sequence */ + (!IN_SET(*i, '\x07', '\x1b') && (uint8_t) *i < 32U) || (uint8_t) *i > 126U) { /* … or invalid chars in sequence */ fputc('\x1B', f); fputc(']', f); advance_offsets(i - *ibuf, highlight, shift, 2); state = STATE_OTHER; i = begin-1; - } else if (*i == '\a') + } else if (*i == '\x07') /* Single character ST */ + state = STATE_OTHER; + else if (*i == '\x1B') + state = STATE_OSC_CLOSING; + + break; + + case STATE_OSC_CLOSING: + if (i >= *ibuf + isz || /* EOT … */ + *i != '\x5c') { /* … or incomplete two-byte ST in sequence */ + fputc('\x1B', f); + fputc(']', f); + advance_offsets(i - *ibuf, highlight, shift, 2); + state = STATE_OTHER; + i = begin-1; + } else if (*i == '\x5c') state = STATE_OTHER; break; diff --git a/src/test/test-strip-tab-ansi.c b/src/test/test-strip-tab-ansi.c index 3ad0fcd90e..00bd7cdaf1 100644 --- a/src/test/test-strip-tab-ansi.c +++ b/src/test/test-strip-tab-ansi.c @@ -67,6 +67,15 @@ TEST(strip_tab_ansi) { assert_se(strip_tab_ansi(&q, NULL, NULL)); ASSERT_STREQ(q, qq); } + + /* Test that both kinds of ST are recognized after OSC */ + assert_se(p = strdup("before" ANSI_OSC "inside1" ANSI_ST + "between1" ANSI_OSC "inside2\a" + "between2" ANSI_OSC "inside3\x1b\x5c" + "after")); + assert_se(strip_tab_ansi(&p, NULL, NULL)); + ASSERT_STREQ(p, "beforebetween1between2after"); + free(p); } DEFINE_TEST_MAIN(LOG_INFO);