diff --git a/src/basic/utf8.c b/src/basic/utf8.c index c4298cfe76..7cd0d74871 100644 --- a/src/basic/utf8.c +++ b/src/basic/utf8.c @@ -11,6 +11,7 @@ #include "gunicode.h" #include "hexdecoct.h" #include "string-util.h" +#include "unaligned.h" #include "utf8.h" bool unichar_is_valid(char32_t ch) { @@ -102,6 +103,20 @@ int utf8_encoded_to_unichar(const char *str, char32_t *ret_unichar) { return len; } +/* Tests whether all eight bytes of a word are plain printable ASCII, i.e. within 0x20…0x7e. The three + * terms below cover the three ways a byte can fail. A byte that fails is always flagged in its own + * lane, but the converse does not hold: the two subtractions are whole-word ones, so a borrow out of + * a failing lane also flags bytes above it. Only the aggregate test is meaningful, the mask is not a + * per-byte map of which bytes failed. */ +static bool is_printable_fastpath_u64(uint64_t w) { + uint64_t x = w ^ UINT64_C(0x7f7f7f7f7f7f7f7f); + + return (((w & UINT64_C(0x8080808080808080)) | /* >= 0x80 */ + ((w - UINT64_C(0x2020202020202020)) & ~w) | /* < 0x20 */ + ((x - UINT64_C(0x0101010101010101)) & ~x)) & /* == 0x7f */ + UINT64_C(0x8080808080808080)) == 0; +} + bool utf8_is_printable_newline(const char* str, size_t length, bool allow_newline) { assert(str); @@ -109,14 +124,41 @@ bool utf8_is_printable_newline(const char* str, size_t length, bool allow_newlin int encoded_len; char32_t val; - encoded_len = utf8_encoded_valid_unichar(p, length); + /* Bulk fast path: almost everything we are asked about is plain printable ASCII, so + * test eight bytes of it at once. Anything else falls through to the per-character + * path below, which handles it as before. A remainder of less than eight bytes is + * copied into a word padded with spaces, so that it can be tested the same way. */ + while (length > 0) { + uint64_t w; + size_t n; + + if (length < sizeof(uint64_t)) { + n = length; + w = UINT64_C(0x2020202020202020); + memcpy(&w, p, n); + } else { + n = sizeof(uint64_t); + w = unaligned_read_ne64(p); + } + + if (!is_printable_fastpath_u64(w)) + break; + + length -= n; + p += n; + } + + /* Note that this is not redundant with the loop condition above: the fast path may + * have consumed the remainder of the string, and then there's nothing left to decode. */ + if (length == 0) + break; + + encoded_len = utf8_encoded_valid_unichar_full(p, length, &val); if (encoded_len < 0) return false; assert(encoded_len > 0 && (size_t) encoded_len <= length); - if (utf8_encoded_to_unichar(p, &val) < 0 || - unichar_is_control(val) || - (!allow_newline && val == '\n')) + if (unichar_is_control(val) || (!allow_newline && val == '\n')) return false; length -= encoded_len; @@ -541,7 +583,7 @@ static int utf8_unichar_to_encoded_len(char32_t unichar) { } /* validate one encoded unicode char and return its length */ -int utf8_encoded_valid_unichar(const char *str, size_t length /* bytes */) { +int utf8_encoded_valid_unichar_full(const char *str, size_t length /* bytes */, char32_t *ret_unichar) { char32_t unichar; size_t len; int r; @@ -549,7 +591,9 @@ int utf8_encoded_valid_unichar(const char *str, size_t length /* bytes */) { assert(str); assert(length > 0); - /* We read until NUL, at most length bytes. SIZE_MAX may be used to disable the length check. */ + /* We read until NUL, at most length bytes. SIZE_MAX may be used to disable the length check. + * If ret_unichar is non-NULL the decoded character is returned in it, so that callers + * interested in both validity and value needn't decode the character a second time. */ len = utf8_encoded_expected_len(str[0]); if (len == 0) @@ -560,8 +604,11 @@ int utf8_encoded_valid_unichar(const char *str, size_t length /* bytes */) { return -EINVAL; /* ascii is valid */ - if (len == 1) + if (len == 1) { + if (ret_unichar) + *ret_unichar = (char32_t) (uint8_t) str[0]; return 1; + } /* check if expected encoded chars are available */ for (size_t i = 0; i < len; i++) @@ -580,6 +627,9 @@ int utf8_encoded_valid_unichar(const char *str, size_t length /* bytes */) { if (!unichar_is_valid(unichar)) return -EINVAL; + if (ret_unichar) + *ret_unichar = unichar; + return (int) len; } diff --git a/src/basic/utf8.h b/src/basic/utf8.h index fde72a0d54..19dca605f7 100644 --- a/src/basic/utf8.h +++ b/src/basic/utf8.h @@ -38,7 +38,10 @@ char16_t *utf8_to_utf16(const char *s, size_t length); size_t char16_strlen(const char16_t *s) _pure_; /* returns the number of 16-bit words in the string (not bytes!) */ size_t char16_strsize(const char16_t *s) _pure_; -int utf8_encoded_valid_unichar(const char *str, size_t length) _pure_; +int utf8_encoded_valid_unichar_full(const char *str, size_t length, char32_t *ret_unichar); +static inline int _pure_ utf8_encoded_valid_unichar(const char *str, size_t length) { + return utf8_encoded_valid_unichar_full(str, length, NULL); +} int utf8_encoded_to_unichar(const char *str, char32_t *ret_unichar); static inline bool utf16_is_surrogate(char16_t c) { diff --git a/src/test/test-utf8.c b/src/test/test-utf8.c index 875700fa7b..c6071d2cdd 100644 --- a/src/test/test-utf8.c +++ b/src/test/test-utf8.c @@ -7,6 +7,13 @@ #include "utf8.h" TEST(utf8_is_printable) { + /* Control bytes, plus the exact values the fast path's masks turn on: the last byte below + * 0x20 and the first at it, the last byte at or below 0x7e and the first above it, and the + * first byte with the topmost bit set. */ + static const char interesting[] = { + '\r', '\177', '\001', '\033', '\t', '\n', '\037', '\040', '\176', '\200', + }; + assert_se(utf8_is_printable("ascii is valid\tunicode", 22)); assert_se(utf8_is_printable("\342\204\242", 3)); assert_se(!utf8_is_printable("\341\204", 2)); @@ -14,6 +21,34 @@ TEST(utf8_is_printable) { assert_se(!utf8_is_printable("\r", 1)); assert_se(utf8_is_printable("\n", 1)); assert_se(utf8_is_printable("\t", 1)); + ASSERT_TRUE(utf8_is_printable("", 0)); + + /* DEL and the C1 range */ + ASSERT_FALSE(utf8_is_printable("\177", 1)); + ASSERT_FALSE(utf8_is_printable("\302\233", 2)); + + /* Lengths around the eight byte word the fast path works on: an exact multiple of it, one + * short of it, and a multi-byte character straddling its end. */ + ASSERT_TRUE(utf8_is_printable("abcdefgh", 8)); + ASSERT_TRUE(utf8_is_printable("abcdefg", 7)); + ASSERT_TRUE(utf8_is_printable("aaaaaaa\342\204\242aaaaa", 15)); + ASSERT_FALSE(utf8_is_printable("aaaaaaa\342\204", 9)); + + /* Put each of the interesting bytes at each offset of a longer string, so that a fast path + * that mishandles a single byte position doesn't go unnoticed. */ + char buf[16]; + FOREACH_ELEMENT(c, interesting) + for (size_t i = 0; i < sizeof(buf); i++) { + memset(buf, 'a', sizeof(buf)); + buf[i] = *c; + + /* Note that 0x80 on its own is not a valid encoding either, so it is + * expected to be rejected just like the control bytes. */ + bool printable = (uint8_t) *c >= 0x20 && (uint8_t) *c <= 0x7e; + + ASSERT_EQ(utf8_is_printable(buf, sizeof(buf)), printable || IN_SET(*c, '\t', '\n')); + ASSERT_EQ(utf8_is_printable_newline(buf, sizeof(buf), false), printable || *c == '\t'); + } } TEST(utf8_n_is_valid) {