khash: try to detect broken AF_ALG support in centos kernels

Fixes: #8278

src/basic/khash.c

@@ -43,6 +43,66 @@ struct khash {
         bool digest_valid;
 };
 
+int khash_supported(void) {
+        static const union {
+                struct sockaddr sa;
+                struct sockaddr_alg alg;
+        } sa = {
+                .alg.salg_family = AF_ALG,
+                .alg.salg_type = "hash",
+                .alg.salg_name = "sha256", /* a very common algorithm */
+        };
+
+        static int cached = -1;
+
+        if (cached < 0) {
+                _cleanup_close_ int fd1 = -1, fd2 = -1;
+                uint8_t buf[LONGEST_DIGEST+1];
+
+                fd1 = socket(AF_ALG, SOCK_SEQPACKET|SOCK_CLOEXEC, 0);
+                if (fd1 < 0) {
+                        /* The kernel returns EAFNOSUPPORT if AF_ALG is not supported at all */
+                        if (IN_SET(errno, EAFNOSUPPORT, EOPNOTSUPP))
+                                return (cached = false);
+
+                        return -errno;
+                }
+
+                if (bind(fd1, &sa.sa, sizeof(sa)) < 0) {
+                        /* The kernel returns ENOENT if the selected algorithm is not supported at all. We use a check
+                         * for SHA256 as a proxy for whether the whole API is supported at all. After all it's one of
+                         * the most common hash functions, and if it isn't supported, that's ample indication that
+                         * something is really off. */
+
+                        if (IN_SET(errno, ENOENT, EOPNOTSUPP))
+                                return (cached = false);
+
+                        return -errno;
+                }
+
+                fd2 = accept4(fd1, NULL, 0, SOCK_CLOEXEC);
+                if (fd2 < 0) {
+                        if (errno == EOPNOTSUPP)
+                                return (cached = false);
+
+                        return -errno;
+                }
+
+                if (recv(fd2, buf, sizeof(buf), 0) < 0) {
+                        /* On some kernels we get ENOKEY for non-keyed hash functions (such as sha256), let's refuse
+                         * using the API in those cases, since the kernel is
+                         * broken. https://github.com/systemd/systemd/issues/8278 */
+
+                        if (IN_SET(errno, ENOKEY, EOPNOTSUPP))
+                                return (cached = false);
+                }
+
+                cached = true;
+        }
+
+        return cached;
+}
+
 int khash_new_with_key(khash **ret, const char *algorithm, const void *key, size_t key_size) {
         union {
                 struct sockaddr sa;
@@ -54,6 +114,7 @@ int khash_new_with_key(khash **ret, const char *algorithm, const void *key, size
 
         _cleanup_(khash_unrefp) khash *h = NULL;
         _cleanup_close_ int fd = -1;
+        int supported;
         ssize_t n;
 
         assert(ret);
@@ -67,6 +128,12 @@ int khash_new_with_key(khash **ret, const char *algorithm, const void *key, size
         if (strlen(algorithm) >= sizeof(sa.alg.salg_name))
                 return -EOPNOTSUPP;
 
+        supported = khash_supported();
+        if (supported < 0)
+                return supported;
+        if (supported == 0)
+                return -EOPNOTSUPP;
+
         fd = socket(AF_ALG, SOCK_SEQPACKET|SOCK_CLOEXEC, 0);
         if (fd < 0)
                 return -errno;

src/basic/khash.h

@@ -28,6 +28,8 @@
 
 typedef struct khash khash;
 
+int khash_supported(void);
+
 /* For plain hash functions. Hash functions commonly supported on today's kernels are: crc32c, crct10dif, crc32,
  * sha224, sha256, sha512, sha384, sha1, md5, md4, sha3-224, sha3-256, sha3-384, sha3-512, and more. */
 int khash_new(khash **ret, const char *algorithm);