resolved: resolve insecure answers with unsupported sig algorithms

sd-resolved does not support all the permissible DNSSEC signature
algorithms, and some are intentionally unsupported as a matter of
policy. Answers that can only be validated via unsupported algorithms
should be treated as if they were unsigned, per RFC4035§5.2.

Previously, sd-resolved tried to properly record insecure answers for
unsupported algortihms, but did not record this status for each of the
auxilliary DNSSEC transactions, so the primary transaction had no way to
know if there was a plausible DNSKEY with an unsupported signature
algorithm in the chain of trust.

This commit adds the insecure DNSKEYs that use unsupported algorithms to
the list of validated keys for each transaction, so that dependent
transactions can learn that a plausible chain of trust exists, even if
no authenticated one does, and report the insecure answer.

src/resolve/resolved-dns-dnssec.c

@@ -646,8 +646,10 @@ static int dnssec_rrset_verify_sig(
                 if (!ctx)
                         return -ENOMEM;
 
+                /* If the signature algorithm is supported by systemd-resolved but disabled by host policy,
+                 * also return -EOPNOTSUPP. */
                 if (EVP_DigestInit_ex(ctx, md_algorithm, NULL) <= 0)
-                        return -EIO;
+                        return -EOPNOTSUPP;
 
                 if (EVP_DigestUpdate(ctx, sig_data, sig_size) <= 0)
                         return -EIO;
@@ -912,9 +914,6 @@ int dnssec_verify_rrset_search(
                 DNS_ANSWER_FOREACH_FLAGS(dnskey, flags, validated_dnskeys) {
                         DnssecResult one_result;
 
-                        if ((flags & DNS_ANSWER_AUTHENTICATED) == 0)
-                                continue;
-
                         /* Is this a DNSKEY RR that matches they key of our RRSIG? */
                         r = dnssec_rrsig_match_dnskey(rrsig, dnskey, false);
                         if (r < 0)
@@ -922,6 +921,14 @@ int dnssec_verify_rrset_search(
                         if (r == 0)
                                 continue;
 
+                        if ((flags & DNS_ANSWER_AUTHENTICATED) == 0) {
+                                /* An unauthenticated DNSKEY in validated_dnskeys is a key we are not able to
+                                 * authenticate, but might still be valid. Record this as an unsupported
+                                 * algorithm so we can still at least report an insecure answer. */
+                                found_unsupported_algorithm = true;
+                                continue;
+                        }
+
                         /* Take the time here, if it isn't set yet, so
                          * that we do all validations with the same
                          * time. */
@@ -1201,7 +1208,7 @@ int dnssec_nsec3_hash(DnsResourceRecord *nsec3, const char *name, void *ret) {
                 return -ENOMEM;
 
         if (EVP_DigestInit_ex(ctx, algorithm, NULL) <= 0)
-                return -EIO;
+                return -EOPNOTSUPP;
 
         r = dns_name_to_wire_format(name, wire_format, sizeof(wire_format), true);
         if (r < 0)
@@ -1218,7 +1225,7 @@ int dnssec_nsec3_hash(DnsResourceRecord *nsec3, const char *name, void *ret) {
 
         for (unsigned k = 0; k < nsec3->nsec3.iterations; k++) {
                 if (EVP_DigestInit_ex(ctx, algorithm, NULL) <= 0)
-                        return -EIO;
+                        return -EOPNOTSUPP;
                 if (EVP_DigestUpdate(ctx, result, hash_size) <= 0)
                         return -EIO;
                 if (EVP_DigestUpdate(ctx, nsec3->nsec3.salt, nsec3->nsec3.salt_size) <= 0)

src/resolve/resolved-dns-transaction.c

@@ -3259,6 +3259,12 @@ static int dns_transaction_copy_validated(DnsTransaction *t) {
                 if (DNS_TRANSACTION_IS_LIVE(dt->state))
                         continue;
 
+                /* Some of the validated keys may not be authenticated, but are still useful to report
+                 * insecure answers when the domain is signed only by unsupported algorithms. */
+                r = dns_answer_extend(&t->validated_keys, dt->validated_keys);
+                if (r < 0)
+                        return r;
+
                 if (!FLAGS_SET(dt->answer_query_flags, SD_RESOLVED_AUTHENTICATED))
                         continue;
 
@@ -3478,6 +3484,23 @@ static int dnssec_validate_records(
 
                 /* https://datatracker.ietf.org/doc/html/rfc6840#section-5.2 */
                 if (result == DNSSEC_UNSUPPORTED_ALGORITHM) {
+                        if (rr->key->type == DNS_TYPE_DNSKEY) {
+                                /* This is a DNSKEY we cannot authenticate, but it might still be the best
+                                 * offer from the resolver. Add it to the validated keys in case it's the
+                                 * best we can find, but do not mark it as authenticated.
+                                 */
+
+                                r = dns_answer_copy_by_key(&t->validated_keys, t->answer, rr->key, 0, NULL);
+                                if (r < 0)
+                                        return r;
+
+                                /* Some of the DNSKEYs we just added might already have been revoked,
+                                 * remove them again in that case. */
+                                r = dns_transaction_invalidate_revoked_keys(t);
+                                if (r < 0)
+                                        return r;
+                        }
+
                         r = dns_answer_move_by_key(validated, &t->answer, rr->key, 0, NULL);
                         if (r < 0)
                                 return r;