cryptenroll,homectl: Introduce --fido2-credential-algorithm option

* Some authenticators(like Yubikey) support credential algorithm other than ES256
* Introduce a new option so users can make use of it

src/shared/libfido2-util.c

@@ -450,6 +450,20 @@ static int fido2_use_hmac_hash_specific_token(
         return 0;
 }
 
+/* COSE_ECDH_ES256 is not usable with fido_cred_set_type() thus it's not listed here. */
+static const char *fido2_algorithm_to_string(int alg) {
+        switch(alg) {
+                case COSE_ES256:
+                        return "es256";
+                case COSE_RS256:
+                        return "rs256";
+                case COSE_EDDSA:
+                        return "eddsa";
+                default:
+                        return NULL;
+        }
+}
+
 int fido2_use_hmac_hash(
                 const char *device,
                 const char *rp_id,
@@ -532,6 +546,7 @@ int fido2_generate_hmac_hash(
                 const char *user_icon,
                 const char *askpw_icon_name,
                 Fido2EnrollFlags lock_with,
+                int cred_alg,
                 void **ret_cid, size_t *ret_cid_size,
                 void **ret_salt, size_t *ret_salt_size,
                 void **ret_secret, size_t *ret_secret_size,
@@ -628,10 +643,10 @@ int fido2_generate_hmac_hash(
                 return log_error_errno(SYNTHETIC_ERRNO(EIO),
                                        "Failed to set FIDO2 credential relying party ID/name: %s", sym_fido_strerr(r));
 
-        r = sym_fido_cred_set_type(c, COSE_ES256);
+        r = sym_fido_cred_set_type(c, cred_alg);
         if (r != FIDO_OK)
                 return log_error_errno(SYNTHETIC_ERRNO(EIO),
-                                       "Failed to set FIDO2 credential type to ES256: %s", sym_fido_strerr(r));
+                                       "Failed to set FIDO2 credential type to %s: %s", fido2_algorithm_to_string(cred_alg), sym_fido_strerr(r));
 
         r = sym_fido_cred_set_user(
                         c,
@@ -721,6 +736,9 @@ int fido2_generate_hmac_hash(
         if (r == FIDO_ERR_ACTION_TIMEOUT)
                 return log_error_errno(SYNTHETIC_ERRNO(ENOSTR),
                                        "Token action timeout. (User didn't interact with token quickly enough.)");
+        if (r == FIDO_ERR_UNSUPPORTED_ALGORITHM)
+                return log_error_errno(SYNTHETIC_ERRNO(EOPNOTSUPP),
+                                        "Token doesn't support credential algorithm %s.", fido2_algorithm_to_string(cred_alg));
         if (r != FIDO_OK)
                 return log_error_errno(SYNTHETIC_ERRNO(EIO),
                                        "Failed to generate FIDO2 credential: %s", sym_fido_strerr(r));
@@ -1124,3 +1142,24 @@ finish:
                                "FIDO2 tokens not supported on this build.");
 #endif
 }
+
+#if HAVE_LIBFIDO2
+int parse_fido2_algorithm(const char *s, int *ret) {
+        int a;
+
+        assert(s);
+
+        if (streq(s, "es256"))
+                a = COSE_ES256;
+        else if (streq(s, "rs256"))
+                a = COSE_RS256;
+        else if (streq(s, "eddsa"))
+                a = COSE_EDDSA;
+        else
+                return -EINVAL;
+
+        if (ret)
+                *ret = a;
+        return 0;
+}
+#endif

src/shared/libfido2-util.h

@@ -109,12 +109,18 @@ int fido2_generate_hmac_hash(
                 const char *user_icon,
                 const char *askpw_icon_name,
                 Fido2EnrollFlags lock_with,
+                int cred_alg,
                 void **ret_cid, size_t *ret_cid_size,
                 void **ret_salt, size_t *ret_salt_size,
                 void **ret_secret, size_t *ret_secret_size,
                 char **ret_usedpin,
                 Fido2EnrollFlags *ret_locked_with);
 
+int parse_fido2_algorithm(const char *s, int *ret);
+#else
+static inline int parse_fido2_algorithm(const char *s, int *ret) {
+        return -EOPNOTSUPP;
+}
 #endif
 
 int fido2_list_devices(void);