[PATCH v2 2/2] kex: honor client preference for rsa-sha2-{256,512} host key algorithms

[Jon Simons <jon@xxxxxxxxxxxxx>]

Ensure to honor the client preference ordering when enabling one of
the RFC8332 RSA signature extensions (`rsa-sha2-{256,512}`).

Before this change, libssh unconditionally selects the `rsa-sha2-512`
algorithm for clients which may have offered "rsa-sha2-256,rsa-sha2-512".

The change can be observed before-and-after with the pkd tests:

    ./pkd_hello -t torture_pkd_openssh_rsa_rsa_sha2_256_512

Signed-off-by: Jon Simons <jon@xxxxxxxxxxxxx>
---
 src/kex.c | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/src/kex.c b/src/kex.c
index 59522d22..2308a222 100644
--- a/src/kex.c
+++ b/src/kex.c
@@ -444,6 +444,7 @@ SSH_PACKET_CALLBACK(ssh_packet_kexinit)
     int server_kex = session->server;
     ssh_string str = NULL;
     char *strings[KEX_METHODS_SIZE] = {0};
+    char *rsa_sig_ext = NULL;
     int rc = SSH_ERROR;
 
     uint8_t first_kex_packet_follows = 0;
@@ -569,6 +570,29 @@ SSH_PACKET_CALLBACK(ssh_packet_kexinit)
             if (ok) {
                 session->extensions |= SSH_EXT_SIG_RSA_SHA256;
             }
+
+            /*
+             * Ensure that the client preference is honored for the case
+             * both signature types are enabled.
+             */
+            if ((session->extensions & SSH_EXT_SIG_RSA_SHA256) &&
+                (session->extensions & SSH_EXT_SIG_RSA_SHA512)) {
+                session->extensions &= ~(SSH_EXT_SIG_RSA_SHA256 | SSH_EXT_SIG_RSA_SHA512);
+                rsa_sig_ext = ssh_find_matching("rsa-sha2-512,rsa-sha2-256",
+                                                session->next_crypto->client_kex.methods[SSH_HOSTKEYS]);
+                if (rsa_sig_ext == NULL) {
+                    goto error; /* should never happen */
+                } else if (strcmp(rsa_sig_ext, "rsa-sha2-512") == 0) {
+                    session->extensions |= SSH_EXT_SIG_RSA_SHA512;
+                } else if (strcmp(rsa_sig_ext, "rsa-sha2-256") == 0) {
+                    session->extensions |= SSH_EXT_SIG_RSA_SHA256;
+                } else {
+                    SAFE_FREE(rsa_sig_ext);
+                    goto error; /* should never happen */
+                }
+                SAFE_FREE(rsa_sig_ext);
+            }
+
             SSH_LOG(SSH_LOG_DEBUG, "The client supports extension "
                     "negotiation. Enabled signature algorithms: %s%s",
                     session->extensions & SSH_EXT_SIG_RSA_SHA256 ? "SHA256" : "",
-- 
2.19.1.593.gc670b1f