[PATCH 06/11] pki_crypto: get compiling with BoringSSL

[Jon Simons <jon@xxxxxxxxxxxxx>]

With this change, pki_crypto.c will compile with BoringSSL.  To
get this working here's what I did:

 * Introduce a libcrypto-boringssl-compat.{c,h}.  These files
   are reduced copies of the libcrypto-compat.{c,h} ones, with
   the intent that they be as small as is necessary going forward.

 * Tweak src/CMakeLists.txt so that the BoringSSL compat shim
   is built when WITH_BORINGSSL is in effect.

 * Bring in the BoringSSL compat shim from pki_crypto.c whenever
   OPENSSL_IS_BORINGSSL is set.

Signed-off-by: Jon Simons <jon@xxxxxxxxxxxxx>
---
 src/CMakeLists.txt               |  10 ++-
 src/libcrypto-boringssl-compat.c | 181 +++++++++++++++++++++++++++++++++++++++
 src/libcrypto-boringssl-compat.h |  28 ++++++
 src/pki_crypto.c                 |  20 +++--
 4 files changed, 226 insertions(+), 13 deletions(-)
 create mode 100644 src/libcrypto-boringssl-compat.c
 create mode 100644 src/libcrypto-boringssl-compat.h

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 489e4ab4..6496d9f6 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -167,11 +167,13 @@ else (WITH_GCRYPT)
         ecdh_crypto.c
         libcrypto.c
        )
-    if(OPENSSL_VERSION VERSION_LESS "1.1.0")
-        if (NOT WITH_BORINGSSL)
+    if (WITH_BORINGSSL)
+        set(libssh_SRCS ${libssh_SRCS} libcrypto-boringssl-compat.c)
+    else (WITH_BORINGSSL)
+        if (OPENSSL_VERSION VERSION_LESS "1.1.0")
             set(libssh_SRCS ${libssh_SRCS} libcrypto-compat.c)
-        endif (NOT WITH_BORINGSSL)
-    endif()
+        endif (OPENSSL_VERSION VERSION_LESS "1.1.0")
+    endif (WITH_BORINGSSL)
 endif (WITH_GCRYPT)
 
 if (WITH_SFTP)
diff --git a/src/libcrypto-boringssl-compat.c b/src/libcrypto-boringssl-compat.c
new file mode 100644
index 00000000..ac11d1e3
--- /dev/null
+++ b/src/libcrypto-boringssl-compat.c
@@ -0,0 +1,181 @@
+/*
+ * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+/*
+ * libcrypto-boringssl-compat.c --
+ *   OpenSSL compat shim adapted for use by libssh with BoringSSL.
+ */
+
+#include "config.h"
+
+#include <string.h>
+#include <openssl/engine.h>
+#include "libcrypto-boringssl-compat.h"
+
+#if !defined(OPENSSL_IS_BORINGSSL)
+#error "BoringSSL libcrypto compat used for OpenSSL build"
+#endif /* !defined(OPENSSL_IS_BORINGSSL) */
+
+int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d)
+{
+    /* If the fields n and e in r are NULL, the corresponding input
+     * parameters MUST be non-NULL for n and e.  d may be
+     * left NULL (in case only the public key is used).
+     */
+    if ((r->n == NULL && n == NULL)
+        || (r->e == NULL && e == NULL))
+        return 0;
+
+    if (n != NULL) {
+        BN_free(r->n);
+        r->n = n;
+    }
+    if (e != NULL) {
+        BN_free(r->e);
+        r->e = e;
+    }
+    if (d != NULL) {
+        BN_free(r->d);
+        r->d = d;
+    }
+
+    return 1;
+}
+
+int RSA_set0_factors(RSA *r, BIGNUM *p, BIGNUM *q)
+{
+    /* If the fields p and q in r are NULL, the corresponding input
+     * parameters MUST be non-NULL.
+     */
+    if ((r->p == NULL && p == NULL)
+        || (r->q == NULL && q == NULL))
+        return 0;
+
+    if (p != NULL) {
+        BN_free(r->p);
+        r->p = p;
+    }
+    if (q != NULL) {
+        BN_free(r->q);
+        r->q = q;
+    }
+
+    return 1;
+}
+
+int RSA_set0_crt_params(RSA *r, BIGNUM *dmp1, BIGNUM *dmq1, BIGNUM *iqmp)
+{
+    /* If the fields dmp1, dmq1 and iqmp in r are NULL, the corresponding input
+     * parameters MUST be non-NULL.
+     */
+    if ((r->dmp1 == NULL && dmp1 == NULL)
+        || (r->dmq1 == NULL && dmq1 == NULL)
+        || (r->iqmp == NULL && iqmp == NULL))
+        return 0;
+
+    if (dmp1 != NULL) {
+        BN_free(r->dmp1);
+        r->dmp1 = dmp1;
+    }
+    if (dmq1 != NULL) {
+        BN_free(r->dmq1);
+        r->dmq1 = dmq1;
+    }
+    if (iqmp != NULL) {
+        BN_free(r->iqmp);
+        r->iqmp = iqmp;
+    }
+
+    return 1;
+}
+
+int DSA_set0_pqg(DSA *d, BIGNUM *p, BIGNUM *q, BIGNUM *g)
+{
+    /* If the fields p, q and g in d are NULL, the corresponding input
+     * parameters MUST be non-NULL.
+     */
+    if ((d->p == NULL && p == NULL)
+        || (d->q == NULL && q == NULL)
+        || (d->g == NULL && g == NULL))
+        return 0;
+
+    if (p != NULL) {
+        BN_free(d->p);
+        d->p = p;
+    }
+    if (q != NULL) {
+        BN_free(d->q);
+        d->q = q;
+    }
+    if (g != NULL) {
+        BN_free(d->g);
+        d->g = g;
+    }
+
+    return 1;
+}
+
+int DSA_set0_key(DSA *d, BIGNUM *pub_key, BIGNUM *priv_key)
+{
+    /* If the field pub_key in d is NULL, the corresponding input
+     * parameters MUST be non-NULL.  The priv_key field may
+     * be left NULL.
+     */
+    if (d->pub_key == NULL && pub_key == NULL)
+        return 0;
+
+    if (pub_key != NULL) {
+        BN_free(d->pub_key);
+        d->pub_key = pub_key;
+    }
+    if (priv_key != NULL) {
+        BN_free(d->priv_key);
+        d->priv_key = priv_key;
+    }
+
+    return 1;
+}
+
+void DSA_SIG_get0(const DSA_SIG *sig, const BIGNUM **pr, const BIGNUM **ps)
+{
+    if (pr != NULL)
+        *pr = sig->r;
+    if (ps != NULL)
+        *ps = sig->s;
+}
+
+int DSA_SIG_set0(DSA_SIG *sig, BIGNUM *r, BIGNUM *s)
+{
+    if (r == NULL || s == NULL)
+        return 0;
+    BN_clear_free(sig->r);
+    BN_clear_free(sig->s);
+    sig->r = r;
+    sig->s = s;
+    return 1;
+}
+
+void ECDSA_SIG_get0(const ECDSA_SIG *sig, const BIGNUM **pr, const BIGNUM **ps)
+{
+    if (pr != NULL)
+        *pr = sig->r;
+    if (ps != NULL)
+        *ps = sig->s;
+}
+
+int ECDSA_SIG_set0(ECDSA_SIG *sig, BIGNUM *r, BIGNUM *s)
+{
+    if (r == NULL || s == NULL)
+        return 0;
+    BN_clear_free(sig->r);
+    BN_clear_free(sig->s);
+    sig->r = r;
+    sig->s = s;
+    return 1;
+}
diff --git a/src/libcrypto-boringssl-compat.h b/src/libcrypto-boringssl-compat.h
new file mode 100644
index 00000000..66fed529
--- /dev/null
+++ b/src/libcrypto-boringssl-compat.h
@@ -0,0 +1,28 @@
+#ifndef LIBCRYPTO_BORINGSSL_COMPAT_H
+#define LIBCRYPTO_BORINGSSL_COMPAT_H
+
+#include <openssl/opensslv.h>
+
+#if !defined(OPENSSL_IS_BORINGSSL)
+#error "BoringSSL libcrypto compat header used for OpenSSL build"
+#endif /* !defined(OPENSSL_IS_BORINGSSL) */
+
+#include <openssl/bn.h>
+#include <openssl/rsa.h>
+#include <openssl/dsa.h>
+#include <openssl/ecdsa.h>
+
+int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d);
+int RSA_set0_factors(RSA *r, BIGNUM *p, BIGNUM *q);
+int RSA_set0_crt_params(RSA *r, BIGNUM *dmp1, BIGNUM *dmq1, BIGNUM *iqmp);
+
+int DSA_set0_pqg(DSA *d, BIGNUM *p, BIGNUM *q, BIGNUM *g);
+int DSA_set0_key(DSA *d, BIGNUM *pub_key, BIGNUM *priv_key);
+
+void DSA_SIG_get0(const DSA_SIG *sig, const BIGNUM **pr, const BIGNUM **ps);
+int DSA_SIG_set0(DSA_SIG *sig, BIGNUM *r, BIGNUM *s);
+
+void ECDSA_SIG_get0(const ECDSA_SIG *sig, const BIGNUM **pr, const BIGNUM **ps);
+int ECDSA_SIG_set0(ECDSA_SIG *sig, BIGNUM *r, BIGNUM *s);
+
+#endif /* LIBCRYPTO_BORINGSSL_COMPAT_H */
diff --git a/src/pki_crypto.c b/src/pki_crypto.c
index e061696d..f2faa15f 100644
--- a/src/pki_crypto.c
+++ b/src/pki_crypto.c
@@ -34,6 +34,8 @@
 
 #if !defined(OPENSSL_IS_BORINGSSL)
 #include "libcrypto-compat.h"
+#else  /* !defined(OPENSSL_IS_BORINGSSL) */
+#include "libcrypto-boringssl-compat.h"
 #endif /* !defined(OPENSSL_IS_BORINGSSL) */
 
 #ifdef HAVE_OPENSSL_EC_H
@@ -261,7 +263,7 @@ ssh_key pki_key_dup(const ssh_key key, int demote)
             goto fail;
         }
 
-        rc = DSA_set0_pqg(new->dsa, np, nq, ng); /* TODO: not in BoringSSL */
+        rc = DSA_set0_pqg(new->dsa, np, nq, ng);
         if (rc == 0) {
             BN_free(np);
             BN_free(nq);
@@ -275,7 +277,7 @@ ssh_key pki_key_dup(const ssh_key key, int demote)
             goto fail;
         }
 
-        rc = DSA_set0_key(new->dsa, npub_key, NULL); /* TODO: not in BoringSSL */
+        rc = DSA_set0_key(new->dsa, npub_key, NULL);
         if (rc == 0) {
             goto fail;
         }
@@ -322,7 +324,7 @@ ssh_key pki_key_dup(const ssh_key key, int demote)
             goto fail;
         }
 
-        rc = RSA_set0_key(new->rsa, nn, ne, NULL); /* TODO: not in BoringSSL */
+        rc = RSA_set0_key(new->rsa, nn, ne, NULL);
         if (rc == 0) {
             BN_free(nn);
             BN_free(ne);
@@ -357,7 +359,7 @@ ssh_key pki_key_dup(const ssh_key key, int demote)
                     goto fail;
                 }
 
-                rc = RSA_set0_factors(new->rsa, np, nq); /* TODO: not in BoringSSL */
+                rc = RSA_set0_factors(new->rsa, np, nq);
                 if (rc == 0) {
                     BN_free(np);
                     BN_free(nq);
@@ -377,7 +379,7 @@ ssh_key pki_key_dup(const ssh_key key, int demote)
                     goto fail;
                 }
 
-                rc =  RSA_set0_crt_params(new->rsa, ndmp1, ndmq1, niqmp); /* TODO: not in BoringSSL */
+                rc =  RSA_set0_crt_params(new->rsa, ndmp1, ndmq1, niqmp);
                 if (rc == 0) {
                     BN_free(ndmp1);
                     BN_free(ndmq1);
@@ -1250,7 +1252,7 @@ static ssh_string pki_dsa_signature_to_blob(const ssh_signature sig)
     ssh_string s;
     int s_len, s_offset_in, s_offset_out;
 
-    DSA_SIG_get0(sig->dsa_sig, &pr, &ps); /* TODO: not in BoringSSL */
+    DSA_SIG_get0(sig->dsa_sig, &pr, &ps);
     r = ssh_make_bignum_string((BIGNUM *)pr);
     if (r == NULL) {
         return NULL;
@@ -1316,7 +1318,7 @@ ssh_string pki_signature_to_blob(const ssh_signature sig)
                 return NULL;
             }
 
-            ECDSA_SIG_get0(sig->ecdsa_sig, &pr, &ps); /* TODO: not in BoringSSL */
+            ECDSA_SIG_get0(sig->ecdsa_sig, &pr, &ps);
             r = ssh_make_bignum_string((BIGNUM *)pr);
             if (r == NULL) {
                 ssh_buffer_free(b);
@@ -1500,7 +1502,7 @@ ssh_signature pki_signature_from_blob(const ssh_key pubkey,
                 return NULL;
             }
 
-            rc = DSA_SIG_set0(sig->dsa_sig, pr, ps); /* TODO: not in BoringSSL */
+            rc = DSA_SIG_set0(sig->dsa_sig, pr, ps);
             if (rc == 0) {
                 ssh_signature_free(sig);
                 return NULL;
@@ -1578,7 +1580,7 @@ ssh_signature pki_signature_from_blob(const ssh_key pubkey,
                     return NULL;
                 }
 
-                rc = ECDSA_SIG_set0(sig->ecdsa_sig, pr, ps); /* TODO: not in BoringSSL */
+                rc = ECDSA_SIG_set0(sig->ecdsa_sig, pr, ps);
                 if (rc == 0) {
                     ssh_signature_free(sig);
                     return NULL;
-- 
2.14.1