common: add RSA SPKI encryption helper

Author: Chapoly1305

src/common/crypto.h

@@ -38,6 +38,12 @@ int encrypt_aes128ecb(void *out, int *out_len, const unsigned char key[16], cons
 /* Derives key material with PBKDF2-HMAC-SHA512. */
 int pbkdf2_hmac_sha512(const uint8_t *password, size_t password_len, const uint8_t *salt, size_t salt_len, uint32_t rounds, uint8_t *out, size_t out_len);
 
+/*
+  Imports an RSA public key from SubjectPublicKeyInfo DER and encrypts 'in'
+  with PKCS#1 v1.5 padding, writing the ciphertext to 'out'.
+ */
+int encrypt_rsa_pkcs1_spki_der(uint8_t *out, size_t *out_len, const uint8_t *der, size_t der_len, const void *in, size_t in_len);
+
 /*
    Generates a Diffie-Hellman public-private keypair using the generator value 'gen' and prime modulo
    'prime', writing the result to 'pub_out' and 'priv_out', which must be 'keylen' in size.

src/common/crypto_included.c

@@ -102,6 +102,17 @@ int pbkdf2_hmac_sha512(const uint8_t *password, size_t password_len, const uint8
     return 0;
 }
 
+int encrypt_rsa_pkcs1_spki_der(uint8_t *out, size_t *out_len, const uint8_t *der, size_t der_len, const void *in, size_t in_len)
+{
+    (void)out;
+    (void)out_len;
+    (void)der;
+    (void)der_len;
+    (void)in;
+    (void)in_len;
+    return 0;
+}
+
 int dh_generate_keypair(uint8_t *priv_out, uint8_t *pub_out, const uint8_t *gen, const size_t gen_len, const uint8_t *prime, const size_t keylen)
 {
     return 0;

src/common/crypto_libgcrypt.c

@@ -234,6 +234,19 @@ int pbkdf2_hmac_sha512(const uint8_t *password, size_t password_len,
     return gcry_err_code(error) == GPG_ERR_NO_ERROR;
 }
 
+int encrypt_rsa_pkcs1_spki_der(uint8_t *out, size_t *out_len,
+			       const uint8_t *der, size_t der_len,
+			       const void *in, size_t in_len)
+{
+    (void)out;
+    (void)out_len;
+    (void)der;
+    (void)der_len;
+    (void)in;
+    (void)in_len;
+    return 0;
+}
+
 int dh_generate_keypair(uint8_t *priv_out, uint8_t *pub_out, const uint8_t *gen, const size_t gen_len, const uint8_t *prime, const size_t keylen)
 {
     int result = 0;

src/common/crypto_openssl.c

@@ -28,6 +28,8 @@
 #include <openssl/dh.h>
 #include <openssl/evp.h>
 #include <openssl/rand.h>
+#include <openssl/rsa.h>
+#include <openssl/x509.h>
 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
 #include <openssl/provider.h>
 #endif
@@ -210,6 +212,46 @@ int pbkdf2_hmac_sha512(const uint8_t *password, size_t password_len,
 			     EVP_sha512(), (int)out_len, out) == 1;
 }
 
+int encrypt_rsa_pkcs1_spki_der(uint8_t *out, size_t *out_len,
+			       const uint8_t *der, size_t der_len,
+			       const void *in, size_t in_len)
+{
+    int result = 0;
+    EVP_PKEY *pkey = NULL;
+    EVP_PKEY_CTX *ctx = NULL;
+    const unsigned char *derp = der;
+    size_t required_len = 0;
+
+    if (!out || !out_len || !der || !in)
+	goto out;
+
+    pkey = d2i_PUBKEY(NULL, &derp, (long)der_len);
+    if (!pkey || EVP_PKEY_base_id(pkey) != EVP_PKEY_RSA)
+	goto out;
+
+    ctx = EVP_PKEY_CTX_new(pkey, NULL);
+    if (!ctx)
+	goto out;
+    if (EVP_PKEY_encrypt_init(ctx) <= 0)
+	goto out;
+    if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING) <= 0)
+	goto out;
+    if (EVP_PKEY_encrypt(ctx, NULL, &required_len, in, in_len) <= 0)
+	goto out;
+    if (required_len > *out_len)
+	goto out;
+    if (EVP_PKEY_encrypt(ctx, out, &required_len, in, in_len) <= 0)
+	goto out;
+
+    *out_len = required_len;
+    result = 1;
+
+ out:
+    EVP_PKEY_CTX_free(ctx);
+    EVP_PKEY_free(pkey);
+    return result;
+}
+
 static void pad_leading_zeros(uint8_t *out, const size_t current_len, const size_t expected_len) {
     if (current_len >= expected_len || expected_len < 1)
         return;