From 6d0624f7a5c7b0b6801fb64532de3b0c35064903 Mon Sep 17 00:00:00 2001 From: Daniele Lacamera Date: Thu, 29 Oct 2026 22:12:04 +0100 Subject: [PATCH 1/2] Check output buffer size in wolfSSL_EVP_PKEY_encrypt. --- tests/api/test_evp_pkey.c | 74 +++++++++++++++++++++++++++++++++++++++ tests/api/test_evp_pkey.h | 2 ++ wolfcrypt/src/evp.c | 41 +++++++++++++--------- 3 files changed, 101 insertions(+), 16 deletions(-) diff --git a/tests/api/test_evp_pkey.c b/tests/api/test_evp_pkey.c index 632af9eae4d..7e55ec0e902 100644 --- a/tests/api/test_evp_pkey.c +++ b/tests/api/test_evp_pkey.c @@ -2804,6 +2804,80 @@ int test_wolfSSL_EVP_PKEY_decrypt_outLen(void) return EXPECT_RESULT(); } +int test_wolfSSL_EVP_PKEY_encrypt_outLen(void) +{ + EXPECT_DECLS; +#if defined(OPENSSL_EXTRA) && !defined(NO_RSA) && defined(WOLFSSL_KEY_GEN) + WOLFSSL_RSA* rsa = NULL; + WOLFSSL_EVP_PKEY* pkey = NULL; + WOLFSSL_EVP_PKEY_CTX* ctx = NULL; + const size_t rsaKeySz = 2048/8; + const size_t bufSz = 32; + byte in[200]; + byte* enc = NULL; + byte* dec = NULL; + size_t encLen = 2048/8; + size_t decLen = 0; + size_t i = 0; + + XMEMSET(in, 0x5a, sizeof(in)); + + ExpectNotNull(enc = (byte*)XMALLOC(rsaKeySz, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER)); + if (enc != NULL) { + XMEMSET(enc, 0xa5, rsaKeySz); + } + ExpectNotNull(dec = (byte*)XMALLOC(rsaKeySz, HEAP_HINT, + DYNAMIC_TYPE_TMP_BUFFER)); + if (dec != NULL) { + XMEMSET(dec, 0, rsaKeySz); + } + + ExpectNotNull(rsa = RSA_generate_key(2048, WC_RSA_EXPONENT, NULL, NULL)); + ExpectNotNull(pkey = wolfSSL_EVP_PKEY_new()); + ExpectIntEQ(EVP_PKEY_assign_RSA(pkey, rsa), WOLFSSL_SUCCESS); + if (EXPECT_FAIL()) { + RSA_free(rsa); + } + ExpectNotNull(ctx = EVP_PKEY_CTX_new(pkey, NULL)); + if (pkey != NULL) { + EVP_PKEY_free(pkey); + } + + ExpectIntEQ(EVP_PKEY_encrypt_init(ctx), WOLFSSL_SUCCESS); + ExpectIntEQ(EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING), + WOLFSSL_SUCCESS); + + encLen = bufSz; + ExpectIntEQ(EVP_PKEY_encrypt(ctx, enc, &encLen, in, sizeof(in)), + WOLFSSL_FAILURE); + if (enc != NULL) { + for (i = 0; i < rsaKeySz; i++) { + if (enc[i] != 0xa5) + break; + } + } + ExpectIntEQ(i, rsaKeySz); + + encLen = rsaKeySz; + ExpectIntEQ(EVP_PKEY_encrypt(ctx, enc, &encLen, in, sizeof(in)), + WOLFSSL_SUCCESS); + ExpectIntEQ(encLen, rsaKeySz); + + ExpectIntEQ(EVP_PKEY_decrypt_init(ctx), WOLFSSL_SUCCESS); + decLen = rsaKeySz; + ExpectIntEQ(EVP_PKEY_decrypt(ctx, dec, &decLen, enc, encLen), + WOLFSSL_SUCCESS); + ExpectIntEQ(decLen, sizeof(in)); + ExpectIntEQ(XMEMCMP(dec, in, sizeof(in)), 0); + + EVP_PKEY_CTX_free(ctx); + XFREE(enc, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(dec, HEAP_HINT, DYNAMIC_TYPE_TMP_BUFFER); +#endif + return EXPECT_RESULT(); +} + int test_wolfSSL_EVP_PKEY_derive(void) { EXPECT_DECLS; diff --git a/tests/api/test_evp_pkey.h b/tests/api/test_evp_pkey.h index 874391edb21..a5da57a8c0b 100644 --- a/tests/api/test_evp_pkey.h +++ b/tests/api/test_evp_pkey.h @@ -69,6 +69,7 @@ int test_wolfSSL_EVP_DigestSign(void); int test_wolfSSL_EVP_DigestSign_ecc(void); int test_wolfSSL_EVP_PKEY_encrypt(void); int test_wolfSSL_EVP_PKEY_decrypt_outLen(void); +int test_wolfSSL_EVP_PKEY_encrypt_outLen(void); int test_wolfSSL_EVP_PKEY_derive(void); int test_wolfSSL_EVP_PKEY_print_public(void); int test_wolfSSL_EVP_PKEY_ed25519(void); @@ -130,6 +131,7 @@ int test_wolfSSL_CTX_use_PrivateKey_pkcs8_repopulate(void); TEST_DECL_GROUP("evp_pkey", test_wolfSSL_EVP_DigestSign_ecc), \ TEST_DECL_GROUP("evp_pkey", test_wolfSSL_EVP_PKEY_encrypt), \ TEST_DECL_GROUP("evp_pkey", test_wolfSSL_EVP_PKEY_decrypt_outLen), \ + TEST_DECL_GROUP("evp_pkey", test_wolfSSL_EVP_PKEY_encrypt_outLen), \ TEST_DECL_GROUP("evp_pkey", test_wolfSSL_EVP_PKEY_derive), \ TEST_DECL_GROUP("evp_pkey", test_wolfSSL_EVP_PKEY_print_public), \ TEST_DECL_GROUP("evp_pkey", test_wolfSSL_EVP_PKEY_ed25519), \ diff --git a/wolfcrypt/src/evp.c b/wolfcrypt/src/evp.c index 3b212155b64..ee7594dccf1 100644 --- a/wolfcrypt/src/evp.c +++ b/wolfcrypt/src/evp.c @@ -3342,33 +3342,42 @@ int wolfSSL_EVP_PKEY_encrypt(WOLFSSL_EVP_PKEY_CTX *ctx, switch (ctx->pkey->type) { #if !defined(NO_RSA) case WC_EVP_PKEY_RSA: - if (out == NULL) { - if (ctx->pkey->rsa == NULL) { - WOLFSSL_MSG("Internal wolfCrypt RSA object is NULL."); - return WOLFSSL_FAILURE; - } - len = wolfSSL_RSA_size(ctx->pkey->rsa); - if (len <= 0) { - WOLFSSL_MSG("Error getting RSA size."); - return WOLFSSL_FAILURE; - } - if (outLen == NULL) { - WOLFSSL_MSG("outLen is NULL."); - return WOLFSSL_FAILURE; - } + { + int rsaSz; - *outLen = (size_t)len; + if (ctx->pkey->rsa == NULL) { + WOLFSSL_MSG("Internal wolfCrypt RSA object is NULL."); + return WOLFSSL_FAILURE; + } + if (outLen == NULL) { + WOLFSSL_MSG("outLen is NULL."); + return WOLFSSL_FAILURE; + } + rsaSz = wolfSSL_RSA_size(ctx->pkey->rsa); + if (rsaSz <= 0) { + WOLFSSL_MSG("Error getting RSA size."); + return WOLFSSL_FAILURE; + } + + if (out == NULL) { + *outLen = (size_t)rsaSz; return WOLFSSL_SUCCESS; } + if (*outLen < (size_t)rsaSz) { + WOLFSSL_MSG("Output buffer too small."); + return WOLFSSL_FAILURE; + } + len = wolfSSL_RSA_public_encrypt((int)inLen, (unsigned char *)in, out, - ctx->pkey->rsa, ctx->padding); + ctx->pkey->rsa, ctx->padding); if (len < 0) break; else { *outLen = (size_t)len; return WOLFSSL_SUCCESS; } + } #endif /* NO_RSA */ case WC_EVP_PKEY_EC: From 3ea4071980e13a82bda4bb2a877fd7dce42ad220 Mon Sep 17 00:00:00 2001 From: Daniele Lacamera Date: Fri, 30 Oct 2026 09:47:00 +0100 Subject: [PATCH 2/2] Set outlen to buffer size before EVP_PKEY_encrypt in openssl_pkey0_test. --- wolfcrypt/test/test.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wolfcrypt/test/test.c b/wolfcrypt/test/test.c index 8f9cd7acda9..ad6bc53c3a4 100644 --- a/wolfcrypt/test/test.c +++ b/wolfcrypt/test/test.c @@ -48328,6 +48328,7 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t openssl_pkey0_test(void) goto openssl_pkey0_test_done; } XMEMSET(out, 0, sizeof(out)); + outlen = sizeof(out); ret = wolfSSL_EVP_PKEY_encrypt(enc, out, &outlen, in, sizeof(in)); if (ret != 1) { printf("error encrypting msg\n"); @@ -48381,6 +48382,7 @@ WOLFSSL_TEST_SUBROUTINE wc_test_ret_t openssl_pkey0_test(void) #endif XMEMSET(out, 0, sizeof(out)); + outlen = sizeof(out); ret = wolfSSL_EVP_PKEY_encrypt(enc, out, &outlen, in, sizeof(in)); if (ret != 1) { printf("error encrypting msg\n");