From 599d02e1e79321e943440faf0c3a31f908ef4893 Mon Sep 17 00:00:00 2001 From: Anthony Hu Date: Mon, 23 Mar 2026 16:16:36 -0400 Subject: [PATCH 1/3] Add bounds checks for MP integer size in SizeASN_Items --- tests/api/test_rsa.c | 127 +++++++++++++++++++++++++++++++++++++++++++ tests/api/test_rsa.h | 4 +- wolfcrypt/src/asn.c | 10 ++++ 3 files changed, 140 insertions(+), 1 deletion(-) diff --git a/tests/api/test_rsa.c b/tests/api/test_rsa.c index 862af44d1ce..6487a3ea797 100644 --- a/tests/api/test_rsa.c +++ b/tests/api/test_rsa.c @@ -1152,3 +1152,130 @@ int test_wc_RsaDecrypt_BoundsCheck(void) return EXPECT_RESULT(); } /* END test_wc_RsaDecryptBoundsCheck */ +/* + * Test wc_RsaKeyToDer with an mp_int large enough to wrap size calculations. + */ +int test_wc_RsaKeyToDer_SizeOverflow(void) +{ + EXPECT_DECLS; +#if !defined(NO_RSA) && defined(USE_INTEGER_HEAP_MATH) && \ + defined(WOLFSSL_ASN_TEMPLATE) && \ + (defined(WOLFSSL_KEY_GEN) || defined(WOLFSSL_KEY_TO_DER)) && \ + !defined(_WIN32) +#include + RsaKey key; + int i; + int derRet; + int crafted_used; + int top_bits; + mp_digit top_digit; + mp_digit* crafted_dp = MAP_FAILED; + size_t dp_bytes = 0; + + int orig_used = 0; + int orig_alloc = 0; + int orig_sign = 0; + mp_digit* orig_dp = NULL; + + mp_int* fields[8]; + + XMEMSET(&key, 0, sizeof(key)); + + /* Find 'used' count that makes (used-1)*DIGIT_BIT + top_bits = -48 + * as signed int, causing mp_unsigned_bin_size to return -6. */ + { + unsigned int target = 0xFFFFFFD0u; /* -48 as unsigned 32-bit */ + int found = 0; + + crafted_used = 0; + top_bits = 0; + top_digit = 0; + + for (top_bits = 1; top_bits < DIGIT_BIT; top_bits++) { + unsigned int base = target - (unsigned int)top_bits; + if (base % (unsigned int)DIGIT_BIT == 0) { + crafted_used = (int)(base / (unsigned int)DIGIT_BIT) + 1; + top_digit = (mp_digit)1 << (top_bits - 1); + found = 1; + break; + } + } + if (!found) { + return TEST_SKIPPED; + } + } + + dp_bytes = (size_t)crafted_used * sizeof(mp_digit); + + ExpectIntEQ(wc_InitRsaKey(&key, HEAP_HINT), 0); + + /* Set up dummy RSA private key fields. */ + key.type = RSA_PRIVATE; + fields[0] = &key.n; + fields[1] = &key.e; + fields[2] = &key.d; + fields[3] = &key.p; + fields[4] = &key.q; + fields[5] = &key.dP; + fields[6] = &key.dQ; + fields[7] = &key.u; + + for (i = 0; i < 8; i++) { + if (EXPECT_SUCCESS()) { + ExpectIntEQ(mp_init(fields[i]), 0); + mp_set(fields[i], 0x42); + } + } + + if (EXPECT_SUCCESS()) { + orig_used = key.p.used; + orig_alloc = key.p.alloc; + orig_sign = key.p.sign; + orig_dp = key.p.dp; + } + + /* Sparse mmap — only the page at dp[used-1] is faulted in. */ + if (EXPECT_SUCCESS()) { + crafted_dp = (mp_digit*)mmap(NULL, dp_bytes, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS +#ifdef MAP_NORESERVE + | MAP_NORESERVE +#endif + , -1, 0); + } + if (crafted_dp == MAP_FAILED) { + key.p.dp = orig_dp; + key.p.used = orig_used; + key.p.alloc = orig_alloc; + key.p.sign = orig_sign; + DoExpectIntEQ(wc_FreeRsaKey(&key), 0); + return TEST_SKIPPED; + } + + crafted_dp[crafted_used - 1] = top_digit; + + key.p.dp = crafted_dp; + key.p.used = crafted_used; + key.p.alloc = crafted_used; + key.p.sign = 0; /* MP_ZPOS */ + + /* Should return an error, not a bogus small size. */ + derRet = wc_RsaKeyToDer(&key, NULL, 0); + ExpectIntLT(derRet, 0); + + /* Restore key.p before cleanup. */ + key.p.dp = orig_dp; + key.p.used = orig_used; + key.p.alloc = orig_alloc; + key.p.sign = orig_sign; + + if (crafted_dp != MAP_FAILED) { + munmap(crafted_dp, dp_bytes); + } + + DoExpectIntEQ(wc_FreeRsaKey(&key), 0); +#endif + return EXPECT_RESULT(); +} /* END test_wc_RsaKeyToDer_SizeOverflow */ + diff --git a/tests/api/test_rsa.h b/tests/api/test_rsa.h index ff1af18e812..028c76543f4 100644 --- a/tests/api/test_rsa.h +++ b/tests/api/test_rsa.h @@ -42,6 +42,7 @@ int test_wc_RsaEncryptSize(void); int test_wc_RsaSSL_SignVerify(void); int test_wc_RsaFlattenPublicKey(void); int test_wc_RsaDecrypt_BoundsCheck(void); +int test_wc_RsaKeyToDer_SizeOverflow(void); #define TEST_RSA_DECLS \ TEST_DECL_GROUP("rsa", test_wc_InitRsaKey), \ @@ -61,6 +62,7 @@ int test_wc_RsaDecrypt_BoundsCheck(void); TEST_DECL_GROUP("rsa", test_wc_RsaEncryptSize), \ TEST_DECL_GROUP("rsa", test_wc_RsaSSL_SignVerify), \ TEST_DECL_GROUP("rsa", test_wc_RsaFlattenPublicKey), \ - TEST_DECL_GROUP("rsa", test_wc_RsaDecrypt_BoundsCheck) + TEST_DECL_GROUP("rsa", test_wc_RsaDecrypt_BoundsCheck), \ + TEST_DECL_GROUP("rsa", test_wc_RsaKeyToDer_SizeOverflow) #endif /* WOLFCRYPT_TEST_RSA_H */ diff --git a/wolfcrypt/src/asn.c b/wolfcrypt/src/asn.c index b095b5c204e..cbbe218cc41 100644 --- a/wolfcrypt/src/asn.c +++ b/wolfcrypt/src/asn.c @@ -864,8 +864,18 @@ int SizeASN_Items(const ASNItem* asn, ASNSetData *data, int count, case ASN_DATA_TYPE_MP: /* Calculate the size of the MP integer data. */ length = mp_unsigned_bin_size(data[i].data.mp); + if (length < 0) { + return ASN_PARSE_E; + } length += mp_leading_bit(data[i].data.mp) ? 1 : 0; + if (length < 0) { + return ASN_PARSE_E; + } len = (word32)SizeASNHeader((word32)length) + (word32)length; + /* Check for overflow: header + length must not wrap word32. */ + if (len < (word32)length) { + return ASN_PARSE_E; + } break; case ASN_DATA_TYPE_REPLACE_BUFFER: From 573ff92d13544f9f36f1ba7f9d3079f00131d43a Mon Sep 17 00:00:00 2001 From: Anthony Hu Date: Thu, 9 Apr 2026 14:10:59 -0400 Subject: [PATCH 2/3] test improvement --- tests/api/test_rsa.c | 54 ++++++++++++++++---------------------------- 1 file changed, 20 insertions(+), 34 deletions(-) diff --git a/tests/api/test_rsa.c b/tests/api/test_rsa.c index 6487a3ea797..94ed88ad3ba 100644 --- a/tests/api/test_rsa.c +++ b/tests/api/test_rsa.c @@ -1160,17 +1160,15 @@ int test_wc_RsaKeyToDer_SizeOverflow(void) EXPECT_DECLS; #if !defined(NO_RSA) && defined(USE_INTEGER_HEAP_MATH) && \ defined(WOLFSSL_ASN_TEMPLATE) && \ - (defined(WOLFSSL_KEY_GEN) || defined(WOLFSSL_KEY_TO_DER)) && \ - !defined(_WIN32) -#include + (defined(WOLFSSL_KEY_GEN) || defined(WOLFSSL_KEY_TO_DER)) RsaKey key; int i; int derRet; int crafted_used; int top_bits; mp_digit top_digit; - mp_digit* crafted_dp = MAP_FAILED; - size_t dp_bytes = 0; + mp_digit storage = 0; /* the only digit mp_count_bits ever reads */ + mp_digit* fake_dp = NULL; int orig_used = 0; int orig_alloc = 0; @@ -1181,6 +1179,11 @@ int test_wc_RsaKeyToDer_SizeOverflow(void) XMEMSET(&key, 0, sizeof(key)); + /* Skip on 32-bit: biasing dp by ~half the address space is unsafe. */ + if (sizeof(void*) < 8) { + return TEST_SKIPPED; + } + /* Find 'used' count that makes (used-1)*DIGIT_BIT + top_bits = -48 * as signed int, causing mp_unsigned_bin_size to return -6. */ { @@ -1205,8 +1208,6 @@ int test_wc_RsaKeyToDer_SizeOverflow(void) } } - dp_bytes = (size_t)crafted_used * sizeof(mp_digit); - ExpectIntEQ(wc_InitRsaKey(&key, HEAP_HINT), 0); /* Set up dummy RSA private key fields. */ @@ -1234,31 +1235,20 @@ int test_wc_RsaKeyToDer_SizeOverflow(void) orig_dp = key.p.dp; } - /* Sparse mmap — only the page at dp[used-1] is faulted in. */ + /* The vulnerable path (mp_unsigned_bin_size -> mp_count_bits, and + * mp_leading_bit) only reads dp[used-1]. Bias dp so that index + * (used-1) lands on our single real digit -- no giant allocation + * (and no mmap/VirtualAlloc) needed. */ if (EXPECT_SUCCESS()) { - crafted_dp = (mp_digit*)mmap(NULL, dp_bytes, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS -#ifdef MAP_NORESERVE - | MAP_NORESERVE -#endif - , -1, 0); + storage = top_digit; + fake_dp = (mp_digit*)((wc_ptr_t)&storage + - (wc_ptr_t)(crafted_used - 1) * sizeof(mp_digit)); + + key.p.dp = fake_dp; + key.p.used = crafted_used; + key.p.alloc = crafted_used; + key.p.sign = 0; /* MP_ZPOS */ } - if (crafted_dp == MAP_FAILED) { - key.p.dp = orig_dp; - key.p.used = orig_used; - key.p.alloc = orig_alloc; - key.p.sign = orig_sign; - DoExpectIntEQ(wc_FreeRsaKey(&key), 0); - return TEST_SKIPPED; - } - - crafted_dp[crafted_used - 1] = top_digit; - - key.p.dp = crafted_dp; - key.p.used = crafted_used; - key.p.alloc = crafted_used; - key.p.sign = 0; /* MP_ZPOS */ /* Should return an error, not a bogus small size. */ derRet = wc_RsaKeyToDer(&key, NULL, 0); @@ -1270,10 +1260,6 @@ int test_wc_RsaKeyToDer_SizeOverflow(void) key.p.alloc = orig_alloc; key.p.sign = orig_sign; - if (crafted_dp != MAP_FAILED) { - munmap(crafted_dp, dp_bytes); - } - DoExpectIntEQ(wc_FreeRsaKey(&key), 0); #endif return EXPECT_RESULT(); From 6377ed03ce07e67eec939b094711156944cbb0c2 Mon Sep 17 00:00:00 2001 From: Anthony Hu Date: Fri, 10 Apr 2026 14:27:48 -0400 Subject: [PATCH 3/3] better gating. --- tests/api/test_rsa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api/test_rsa.c b/tests/api/test_rsa.c index 94ed88ad3ba..05fdd298bb1 100644 --- a/tests/api/test_rsa.c +++ b/tests/api/test_rsa.c @@ -1159,7 +1159,7 @@ int test_wc_RsaKeyToDer_SizeOverflow(void) { EXPECT_DECLS; #if !defined(NO_RSA) && defined(USE_INTEGER_HEAP_MATH) && \ - defined(WOLFSSL_ASN_TEMPLATE) && \ + defined(WOLFSSL_ASN_TEMPLATE) && defined(WOLFSSL_PUBLIC_MP) && \ (defined(WOLFSSL_KEY_GEN) || defined(WOLFSSL_KEY_TO_DER)) RsaKey key; int i;