From 8b667e46c26e38d0af0457b7667d0b064f87a0ea Mon Sep 17 00:00:00 2001 From: Mark Atwood Date: Fri, 17 Apr 2026 13:59:13 -0700 Subject: [PATCH 1/2] perf: embed Hmac in noise_handshake to eliminate per-kdf-call malloc kdf() is called 1-4 times per handshake operation, and previously malloc'd and free'd an 832-byte Hmac struct on every call. Add a struct Hmac kdf_hmac field to struct noise_handshake (which is embedded in the heap-allocated struct wg_peer). Thread a struct Hmac * parameter through kdf() and its callers (mix_dh, mix_precomputed_dh, mix_psk, message_ephemeral, derive_keys). Callers that hold handshake->lock for write throughout (create_initiation, create_response, begin_session) pass &handshake->kdf_hmac, eliminating all per-call kmalloc for those paths. Callers that perform kdf operations outside the write lock (consume_initiation, consume_response) pass NULL, causing kdf() to fall back to kmalloc, preserving correctness under concurrent access. --- kernel-src/noise.c | 119 ++++++++++++++++++++++++++++++--------------- kernel-src/noise.h | 10 ++++ 2 files changed, 91 insertions(+), 38 deletions(-) diff --git a/kernel-src/noise.c b/kernel-src/noise.c index cd006e7..ad04955 100644 --- a/kernel-src/noise.c +++ b/kernel-src/noise.c @@ -11,6 +11,7 @@ #include "messages.h" #include "queueing.h" #include "peerlookup.h" +#include "wolfcrypt_glue.h" #include #include @@ -155,6 +156,10 @@ static void keypair_free_kref(struct kref *kref) keypair->entry.peer->internal_id); wg_index_hashtable_remove(keypair->entry.peer->device->index_hashtable, &keypair->entry); + if (keypair->aes_inited) { + wc_AesFree(&keypair->aes_encrypt); + wc_AesFree(&keypair->aes_decrypt); + } call_rcu(&keypair->rcu, keypair_free_rcu); } @@ -339,14 +344,21 @@ void wg_noise_set_static_identity_private_key( /* This is Hugo Krawczyk's HKDF: * - https://eprint.iacr.org/2010/264.pdf * - https://tools.ietf.org/html/rfc5869 + * + * hmac_buf: caller-supplied Hmac scratch buffer to avoid per-call kmalloc. + * Must be non-NULL only when called while holding handshake->lock for write, + * ensuring no concurrent use of the same buffer. Pass NULL when concurrency + * cannot be ruled out; in that case kdf() falls back to kmalloc internally. */ -static int __must_check kdf(u8 *first_dst, u8 *second_dst, u8 *third_dst, const u8 *data, +static int __must_check kdf(struct Hmac *hmac_buf, + u8 *first_dst, u8 *second_dst, u8 *third_dst, const u8 *data, size_t first_len, size_t second_len, size_t third_len, size_t data_len, const u8 chaining_key[NOISE_HASH_LEN]) { u8 output[WC_SHA256_DIGEST_SIZE + 1]; u8 secret[WC_SHA256_DIGEST_SIZE]; struct Hmac *wc_hmac; + bool hmac_alloced = false; int ret; WARN_ON(IS_ENABLED(DEBUG) && @@ -357,11 +369,16 @@ static int __must_check kdf(u8 *first_dst, u8 *second_dst, u8 *third_dst, const (!first_len || !first_dst)) || ((third_len || third_dst) && (!second_len || !second_dst)))); - wc_hmac = (struct Hmac *)malloc(sizeof(*wc_hmac)); - if (! wc_hmac) { - ret = -ENOMEM; - WC_DEBUG_PR_CODEPOINT(); - goto out; + if (hmac_buf) { + wc_hmac = hmac_buf; + } else { + wc_hmac = (struct Hmac *)malloc(sizeof(*wc_hmac)); + if (!wc_hmac) { + ret = -ENOMEM; + WC_DEBUG_PR_CODEPOINT(); + goto out; + } + hmac_alloced = true; } /* Extract entropy from data into secret */ @@ -419,18 +436,19 @@ static int __must_check kdf(u8 *first_dst, u8 *second_dst, u8 *third_dst, const memzero_explicit(secret, WC_SHA256_DIGEST_SIZE); memzero_explicit(output, WC_SHA256_DIGEST_SIZE + 1); - if (wc_hmac) + if (hmac_alloced) free(wc_hmac); WC_DEBUG_PR_NEG_RET(ret); } -static int __must_check derive_keys(struct noise_symmetric_key *first_dst, +static int __must_check derive_keys(struct Hmac *hmac_buf, + struct noise_symmetric_key *first_dst, struct noise_symmetric_key *second_dst, const u8 chaining_key[NOISE_HASH_LEN]) { u64 birthdate = ktime_get_coarse_boottime_ns(); - int ret = kdf(first_dst->key, second_dst->key, NULL, NULL, + int ret = kdf(hmac_buf, first_dst->key, second_dst->key, NULL, NULL, NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, 0, 0, chaining_key); if (ret) @@ -440,7 +458,8 @@ static int __must_check derive_keys(struct noise_symmetric_key *first_dst, return 0; } -static bool __must_check mix_dh(u8 chaining_key[NOISE_HASH_LEN], +static bool __must_check mix_dh(struct Hmac *hmac_buf, + u8 chaining_key[NOISE_HASH_LEN], u8 key[NOISE_SYMMETRIC_KEY_LEN], const u8 private[NOISE_PRIVATE_KEY_LEN], const u8 public[NOISE_PUBLIC_KEY_LEN]) @@ -453,7 +472,7 @@ static bool __must_check mix_dh(u8 chaining_key[NOISE_HASH_LEN], public, NOISE_PUBLIC_KEY_LEN, NOISE_CURVE_ID) != 0) WC_DEBUG_PR_FALSE_RET(false); - if (kdf(chaining_key, key, NULL, dh_calculation, NOISE_HASH_LEN, + if (kdf(hmac_buf, chaining_key, key, NULL, dh_calculation, NOISE_HASH_LEN, NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PRIVATE_KEY_LEN, chaining_key) == 0) ret = true; else @@ -462,14 +481,15 @@ static bool __must_check mix_dh(u8 chaining_key[NOISE_HASH_LEN], WC_DEBUG_PR_FALSE_RET(ret); } -static bool __must_check mix_precomputed_dh(u8 chaining_key[NOISE_HASH_LEN], +static bool __must_check mix_precomputed_dh(struct Hmac *hmac_buf, + u8 chaining_key[NOISE_HASH_LEN], u8 key[NOISE_SYMMETRIC_KEY_LEN], const u8 precomputed[NOISE_PRIVATE_KEY_LEN]) { static const u8 zero_point[NOISE_PRIVATE_KEY_LEN]; if (unlikely(!ConstantCompare(precomputed, zero_point, NOISE_PRIVATE_KEY_LEN))) WC_DEBUG_PR_FALSE_RET(false); - if (kdf(chaining_key, key, NULL, precomputed, NOISE_HASH_LEN, + if (kdf(hmac_buf, chaining_key, key, NULL, precomputed, NOISE_HASH_LEN, NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PRIVATE_KEY_LEN, chaining_key) != 0) { @@ -500,14 +520,15 @@ static int __must_check mix_hash(u8 hash[NOISE_HASH_LEN], const u8 *src, size_t WC_DEBUG_PR_NEG_RET(ret); } -static int __must_check mix_psk(u8 chaining_key[NOISE_HASH_LEN], u8 hash[NOISE_HASH_LEN], +static int __must_check mix_psk(struct Hmac *hmac_buf, + u8 chaining_key[NOISE_HASH_LEN], u8 hash[NOISE_HASH_LEN], u8 key[NOISE_SYMMETRIC_KEY_LEN], const u8 psk[NOISE_SYMMETRIC_KEY_LEN]) { u8 temp_hash[NOISE_HASH_LEN]; int ret; - ret = kdf(chaining_key, temp_hash, key, psk, NOISE_HASH_LEN, NOISE_HASH_LEN, + ret = kdf(hmac_buf, chaining_key, temp_hash, key, psk, NOISE_HASH_LEN, NOISE_HASH_LEN, NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, chaining_key); if (ret == 0) ret = mix_hash(hash, temp_hash, NOISE_HASH_LEN); @@ -552,7 +573,8 @@ static bool __must_check message_decrypt(u8 *dst_plaintext, size_t dst_plaintext return true; } -static int __must_check message_ephemeral(u8 ephemeral_dst[NOISE_PUBLIC_KEY_LEN], +static int __must_check message_ephemeral(struct Hmac *hmac_buf, + u8 ephemeral_dst[NOISE_PUBLIC_KEY_LEN], const u8 ephemeral_src[NOISE_PUBLIC_KEY_LEN], u8 chaining_key[NOISE_HASH_LEN], u8 hash[NOISE_HASH_LEN]) @@ -563,7 +585,7 @@ static int __must_check message_ephemeral(u8 ephemeral_dst[NOISE_PUBLIC_KEY_LEN] ret = mix_hash(hash, ephemeral_src, NOISE_PUBLIC_KEY_LEN); if (ret) WC_DEBUG_PR_NEG_RET(ret); - ret = kdf(chaining_key, NULL, NULL, ephemeral_src, NOISE_HASH_LEN, 0, 0, + ret = kdf(hmac_buf, chaining_key, NULL, NULL, ephemeral_src, NOISE_HASH_LEN, 0, 0, NOISE_PUBLIC_KEY_LEN, chaining_key); WC_DEBUG_PR_NEG_RET(ret); } @@ -613,14 +635,14 @@ wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, NOISE_CURVE_ID, WG_PUBLIC_KEY_COMPRESSED) != 0) goto out; - if (message_ephemeral(dst->unencrypted_ephemeral, + if (message_ephemeral(&handshake->kdf_hmac, dst->unencrypted_ephemeral, dst->unencrypted_ephemeral, handshake->chaining_key, handshake->hash) != 0) goto out; /* es */ - if (!mix_dh(handshake->chaining_key, key, handshake->ephemeral_private, - handshake->remote_static)) + if (!mix_dh(&handshake->kdf_hmac, handshake->chaining_key, key, + handshake->ephemeral_private, handshake->remote_static)) goto out; /* s */ @@ -630,7 +652,7 @@ wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, goto out; /* ss */ - if (!mix_precomputed_dh(handshake->chaining_key, key, + if (!mix_precomputed_dh(&handshake->kdf_hmac, handshake->chaining_key, key, handshake->precomputed_static_static)) goto out; @@ -678,11 +700,11 @@ wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, goto out; /* e */ - if (message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash) != 0) + if (message_ephemeral(NULL, e, src->unencrypted_ephemeral, chaining_key, hash) != 0) goto out; /* es */ - if (!mix_dh(chaining_key, key, wg->static_identity.static_private, e)) { + if (!mix_dh(NULL, chaining_key, key, wg->static_identity.static_private, e)) { WC_DEBUG_PR_CODEPOINT(); goto out; } @@ -703,7 +725,7 @@ wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, handshake = &peer->handshake; /* ss */ - if (!mix_precomputed_dh(chaining_key, key, + if (!mix_precomputed_dh(NULL, chaining_key, key, handshake->precomputed_static_static)) { WC_DEBUG_PR_CODEPOINT(); goto out; @@ -775,23 +797,23 @@ bool wg_noise_handshake_create_response(struct message_handshake_response *dst, NOISE_CURVE_ID, WG_PUBLIC_KEY_COMPRESSED) != 0) goto out; - if (message_ephemeral(dst->unencrypted_ephemeral, + if (message_ephemeral(&handshake->kdf_hmac, dst->unencrypted_ephemeral, dst->unencrypted_ephemeral, handshake->chaining_key, handshake->hash) != 0) goto out; /* ee */ - if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, - handshake->remote_ephemeral)) + if (!mix_dh(&handshake->kdf_hmac, handshake->chaining_key, NULL, + handshake->ephemeral_private, handshake->remote_ephemeral)) goto out; /* se */ - if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, - handshake->remote_static)) + if (!mix_dh(&handshake->kdf_hmac, handshake->chaining_key, NULL, + handshake->ephemeral_private, handshake->remote_static)) goto out; /* psk */ - if (mix_psk(handshake->chaining_key, handshake->hash, key, + if (mix_psk(&handshake->kdf_hmac, handshake->chaining_key, handshake->hash, key, handshake->preshared_key) != 0) goto out; @@ -853,19 +875,19 @@ wg_noise_handshake_consume_response(struct message_handshake_response *src, goto fail; /* e */ - if (message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash) != 0) + if (message_ephemeral(NULL, e, src->unencrypted_ephemeral, chaining_key, hash) != 0) goto fail; /* ee */ - if (!mix_dh(chaining_key, NULL, ephemeral_private, e)) + if (!mix_dh(NULL, chaining_key, NULL, ephemeral_private, e)) goto fail; /* se */ - if (!mix_dh(chaining_key, NULL, wg->static_identity.static_private, e)) + if (!mix_dh(NULL, chaining_key, NULL, wg->static_identity.static_private, e)) goto fail; /* psk */ - if (mix_psk(chaining_key, hash, key, preshared_key) != 0) + if (mix_psk(NULL, chaining_key, hash, key, preshared_key) != 0) goto fail; /* {} */ @@ -922,16 +944,37 @@ bool wg_noise_handshake_begin_session(struct noise_handshake *handshake, new_keypair->remote_index = handshake->remote_index; if (new_keypair->i_am_the_initiator) { - if (derive_keys(&new_keypair->sending, &new_keypair->receiving, - handshake->chaining_key) != 0) + if (derive_keys(&handshake->kdf_hmac, &new_keypair->sending, + &new_keypair->receiving, handshake->chaining_key) != 0) goto out; } else { - if (derive_keys(&new_keypair->receiving, &new_keypair->sending, - handshake->chaining_key) != 0) + if (derive_keys(&handshake->kdf_hmac, &new_keypair->receiving, + &new_keypair->sending, handshake->chaining_key) != 0) goto out; } + if (wc_AesInit(&new_keypair->aes_encrypt, NULL, INVALID_DEVID) != 0) + goto out; + if (wc_AesGcmSetKey(&new_keypair->aes_encrypt, + new_keypair->sending.key, + NOISE_SYMMETRIC_KEY_LEN) != 0) { + wc_AesFree(&new_keypair->aes_encrypt); + goto out; + } + if (wc_AesInit(&new_keypair->aes_decrypt, NULL, INVALID_DEVID) != 0) { + wc_AesFree(&new_keypair->aes_encrypt); + goto out; + } + if (wc_AesGcmSetKey(&new_keypair->aes_decrypt, + new_keypair->receiving.key, + NOISE_SYMMETRIC_KEY_LEN) != 0) { + wc_AesFree(&new_keypair->aes_encrypt); + wc_AesFree(&new_keypair->aes_decrypt); + goto out; + } + new_keypair->aes_inited = true; + handshake_zero(handshake); rcu_read_lock_bh(); if (likely(!READ_ONCE(container_of(handshake, struct wg_peer, diff --git a/kernel-src/noise.h b/kernel-src/noise.h index c5e055e..5f68d26 100644 --- a/kernel-src/noise.h +++ b/kernel-src/noise.h @@ -40,6 +40,9 @@ struct noise_keypair { struct kref refcount; struct rcu_head rcu; u64 internal_id; + Aes aes_encrypt; + Aes aes_decrypt; + bool aes_inited; }; struct noise_keypairs { @@ -85,6 +88,13 @@ struct noise_handshake { u8 latest_timestamp[NOISE_TIMESTAMP_LEN]; __le32 remote_index; + /* Scratch buffer for kdf() calls made while holding the handshake + * write lock. Avoids per-call kmalloc of the 832-byte Hmac struct. + * Must not be used from consume_initiation or consume_response, which + * perform kdf operations outside the write lock. + */ + struct Hmac kdf_hmac; + /* Protects all members except the immutable (after noise_handshake_ * init): remote_static, precomputed_static_static, static_identity. */ From e0aa9b39d0355ca5d8ec29855afc376d35571538 Mon Sep 17 00:00:00 2001 From: Mark Atwood Date: Fri, 17 Apr 2026 14:02:29 -0700 Subject: [PATCH 2/2] perf: use prealloc Aes in encrypt/decrypt hot path (ue7.12) Add wc_AesGcm_encrypt_sg_inplace_prealloc and wc_AesGcm_decrypt_sg_inplace_prealloc to wolfcrypt_glue.c/.h. These accept a pre-initialised Aes* instead of allocating one per call. The key schedule set at keypair creation time is reused by passing NULL key to wc_AesGcmEncryptInit/DecryptInit. Update send.c encrypt_packet and receive.c decrypt_packet to call the prealloc variants with the Aes fields embedded in noise_keypair (added in the preceding commit), eliminating the per-packet kmalloc/kfree of the Aes struct. --- kernel-src/receive.c | 7 +- kernel-src/send.c | 6 +- kernel-src/wolfcrypt_glue.c | 261 ++++++++++++++++++++++++++++++++++++ kernel-src/wolfcrypt_glue.h | 10 ++ 4 files changed, 277 insertions(+), 7 deletions(-) diff --git a/kernel-src/receive.c b/kernel-src/receive.c index 72f7536..5110e6b 100644 --- a/kernel-src/receive.c +++ b/kernel-src/receive.c @@ -301,10 +301,9 @@ static bool decrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair) if (skb_to_sgvec(skb, sg, 0, skb->len) <= 0) return false; - if (! wc_AesGcm_decrypt_sg_inplace(sg, skb->len, NULL, 0, - PACKET_CB(skb)->nonce, - keypair->receiving.key, - sizeof(keypair->receiving.key))) + if (! wc_AesGcm_decrypt_sg_inplace_prealloc(sg, skb->len, NULL, 0, + PACKET_CB(skb)->nonce, + &keypair->aes_decrypt)) return false; /* Another ugly situation of pushing and pulling the header so as to diff --git a/kernel-src/send.c b/kernel-src/send.c index c7705bf..3ce77fc 100644 --- a/kernel-src/send.c +++ b/kernel-src/send.c @@ -264,9 +264,9 @@ static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair) if (skb_to_sgvec(skb, sg, sizeof(struct message_data), noise_encrypted_len(plaintext_len)) <= 0) return false; - return wc_AesGcm_encrypt_sg_inplace(sg, plaintext_len, NULL, 0, - PACKET_CB(skb)->nonce, - keypair->sending.key, sizeof(keypair->sending.key)); + return wc_AesGcm_encrypt_sg_inplace_prealloc(sg, plaintext_len, NULL, 0, + PACKET_CB(skb)->nonce, + &keypair->aes_encrypt); } void wg_packet_send_keepalive(struct wg_peer *peer) diff --git a/kernel-src/wolfcrypt_glue.c b/kernel-src/wolfcrypt_glue.c index a3778e5..f20505b 100644 --- a/kernel-src/wolfcrypt_glue.c +++ b/kernel-src/wolfcrypt_glue.c @@ -487,6 +487,267 @@ bool wc_AesGcm_decrypt_sg_inplace(struct scatterlist *src, size_t src_len, ad, ad_len, nonce, key, key_len, 1)); } +#ifdef WOLFSSL_AESGCM_STREAM + +static __always_inline bool wc_AesGcm_crypt_sg_inplace_prealloc( + struct scatterlist *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + u64 nonce, + Aes *aes, + int isDecrypt) +{ + int ret = -1; + struct sg_mapping_iter miter; + int miter_needs_stop = 0; + unsigned int flags; + ssize_t sl; + byte full_nonce[AES_IV_SIZE]; + + if (WARN_ON((src_len > UINT_MAX) || + (ad_len > UINT_MAX))) + { + ret = -EINVAL; + WC_DEBUG_PR_CODEPOINT(); + goto out; + } + + memset(full_nonce, 0, sizeof(full_nonce)); +#ifdef BIG_ENDIAN_ORDER + nonce = cpu_to_le64(nonce); +#endif + memcpy(full_nonce + 4, (u8 *)&nonce, sizeof(nonce)); + + /* Pass NULL key: wc_AesGcmInit skips wc_AesGcmSetKey when key is NULL, + * reusing the key schedule already set at keypair creation time. + */ + if (isDecrypt) + ret = wc_AesGcmDecryptInit(aes, NULL, 0, + full_nonce, (word32)sizeof(full_nonce)); + else + ret = wc_AesGcmEncryptInit(aes, NULL, 0, + full_nonce, (word32)sizeof(full_nonce)); + wc_ForceZero(full_nonce, sizeof full_nonce); + if (ret != 0) { + WC_DEBUG_PR_CODEPOINT(); + goto out; + } + + if (ad) { + if (isDecrypt) + ret = wc_AesGcmDecryptUpdate(aes, NULL, NULL, + 0, ad, ad_len); + else + ret = wc_AesGcmEncryptUpdate(aes, NULL, NULL, + 0, ad, ad_len); + if (ret != 0) { + WC_DEBUG_PR_CODEPOINT(); + goto out; + } + } + + flags = SG_MITER_TO_SG | SG_MITER_ATOMIC; + + sg_miter_start(&miter, src, sg_nents(src), flags); + miter_needs_stop = 1; + + for (sl = (ssize_t)src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { + size_t length = min_t(size_t, sl, (ssize_t)miter.length); + + if (isDecrypt) + ret = wc_AesGcmDecryptUpdate(aes, miter.addr, miter.addr, + length, NULL, 0); + else + ret = wc_AesGcmEncryptUpdate(aes, miter.addr, miter.addr, + length, NULL, 0); + if (ret != 0) { + WC_DEBUG_PR_CODEPOINT(); + goto out; + } + } + + if (sl <= -WC_AES_BLOCK_SIZE) { + if (isDecrypt) + ret = wc_AesGcmDecryptFinal(aes, miter.addr + (ssize_t)miter.length + sl, WC_AES_BLOCK_SIZE); + else + ret = wc_AesGcmEncryptFinal(aes, miter.addr + (ssize_t)miter.length + sl, WC_AES_BLOCK_SIZE); + if (ret < 0) { + WC_DEBUG_PR_CODEPOINT(); + goto out; + } + } + + sg_miter_stop(&miter); + miter_needs_stop = 0; + + if (sl > -WC_AES_BLOCK_SIZE) { + byte AuthTagBuf[WC_AES_BLOCK_SIZE]; + + if (isDecrypt) { + scatterwalk_map_and_copy(AuthTagBuf, src, src_len, + sizeof AuthTagBuf, 0 /* isEncrypt */); + ret = wc_AesGcmDecryptFinal(aes, AuthTagBuf, WC_AES_BLOCK_SIZE); + if (ret < 0) + goto out; + } else { + ret = wc_AesGcmEncryptFinal(aes, AuthTagBuf, WC_AES_BLOCK_SIZE); + if (ret < 0) + goto out; + scatterwalk_map_and_copy(AuthTagBuf, src, src_len, + sizeof AuthTagBuf, 1 /* isEncrypt */); + } + } + + ret = 0; + + out: + + if (miter_needs_stop) + sg_miter_stop(&miter); + + WC_DEBUG_PR_IF_NEG(ret); + + return ret == 0; +} + +#else /* !WOLFSSL_AESGCM_STREAM */ + +static __always_inline bool wc_AesGcm_crypt_sg_inplace_prealloc( + struct scatterlist *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + u64 nonce, + Aes *aes, + int isDecrypt) +{ + int ret = -1; + struct sg_mapping_iter miter; + unsigned int flags; + byte full_nonce[AES_IV_SIZE]; + + if (WARN_ON((src_len > UINT_MAX) || + (ad_len > UINT_MAX))) + { + ret = -EINVAL; + WC_DEBUG_PR_CODEPOINT(); + goto out; + } + + if (sg_nents(src) < 1) { + ret = -EINVAL; + WC_DEBUG_PR_CODEPOINT(); + goto out; + } + + memset(full_nonce, 0, sizeof(full_nonce)); +#ifdef BIG_ENDIAN_ORDER + nonce = cpu_to_le64(nonce); +#endif + memcpy(full_nonce + 4, (u8 *)&nonce, sizeof(nonce)); + + flags = SG_MITER_TO_SG | SG_MITER_ATOMIC; + + if (sg_nents(src) == 1) { + size_t length; + + sg_miter_start(&miter, src, sg_nents(src), flags); + if ((sg_nents(src) == 1) && (! sg_miter_next(&miter))) { + sg_miter_stop(&miter); + ret = -EINVAL; + WC_DEBUG_PR_CODEPOINT(); + goto out; + } + + if (miter.length < src_len + WC_AES_BLOCK_SIZE) { + sg_miter_stop(&miter); + goto copy_after_all; + } + + length = min_t(size_t, src_len, miter.length); + + if (isDecrypt) { + ret = wc_AesGcmDecrypt(aes, miter.addr, + miter.addr, (word32)length, + full_nonce, (word32)sizeof(full_nonce), + miter.addr + length, WC_AES_BLOCK_SIZE, + ad, (word32)ad_len); + } + else { + ret = wc_AesGcmEncrypt(aes, miter.addr, + miter.addr, (word32)length, + full_nonce, (word32)sizeof(full_nonce), + miter.addr + length, WC_AES_BLOCK_SIZE, + ad, (word32)ad_len); + } + + sg_miter_stop(&miter); + + goto out; + } + + copy_after_all: + { + byte *buf = (byte *)XMALLOC(src_len + WC_AES_BLOCK_SIZE, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + if (! buf) { + ret = -ENOMEM; + WC_DEBUG_PR_CODEPOINT(); + goto out; + } + + if (isDecrypt) { + scatterwalk_map_and_copy(buf, src, 0, src_len + WC_AES_BLOCK_SIZE, 0); + ret = wc_AesGcmDecrypt(aes, buf, + buf, (word32)src_len, + full_nonce, (word32)sizeof(full_nonce), + buf + src_len, WC_AES_BLOCK_SIZE, + ad, (word32)ad_len); + if (ret == 0) + scatterwalk_map_and_copy(buf, src, 0, src_len, 1); + wc_ForceZero(buf, src_len + WC_AES_BLOCK_SIZE); + } + else { + scatterwalk_map_and_copy(buf, src, 0, src_len, 0); + ret = wc_AesGcmEncrypt(aes, buf, + buf, (word32)src_len, + full_nonce, (word32)sizeof(full_nonce), + buf + src_len, WC_AES_BLOCK_SIZE, + ad, (word32)ad_len); + if (ret == 0) + scatterwalk_map_and_copy(buf, src, 0, src_len + WC_AES_BLOCK_SIZE, 1); + else + wc_ForceZero(buf, src_len + WC_AES_BLOCK_SIZE); + } + free(buf); + } + + out: + + wc_ForceZero(full_nonce, sizeof full_nonce); + + WC_DEBUG_PR_IF_NEG(ret); + + return ret == 0; +} + +#endif /* !WOLFSSL_AESGCM_STREAM */ + +bool wc_AesGcm_encrypt_sg_inplace_prealloc(struct scatterlist *src, size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + Aes *aes) +{ + WC_DEBUG_PR_FALSE_RET(wc_AesGcm_crypt_sg_inplace_prealloc(src, src_len, ad, ad_len, + nonce, aes, 0)); +} + +bool wc_AesGcm_decrypt_sg_inplace_prealloc(struct scatterlist *src, size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + Aes *aes) +{ + WC_DEBUG_PR_FALSE_RET(wc_AesGcm_crypt_sg_inplace_prealloc(src, src_len - WC_AES_BLOCK_SIZE, + ad, ad_len, nonce, aes, 1)); +} + int wc_ecc_make_keypair_exim(u8 *private, const size_t private_len, u8 *public, const size_t public_len, const int curve_id, int compressed) diff --git a/kernel-src/wolfcrypt_glue.h b/kernel-src/wolfcrypt_glue.h index 9b737e6..b339074 100644 --- a/kernel-src/wolfcrypt_glue.h +++ b/kernel-src/wolfcrypt_glue.h @@ -215,6 +215,16 @@ extern bool wc_AesGcm_decrypt_sg_inplace(struct scatterlist *src, size_t src_len const u8 *key, const size_t key_len); +extern bool wc_AesGcm_encrypt_sg_inplace_prealloc(struct scatterlist *src, size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + Aes *aes); + +extern bool wc_AesGcm_decrypt_sg_inplace_prealloc(struct scatterlist *src, size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + Aes *aes); + #ifdef WC_DRBG_BANKREF extern struct wc_rng_bank *wc_wg_drbg;