Page MenuHomeFreeBSD
Files F96888907

D39967.id121526.diff
No OneTemporary
Actions

D39967.id121526.diff
View Options

diff --git a/sys/crypto/openssl/amd64/ossl_aes_gcm.c b/sys/crypto/openssl/amd64/ossl_aes_gcm.c
--- a/sys/crypto/openssl/amd64/ossl_aes_gcm.c
+++ b/sys/crypto/openssl/amd64/ossl_aes_gcm.c
@@ -9,9 +9,11 @@
*/
/*
- * This file contains a AES-GCM wrapper implementation from OpenSSL 3.1,
- * targeting amd64 VAES extensions. This was ported from
- * cipher_aes_gcm_hw_vaes_avx512.inc.
+ * This file contains 2 AES-GCM wrapper implementations from OpenSSL, using
+ * AES-NI and VAES extensions respectively. These were ported from
+ * cipher_aes_gcm_hw_aesni.inc and cipher_aes_gcm_hw_vaes_avx512.inc. The
+ * AES-NI implementation makes use of a generic C implementation for partial
+ * blocks, ported from gcm128.c with OPENSSL_SMALL_FOOTPRINT defined.
*/
#include <sys/endian.h>
@@ -218,6 +220,304 @@
.tag = gcm_tag,
};
+size_t aesni_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len,
+ const void *key, unsigned char ivec[16], uint64_t *Xi);
+size_t aesni_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len,
+ const void *key, unsigned char ivec[16], uint64_t *Xi);
+void aesni_encrypt(const unsigned char *in, unsigned char *out, void *ks);
+
+void gcm_init_avx(__uint128_t Htable[16], uint64_t Xi[2]);
+void gcm_gmult_avx(uint64_t Xi[2], const __uint128_t Htable[16]);
+void gcm_ghash_avx(uint64_t Xi[2], const __uint128_t Htable[16], const void *in,
+ size_t len);
+
+static void
+gcm_init_aesni(struct ossl_gcm_context *ctx, const void *key, size_t keylen)
+{
+ aesni_encrypt(ctx->gcm.H.c, ctx->gcm.H.c, &ctx->aes_ks);
+
+#if BYTE_ORDER == LITTLE_ENDIAN
+ ctx->gcm.H.u[0] = bswap64(ctx->gcm.H.u[0]);
+ ctx->gcm.H.u[1] = bswap64(ctx->gcm.H.u[1]);
+#endif
+
+ gcm_init_avx(ctx->gcm.Htable, ctx->gcm.H.u);
+}
+
+static void
+gcm_setiv_aesni(struct ossl_gcm_context *ctx, const unsigned char *iv,
+ size_t len)
+{
+ uint32_t ctr;
+
+ KASSERT(len == AES_GCM_IV_LEN,
+ ("%s: invalid IV length %zu", __func__, len));
+
+ ctx->gcm.len.u[0] = 0;
+ ctx->gcm.len.u[1] = 0;
+ ctx->gcm.ares = ctx->gcm.mres = 0;
+
+ memcpy(ctx->gcm.Yi.c, iv, len);
+ ctx->gcm.Yi.c[12] = 0;
+ ctx->gcm.Yi.c[13] = 0;
+ ctx->gcm.Yi.c[14] = 0;
+ ctx->gcm.Yi.c[15] = 1;
+ ctr = 1;
+
+ ctx->gcm.Xi.u[0] = 0;
+ ctx->gcm.Xi.u[1] = 0;
+
+ aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EK0.c, &ctx->aes_ks);
+ ctr++;
+
+#if BYTE_ORDER == LITTLE_ENDIAN
+ ctx->gcm.Yi.d[3] = bswap32(ctr);
+#else
+ ctx->gcm.Yi.d[3] = ctr;
+#endif
+}
+
+static int
+gcm_aad_aesni(struct ossl_gcm_context *ctx, const unsigned char *aad,
+ size_t len)
+{
+ size_t i;
+ unsigned int n;
+ uint64_t alen = ctx->gcm.len.u[0];
+
+ if (ctx->gcm.len.u[1])
+ return -2;
+
+ alen += len;
+ if (alen > (1ull << 61) || (sizeof(len) == 8 && alen < len))
+ return -1;
+ ctx->gcm.len.u[0] = alen;
+
+ n = ctx->gcm.ares;
+ if (n) {
+ while (n && len) {
+ ctx->gcm.Xi.c[n] ^= *(aad++);
+ --len;
+ n = (n + 1) % 16;
+ }
+ if (n == 0)
+ gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+ else {
+ ctx->gcm.ares = n;
+ return 0;
+ }
+ }
+ if ((i = (len & (size_t)-AES_BLOCK_LEN))) {
+ gcm_ghash_avx(ctx->gcm.Xi.u, ctx->gcm.Htable, aad, i);
+ aad += i;
+ len -= i;
+ }
+ if (len) {
+ n = (unsigned int)len;
+ for (i = 0; i < len; ++i)
+ ctx->gcm.Xi.c[i] ^= aad[i];
+ }
+
+ ctx->gcm.ares = n;
+ return 0;
+}
+
+static int
+gcm_encrypt(struct ossl_gcm_context *ctx, const unsigned char *in,
+ unsigned char *out, size_t len)
+{
+ unsigned int n, ctr, mres;
+ size_t i;
+ uint64_t mlen = ctx->gcm.len.u[1];
+
+ mlen += len;
+ if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len))
+ return -1;
+ ctx->gcm.len.u[1] = mlen;
+
+ mres = ctx->gcm.mres;
+
+ if (ctx->gcm.ares) {
+ /* First call to encrypt finalizes GHASH(AAD) */
+ gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+ ctx->gcm.ares = 0;
+ }
+
+#if BYTE_ORDER == LITTLE_ENDIAN
+ ctr = bswap32(ctx->gcm.Yi.d[3]);
+#else
+ ctr = ctx->gcm.Yi.d[3];
+#endif
+
+ n = mres % 16;
+ for (i = 0; i < len; ++i) {
+ if (n == 0) {
+ aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c,
+ &ctx->aes_ks);
+ ++ctr;
+#if BYTE_ORDER == LITTLE_ENDIAN
+ ctx->gcm.Yi.d[3] = bswap32(ctr);
+#else
+ ctx->gcm.Yi.d[3] = ctr;
+#endif
+ }
+ ctx->gcm.Xi.c[n] ^= out[i] = in[i] ^ ctx->gcm.EKi.c[n];
+ mres = n = (n + 1) % 16;
+ if (n == 0)
+ gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+ }
+
+ ctx->gcm.mres = mres;
+ return 0;
+}
+
+static int
+gcm_decrypt(struct ossl_gcm_context *ctx, const unsigned char *in,
+ unsigned char *out, size_t len)
+{
+ unsigned int n, ctr, mres;
+ size_t i;
+ uint64_t mlen = ctx->gcm.len.u[1];
+
+ mlen += len;
+ if (mlen > ((1ull << 36) - 32) || (sizeof(len) == 8 && mlen < len))
+ return -1;
+ ctx->gcm.len.u[1] = mlen;
+
+ mres = ctx->gcm.mres;
+
+ if (ctx->gcm.ares) {
+ /* First call to encrypt finalizes GHASH(AAD) */
+ gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+ ctx->gcm.ares = 0;
+ }
+
+#if BYTE_ORDER == LITTLE_ENDIAN
+ ctr = bswap32(ctx->gcm.Yi.d[3]);
+#else
+ ctr = ctx->gcm.Yi.d[3];
+#endif
+
+ n = mres % 16;
+ for (i = 0; i < len; ++i) {
+ uint8_t c;
+ if (n == 0) {
+ aesni_encrypt(ctx->gcm.Yi.c, ctx->gcm.EKi.c,
+ &ctx->aes_ks);
+ ++ctr;
+#if BYTE_ORDER == LITTLE_ENDIAN
+ ctx->gcm.Yi.d[3] = bswap32(ctr);
+#else
+ ctx->gcm.Yi.d[3] = ctr;
+#endif
+ }
+ c = in[i];
+ out[i] = c ^ ctx->gcm.EKi.c[n];
+ ctx->gcm.Xi.c[n] ^= c;
+ mres = n = (n + 1) % 16;
+ if (n == 0)
+ gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+ }
+
+ ctx->gcm.mres = mres;
+ return 0;
+}
+
+static int
+gcm_encrypt_aesni(struct ossl_gcm_context *ctx, const unsigned char *in,
+ unsigned char *out, size_t len)
+{
+ size_t bulk = 0, res;
+ int error;
+
+ res = (AES_BLOCK_LEN - ctx->gcm.mres) % AES_BLOCK_LEN;
+ if ((error = gcm_encrypt(ctx, in, out, res)) != 0)
+ return error;
+
+ bulk = aesni_gcm_encrypt(in + res, out + res, len - res,
+ &ctx->aes_ks, ctx->gcm.Yi.c, ctx->gcm.Xi.u);
+ ctx->gcm.len.u[1] += bulk;
+ bulk += res;
+
+ if ((error = gcm_encrypt(ctx, in + bulk, out + bulk, len - bulk)) != 0)
+ return error;
+
+ return 0;
+}
+
+static int
+gcm_decrypt_aesni(struct ossl_gcm_context *ctx, const unsigned char *in,
+ unsigned char *out, size_t len)
+{
+ size_t bulk = 0, res;
+ int error;
+
+ res = (AES_BLOCK_LEN - ctx->gcm.mres) % AES_BLOCK_LEN;
+ if ((error = gcm_decrypt(ctx, in, out, res)) != 0)
+ return error;
+
+ bulk = aesni_gcm_decrypt(in, out, len, &ctx->aes_ks, ctx->gcm.Yi.c,
+ ctx->gcm.Xi.u);
+ ctx->gcm.len.u[1] += bulk;
+ bulk += res;
+
+ if ((error = gcm_decrypt(ctx, in + bulk, out + bulk, len - bulk)) != 0)
+ return error;
+
+ return 0;
+}
+
+static int
+gcm_finish_aesni(struct ossl_gcm_context *ctx, const unsigned char *tag,
+ size_t len)
+{
+ uint64_t alen = ctx->gcm.len.u[0] << 3;
+ uint64_t clen = ctx->gcm.len.u[1] << 3;
+
+ if (ctx->gcm.mres || ctx->gcm.ares)
+ gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+
+#if BYTE_ORDER == LITTLE_ENDIAN
+ alen = bswap64(alen);
+ clen = bswap64(clen);
+#endif
+
+ ctx->gcm.Xi.u[0] ^= alen;
+ ctx->gcm.Xi.u[1] ^= clen;
+ gcm_gmult_avx(ctx->gcm.Xi.u, ctx->gcm.Htable);
+
+ ctx->gcm.Xi.u[0] ^= ctx->gcm.EK0.u[0];
+ ctx->gcm.Xi.u[1] ^= ctx->gcm.EK0.u[1];
+
+ if (tag != NULL)
+ return timingsafe_bcmp(ctx->gcm.Xi.c, tag, len);
+ return 0;
+}
+
+static const struct ossl_aes_gcm_ops gcm_ops_aesni = {
+ .init = gcm_init_aesni,
+ .setiv = gcm_setiv_aesni,
+ .aad = gcm_aad_aesni,
+ .encrypt = gcm_encrypt_aesni,
+ .decrypt = gcm_decrypt_aesni,
+ .finish = gcm_finish_aesni,
+ .tag = gcm_tag,
+};
+
+int ossl_aes_gcm_setkey_aesni(const unsigned char *key, int klen, void *_ctx);
+
+int
+ossl_aes_gcm_setkey_aesni(const unsigned char *key, int klen,
+ void *_ctx)
+{
+ struct ossl_gcm_context *ctx;
+
+ ctx = _ctx;
+ ctx->ops = &gcm_ops_aesni;
+ gcm_init(ctx, key, klen);
+ return (0);
+}
+
int ossl_aes_gcm_setkey_avx512(const unsigned char *key, int klen, void *_ctx);
int
diff --git a/sys/crypto/openssl/ossl_x86.c b/sys/crypto/openssl/ossl_x86.c
--- a/sys/crypto/openssl/ossl_x86.c
+++ b/sys/crypto/openssl/ossl_x86.c
@@ -58,6 +58,7 @@
#ifdef __amd64__
int ossl_vaes_vpclmulqdq_capable(void);
+ossl_cipher_setkey_t ossl_aes_gcm_setkey_aesni;
ossl_cipher_setkey_t ossl_aes_gcm_setkey_avx512;
#endif
@@ -139,6 +140,12 @@
ossl_cipher_aes_gcm.set_decrypt_key =
ossl_aes_gcm_setkey_avx512;
sc->has_aes_gcm = true;
+ } else if ((cpu_feature2 &
+ (CPUID2_AVX | CPUID2_PCLMULQDQ | CPUID2_MOVBE)) ==
+ (CPUID2_AVX | CPUID2_PCLMULQDQ | CPUID2_MOVBE)) {
+ ossl_cipher_aes_gcm.set_encrypt_key = ossl_aes_gcm_setkey_aesni;
+ ossl_cipher_aes_gcm.set_decrypt_key = ossl_aes_gcm_setkey_aesni;
+ sc->has_aes_gcm = true;
} else {
sc->has_aes_gcm = false;
}
diff --git a/sys/modules/ossl/Makefile b/sys/modules/ossl/Makefile
--- a/sys/modules/ossl/Makefile
+++ b/sys/modules/ossl/Makefile
@@ -29,7 +29,9 @@
SRCS.amd64= \
aes-gcm-avx512.S \
aesni-x86_64.S \
+ aesni-gcm-x86_64.S \
chacha-x86_64.S \
+ ghash-x86_64.S \
poly1305-x86_64.S \
sha1-x86_64.S \
sha256-x86_64.S \

File Metadata

Mime Type
text/plain
Expires
Sat, Sep 28, 2:23 AM (11 h, 1 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
12965272
Default Alt Text
D39967.id121526.diff (8 KB)

Event Timeline