Skip to content

PPC64 Assembly: ELF V1 headers added. Little-endian asm added. #10871

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dgarske merged 1 commit into from
Jul 10, 2026
+20,314 −230
Merged

PPC64 Assembly: ELF V1 headers added. Little-endian asm added. #10871

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .wolfssl_known_macro_extras
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1084,6 +1084,7 @@ __INTEL_COMPILER
__KEIL__
__KEY_DATA_H__
__LINUX__
__LITTLE_ENDIAN__
__LP64
__LP64__
__MACH__
Expand All @@ -1098,6 +1099,7 @@ __OS2__
__OpenBSD__
__PIC__
__PIE__
__POWER9_VECTOR__
__POWERPC__
__PPC__
__PPU
Expand Down
115 changes: 97 additions & 18 deletions wolfcrypt/src/aes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -749,17 +749,16 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits
#define AESNI_ALIGN 16
#endif

/* note that all write access to these static variables must be idempotent,
* as arranged by Check_CPU_support_AES(), else they will be susceptible to
* data races.
*/
static int checkedAESNI = 0;
static int haveAESNI = 0;
static cpuid_flags_t intel_flags = WC_CPUID_INITIALIZER;
/* Accessed with the wolfSSL atomic APIs so the one-time detection is free of
* data races. Writes are also idempotent (all callers compute the same
* value), so a benign concurrent double-write is harmless. */
static wolfSSL_Atomic_Uint checkedAESNI = WOLFSSL_ATOMIC_INITIALIZER(0);
static wolfSSL_Atomic_Uint haveAESNI = WOLFSSL_ATOMIC_INITIALIZER(0);
static cpuid_flags_atomic_t intel_flags = WC_CPUID_ATOMIC_INITIALIZER;

static WARN_UNUSED_RESULT int Check_CPU_support_AES(void)
{
cpuid_get_flags_ex(&intel_flags);
cpuid_get_flags_atomic(&intel_flags);

return IS_INTEL_AESNI(intel_flags) != 0;
}
Expand Down Expand Up @@ -1163,11 +1162,11 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits

#elif defined(WOLFSSL_ARMASM)
#if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER;
static cpuid_flags_atomic_t cpuid_flags = WC_CPUID_ATOMIC_INITIALIZER;

static void Check_CPU_support_HwCrypto(Aes* aes)
{
cpuid_get_flags_ex(&cpuid_flags);
cpuid_get_flags_atomic(&cpuid_flags);
aes->use_aes_hw_crypto = IS_AARCH64_AES(cpuid_flags);
#ifdef HAVE_AESGCM
aes->use_pmull_hw_crypto = IS_AARCH64_PMULL(cpuid_flags);
Expand Down Expand Up @@ -1242,6 +1241,83 @@ static WARN_UNUSED_RESULT int wc_AesDecrypt(Aes* aes, const byte* inBlock,

#elif (defined(WOLFSSL_PPC64_ASM) || defined(WOLFSSL_PPC32_ASM))

#if defined(WOLFSSL_PPC64_ASM) && defined(WOLFSSL_PPC64_ASM_CRYPTO)
/* POWER8+ has vector AES (vcipher/vncipher...) instructions. When built in,
* select the "_crypto" implementations at run time if the CPU supports them.
*
* A run-time flag with direct calls is used rather than a function pointer: an
* indirect call would require an ELFv1 function descriptor, whereas direct
* calls work under both the ELFv1 and ELFv2 ABIs. The dispatch is expressed as
* self-referential macros - the base name inside each macro is not re-expanded
* (C99 6.10.3.4), so it names the real base function. In a PPC build the ARM
* branches that also call these names are #if'd out, so only the live PPC call
* sites are redirected. */

/* Resolved dispatch decision (0 = base, 1 = vector-crypto), accessed with the
* wolfSSL atomic APIs so the one-time detection is free of data races. The
* master cpuid flags read by cpuid_get_flags() are themselves atomic; the write
* here is idempotent so a benign concurrent double-write is harmless. */
static wolfSSL_Atomic_Uint aes_ppc64_use_crypto = WOLFSSL_ATOMIC_INITIALIZER(0);

/* True when the CPU supports the vector-crypto instructions. */
#define AES_PPC64_USE_CRYPTO() (WOLFSSL_ATOMIC_LOAD(aes_ppc64_use_crypto) != 0)

/* Check and set the decision together (as Check_CPU_support_AES/HwCrypto do);
* called from the key-setup path before any AES_*_crypto use. */
static void Aes_SetCrypto(void)
{
WOLFSSL_ATOMIC_STORE(aes_ppc64_use_crypto,
(unsigned int)(IS_PPC64_VEC_CRYPTO(cpuid_get_flags()) != 0));
}

#define AES_set_encrypt_key(key, len, ks) \
(AES_PPC64_USE_CRYPTO() ? \
AES_set_encrypt_key_crypto((key), (len), (ks)) : \
AES_set_encrypt_key((key), (len), (ks)))
#define AES_invert_key(ks, rounds) \
(AES_PPC64_USE_CRYPTO() ? \
AES_invert_key_crypto((ks), (rounds)) : \
AES_invert_key((ks), (rounds)))
#define AES_ECB_encrypt(in, out, len, ks, nr) \
(AES_PPC64_USE_CRYPTO() ? \
AES_ECB_encrypt_crypto((in), (out), (len), (ks), (nr)) : \
AES_ECB_encrypt((in), (out), (len), (ks), (nr)))
#define AES_ECB_decrypt(in, out, len, ks, nr) \
(AES_PPC64_USE_CRYPTO() ? \
AES_ECB_decrypt_crypto((in), (out), (len), (ks), (nr)) : \
AES_ECB_decrypt((in), (out), (len), (ks), (nr)))
#define AES_CBC_encrypt(in, out, len, ks, nr, iv) \
(AES_PPC64_USE_CRYPTO() ? \
AES_CBC_encrypt_crypto((in), (out), (len), (ks), (nr), (iv)) : \
AES_CBC_encrypt((in), (out), (len), (ks), (nr), (iv)))
#define AES_CBC_decrypt(in, out, len, ks, nr, iv) \
(AES_PPC64_USE_CRYPTO() ? \
AES_CBC_decrypt_crypto((in), (out), (len), (ks), (nr), (iv)) : \
AES_CBC_decrypt((in), (out), (len), (ks), (nr), (iv)))
#define AES_CTR_encrypt(in, out, len, ks, nr, ctr) \
(AES_PPC64_USE_CRYPTO() ? \
AES_CTR_encrypt_crypto((in), (out), (len), (ks), (nr), (ctr)) : \
AES_CTR_encrypt((in), (out), (len), (ks), (nr), (ctr)))
#define AES_GCM_encrypt(in, out, len, ks, nr, ctr) \
(AES_PPC64_USE_CRYPTO() ? \
AES_GCM_encrypt_crypto((in), (out), (len), (ks), (nr), (ctr)) : \
AES_GCM_encrypt((in), (out), (len), (ks), (nr), (ctr)))
#if defined(WOLFSSL_AES_XTS)
#define AES_XTS_encrypt(in, out, sz, i, key, key2, tmp, nr) \
(AES_PPC64_USE_CRYPTO() ? \
AES_XTS_encrypt_crypto((in), (out), (sz), (i), (key), (key2), (tmp), \
(nr)) : \
AES_XTS_encrypt((in), (out), (sz), (i), (key), (key2), (tmp), (nr)))
#define AES_XTS_decrypt(in, out, sz, i, key, key2, tmp, nr) \
(AES_PPC64_USE_CRYPTO() ? \
AES_XTS_decrypt_crypto((in), (out), (sz), (i), (key), (key2), (tmp), \
(nr)) : \
AES_XTS_decrypt((in), (out), (sz), (i), (key), (key2), (tmp), (nr)))
#endif /* WOLFSSL_AES_XTS */
#else
#define Aes_SetCrypto() WC_DO_NOTHING
#endif /* WOLFSSL_PPC64_ASM && WOLFSSL_PPC64_ASM_CRYPTO */

#if defined(WOLFSSL_AES_DIRECT) || defined(HAVE_AESCCM) || \
defined(WOLFSSL_AESGCM_STREAM) || defined(HAVE_AESGCM)
static WARN_UNUSED_RESULT int wc_AesEncrypt(Aes* aes, const byte* inBlock,
Expand Down Expand Up @@ -4955,6 +5031,9 @@ static WARN_UNUSED_RESULT int wc_AesDecrypt(Aes* aes, const byte* inBlock,
aes->keylen = (int)keylen;
aes->rounds = (keylen/4) + 6;

/* Determine base vs vector-crypto before the (dispatched) key setup so
* the schedule matches the mode functions that later consume it. */
Aes_SetCrypto();
AES_set_encrypt_key(userKey, keylen * 8, (byte*)aes->key);

#ifdef HAVE_AES_DECRYPT
Expand Down Expand Up @@ -5637,11 +5716,11 @@ static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir)
* function correctly with default build settings.
*/

if (checkedAESNI == 0) {
haveAESNI = Check_CPU_support_AES();
checkedAESNI = 1;
if (WOLFSSL_ATOMIC_LOAD(checkedAESNI) == 0) {
WOLFSSL_ATOMIC_STORE(haveAESNI, Check_CPU_support_AES());
WOLFSSL_ATOMIC_STORE(checkedAESNI, 1);
}
if (haveAESNI
if (WOLFSSL_ATOMIC_LOAD(haveAESNI)
#if defined(WC_FLAG_DONT_USE_VECTOR_OPS) && !defined(WC_C_DYNAMIC_FALLBACK)
&& (aes->use_aesni != WC_FLAG_DONT_USE_VECTOR_OPS)
#endif
Expand Down Expand Up @@ -18887,7 +18966,7 @@ static AesGcmSivPolyvalFn AesGcmSivPolyvalAsm(void)
static AesGcmSivPolyvalFn AesGcmSivPolyvalAsm(void)
{
#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
cpuid_get_flags_ex(&cpuid_flags);
cpuid_get_flags_atomic(&cpuid_flags);
if (IS_AARCH64_PMULL(cpuid_flags)) {
return &AES_GCMSIV_polyval_pmull;
}
Expand All @@ -18906,7 +18985,7 @@ static AesGcmSivPolyvalFn AesGcmSivPolyvalAsm(void)
* AES-NI gates the base path (matching wolfSSL's AES-GCM). */
static AesGcmSivPolyvalFn AesGcmSivPolyvalAsm(void)
{
cpuid_get_flags_ex(&intel_flags);
cpuid_get_flags_atomic(&intel_flags);
if (!IS_INTEL_AESNI(intel_flags)) {
return NULL;
}
Expand Down Expand Up @@ -18951,7 +19030,7 @@ static AesGcmSivCtrFn AesGcmSivCtrAsm(void)
static AesGcmSivCtrFn AesGcmSivCtrAsm(void)
{
#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
cpuid_get_flags_ex(&cpuid_flags);
cpuid_get_flags_atomic(&cpuid_flags);
if (IS_AARCH64_AES(cpuid_flags)) {
return &AES_GCMSIV_ctr_aarch64;
}
Expand All @@ -18969,7 +19048,7 @@ static AesGcmSivCtrFn AesGcmSivCtrAsm(void)
* the base; AVX1/VAES/AVX512 are progressively wider pipelines. */
static AesGcmSivCtrFn AesGcmSivCtrAsm(void)
{
cpuid_get_flags_ex(&intel_flags);
cpuid_get_flags_atomic(&intel_flags);
if (!IS_INTEL_AESNI(intel_flags)) {
return NULL;
}
Expand Down
Loading
Loading