Merge bitcoin/bitcoin#24115: ARMv8 SHA2 Intrinsics

aaa1d03d3acebeb44fdd40a302f086aad3d329ce Add optimized sha256d64_arm_shani::Transform_2way (Pieter Wuille)
fe0629852aaf3a26f291bfa535e7e455fe7bea06 Implement sha256_arm_shani::Transform (Pavol Rusnak)
48a72fa81f80c8a3c7c6de8339b5feb361dece1c Add sha256_arm_shani to build system (Pavol Rusnak)
c2b79342506e24e9b7100fb7a6025dc870375ef6 Rename SHANI to X86_SHANI to allow future implementation of ARM_SHANI (Pavol Rusnak)

Pull request description:

  This PR adds support for ARMv8 SHA2 Intrinsics.

  Fixes https://github.com/bitcoin/bitcoin/issues/13401 and https://github.com/bitcoin/bitcoin/issues/17414

  * Integration part was done by me.
  * The original SHA2 NI code comes from https://github.com/noloader/SHA-Intrinsics/blob/master/sha256-arm.c
  * Minor optimizations from https://github.com/rollmeister/bitcoin-armv8/blob/master/src/crypto/sha256.cpp are applied too.
  * The 2-way transform added by @sipa

ACKs for top commit:
  laanwj:
    Code review and lightly tested ACK aaa1d03d3acebeb44fdd40a302f086aad3d329ce

Tree-SHA512: 9689d6390c004269cb1ee79ed05430d7d35a6efef2554a2b6732f7258a11e7e959b3306c04b4e8637a9623fb4c12d1c1b3592da0ff0dc6d737932db302509669

# Conflicts:
#	configure.ac
#	src/Makefile.am
#	src/crypto/sha256.cpp
This commit is contained in:
laanwj 2022-02-14 20:47:24 +01:00 committed by pasta
parent 15aceb9ec4
commit a7a3a93489
5 changed files with 1015 additions and 33 deletions

View File

@ -462,7 +462,7 @@ AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers],[[CXXFLAGS="$CXXFLAGS -fno-ext
enable_sse42=no enable_sse42=no
enable_sse41=no enable_sse41=no
enable_avx2=no enable_avx2=no
enable_shani=no enable_x86_shani=no
if test "x$use_asm" = "xyes"; then if test "x$use_asm" = "xyes"; then
@ -471,10 +471,10 @@ dnl be compiled with them, rather that specific objects/libs may use them after
dnl compatibility. dnl compatibility.
dnl x86 dnl x86
AX_CHECK_COMPILE_FLAG([-msse4.2],[[SSE42_CXXFLAGS="-msse4.2"]],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-msse4.2], [SSE42_CXXFLAGS="-msse4.2"], [$CXXFLAG_WERROR])
AX_CHECK_COMPILE_FLAG([-msse4.1],[[SSE41_CXXFLAGS="-msse4.1"]],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-msse4.1], [SSE41_CXXFLAGS="-msse4.1"], [$CXXFLAG_WERROR])
AX_CHECK_COMPILE_FLAG([-mavx -mavx2],[[AVX2_CXXFLAGS="-mavx -mavx2"]],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-mavx -mavx2], [AVX2_CXXFLAGS="-mavx -mavx2"], [$CXXFLAG_WERROR])
AX_CHECK_COMPILE_FLAG([-msse4 -msha],[[SHANI_CXXFLAGS="-msse4 -msha"]],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-msse4 -msha], [X86_SHANI_CXXFLAGS="-msse4 -msha"], [$CXXFLAG_WERROR])
TEMP_CXXFLAGS="$CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS"
CXXFLAGS="$CXXFLAGS $SSE42_CXXFLAGS" CXXFLAGS="$CXXFLAGS $SSE42_CXXFLAGS"
@ -529,8 +529,8 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
CXXFLAGS="$TEMP_CXXFLAGS" CXXFLAGS="$TEMP_CXXFLAGS"
TEMP_CXXFLAGS="$CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS"
CXXFLAGS="$CXXFLAGS $SHANI_CXXFLAGS" CXXFLAGS="$CXXFLAGS $X86_SHANI_CXXFLAGS"
AC_MSG_CHECKING(for SHA-NI intrinsics) AC_MSG_CHECKING(for x86 SHA-NI intrinsics)
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
#include <stdint.h> #include <stdint.h>
#include <immintrin.h> #include <immintrin.h>
@ -540,17 +540,18 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
__m128i k = _mm_set1_epi32(2); __m128i k = _mm_set1_epi32(2);
return _mm_extract_epi32(_mm_sha256rnds2_epu32(i, i, k), 0); return _mm_extract_epi32(_mm_sha256rnds2_epu32(i, i, k), 0);
]])], ]])],
[ AC_MSG_RESULT(yes); enable_shani=yes; AC_DEFINE(ENABLE_SHANI, 1, [Define this symbol to build code that uses SHA-NI intrinsics]) ], [ AC_MSG_RESULT(yes); enable_x86_shani=yes; AC_DEFINE(ENABLE_X86_SHANI, 1, [Define this symbol to build code that uses x86 SHA-NI intrinsics]) ],
[ AC_MSG_RESULT(no)] [ AC_MSG_RESULT(no)]
) )
CXXFLAGS="$TEMP_CXXFLAGS" CXXFLAGS="$TEMP_CXXFLAGS"
# ARM # ARM
AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto],[[ARM_CRC_CXXFLAGS="-march=armv8-a+crc+crypto"]],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto],[[ARM_CRC_CXXFLAGS="-march=armv8-a+crc+crypto"]],,[[$CXXFLAG_WERROR]])
AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_SHANI_CXXFLAGS="-march=armv8-a+crc+crypto"], [], [$CXXFLAG_WERROR])
TEMP_CXXFLAGS="$CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS"
CXXFLAGS="$CXXFLAGS $ARM_CRC_CXXFLAGS" CXXFLAGS="$CXXFLAGS $ARM_CRC_CXXFLAGS"
AC_MSG_CHECKING(for AArch64 CRC32 intrinsics) AC_MSG_CHECKING(for ARMv8 CRC32 intrinsics)
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
#include <arm_acle.h> #include <arm_acle.h>
#include <arm_neon.h> #include <arm_neon.h>
@ -567,6 +568,24 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
) )
CXXFLAGS="$TEMP_CXXFLAGS" CXXFLAGS="$TEMP_CXXFLAGS"
TEMP_CXXFLAGS="$CXXFLAGS"
CXXFLAGS="$CXXFLAGS $ARM_SHANI_CXXFLAGS"
AC_MSG_CHECKING([for ARMv8 SHA-NI intrinsics])
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
#include <arm_acle.h>
#include <arm_neon.h>
]],[[
uint32x4_t a, b, c;
vsha256h2q_u32(a, b, c);
vsha256hq_u32(a, b, c);
vsha256su0q_u32(a, b);
vsha256su1q_u32(a, b, c);
]])],
[ AC_MSG_RESULT([yes]); enable_arm_shani=yes; AC_DEFINE([ENABLE_ARM_SHANI], [1], [Define this symbol to build code that uses ARMv8 SHA-NI intrinsics]) ],
[ AC_MSG_RESULT([no])]
)
CXXFLAGS="$TEMP_CXXFLAGS"
fi fi
CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS"
@ -1790,8 +1809,9 @@ AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes])
AM_CONDITIONAL([ENABLE_SSE42],[test x$enable_sse42 = xyes]) AM_CONDITIONAL([ENABLE_SSE42],[test x$enable_sse42 = xyes])
AM_CONDITIONAL([ENABLE_SSE41],[test x$enable_sse41 = xyes]) AM_CONDITIONAL([ENABLE_SSE41],[test x$enable_sse41 = xyes])
AM_CONDITIONAL([ENABLE_AVX2],[test x$enable_avx2 = xyes]) AM_CONDITIONAL([ENABLE_AVX2],[test x$enable_avx2 = xyes])
AM_CONDITIONAL([ENABLE_SHANI],[test x$enable_shani = xyes]) AM_CONDITIONAL([ENABLE_X86_SHANI],[test $enable_x86_shani = xyes])
AM_CONDITIONAL([ENABLE_ARM_CRC],[test x$enable_arm_crc = xyes]) AM_CONDITIONAL([ENABLE_ARM_CRC],[test x$enable_arm_crc = xyes])
AM_CONDITIONAL([ENABLE_ARM_SHANI], [test "$enable_arm_shani" = "yes"])
AM_CONDITIONAL([USE_ASM],[test x$use_asm = xyes]) AM_CONDITIONAL([USE_ASM],[test x$use_asm = xyes])
AM_CONDITIONAL([WORDS_BIGENDIAN],[test x$ac_cv_c_bigendian = xyes]) AM_CONDITIONAL([WORDS_BIGENDIAN],[test x$ac_cv_c_bigendian = xyes])
@ -1838,8 +1858,9 @@ AC_SUBST(SANITIZER_LDFLAGS)
AC_SUBST(SSE42_CXXFLAGS) AC_SUBST(SSE42_CXXFLAGS)
AC_SUBST(SSE41_CXXFLAGS) AC_SUBST(SSE41_CXXFLAGS)
AC_SUBST(AVX2_CXXFLAGS) AC_SUBST(AVX2_CXXFLAGS)
AC_SUBST(SHANI_CXXFLAGS) AC_SUBST(X86_SHANI_CXXFLAGS)
AC_SUBST(ARM_CRC_CXXFLAGS) AC_SUBST(ARM_CRC_CXXFLAGS)
AC_SUBST(ARM_SHANI_CXXFLAGS)
AC_SUBST(LIBTOOL_APP_LDFLAGS) AC_SUBST(LIBTOOL_APP_LDFLAGS)
AC_SUBST(USE_UPNP) AC_SUBST(USE_UPNP)
AC_SUBST(USE_QRCODE) AC_SUBST(USE_QRCODE)

View File

@ -76,9 +76,13 @@ if ENABLE_AVX2
LIBBITCOIN_CRYPTO_AVX2 = crypto/libdash_crypto_avx2.a LIBBITCOIN_CRYPTO_AVX2 = crypto/libdash_crypto_avx2.a
LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_AVX2) LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_AVX2)
endif endif
if ENABLE_SHANI if ENABLE_X86_SHANI
LIBBITCOIN_CRYPTO_SHANI = crypto/libdash_crypto_shani.a LIBBITCOIN_CRYPTO_X86_SHANI = crypto/libdash_crypto_x86_shani.a
LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SHANI) LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_X86_SHANI)
endif
if ENABLE_ARM_SHANI
LIBBITCOIN_CRYPTO_ARM_SHANI = crypto/libdash_crypto_arm_shani.a
LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_ARM_SHANI)
endif endif
$(LIBSECP256K1): $(wildcard secp256k1/src/*.h) $(wildcard secp256k1/src/*.c) $(wildcard secp256k1/include/*) $(LIBSECP256K1): $(wildcard secp256k1/src/*.h) $(wildcard secp256k1/src/*.c) $(wildcard secp256k1/include/*)
@ -575,11 +579,17 @@ crypto_libdash_crypto_base_a_SOURCES += \
crypto/sph_skein.h \ crypto/sph_skein.h \
crypto/sph_types.h crypto/sph_types.h
crypto_libdash_crypto_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) crypto_libdash_crypto_x86_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
crypto_libdash_crypto_shani_a_CPPFLAGS = $(AM_CPPFLAGS) crypto_libdash_crypto_x86_shani_a_CPPFLAGS = $(AM_CPPFLAGS)
crypto_libdash_crypto_shani_a_CXXFLAGS += $(SHANI_CXXFLAGS) crypto_libdash_crypto_x86_shani_a_CXXFLAGS += $(X86_SHANI_CXXFLAGS)
crypto_libdash_crypto_shani_a_CPPFLAGS += -DENABLE_SHANI crypto_libdash_crypto_x86_shani_a_CPPFLAGS += -DENABLE_X86_SHANI
crypto_libdash_crypto_shani_a_SOURCES = crypto/sha256_shani.cpp crypto_libdash_crypto_x86_shani_a_SOURCES = crypto/sha256_x86_shani.cpp
crypto_libdash_crypto_arm_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
crypto_libdash_crypto_arm_shani_a_CPPFLAGS = $(AM_CPPFLAGS)
crypto_libdash_crypto_arm_shani_a_CXXFLAGS += $(ARM_SHANI_CXXFLAGS)
crypto_libdash_crypto_arm_shani_a_CPPFLAGS += -DENABLE_ARM_SHANI
crypto_libdash_crypto_arm_shani_a_SOURCES = crypto/sha256_arm_shani.cpp
# consensus: shared between all executables that validate any consensus rules. # consensus: shared between all executables that validate any consensus rules.
libdash_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libdash_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES)

View File

@ -8,6 +8,16 @@
#include <assert.h> #include <assert.h>
#include <string.h> #include <string.h>
#if defined(__linux__) && defined(ENABLE_ARM_SHANI) && !defined(BUILD_BITCOIN_INTERNAL)
#include <sys/auxv.h>
#include <asm/hwcap.h>
#endif
#if defined(MAC_OSX) && defined(ENABLE_ARM_SHANI) && !defined(BUILD_BITCOIN_INTERNAL)
#include <sys/types.h>
#include <sys/sysctl.h>
#endif
#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__)
#if defined(USE_ASM) #if defined(USE_ASM)
#include <cpuid.h> #include <cpuid.h>
@ -28,16 +38,26 @@ namespace sha256d64_avx2
void Transform_8way(unsigned char* out, const unsigned char* in); void Transform_8way(unsigned char* out, const unsigned char* in);
} }
namespace sha256d64_shani namespace sha256d64_x86_shani
{ {
void Transform_2way(unsigned char* out, const unsigned char* in); void Transform_2way(unsigned char* out, const unsigned char* in);
} }
namespace sha256_shani namespace sha256_x86_shani
{ {
void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks); void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks);
} }
namespace sha256_arm_shani
{
void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks);
}
namespace sha256d64_arm_shani
{
void Transform_2way(unsigned char* out, const unsigned char* in);
}
// Internal implementation code. // Internal implementation code.
namespace namespace
{ {
@ -577,7 +597,7 @@ std::string SHA256AutoDetect()
bool have_xsave = false; bool have_xsave = false;
bool have_avx = false; bool have_avx = false;
bool have_avx2 = false; bool have_avx2 = false;
bool have_shani = false; bool have_x86_shani = false;
bool enabled_avx = false; bool enabled_avx = false;
(void)AVXEnabled; (void)AVXEnabled;
@ -585,7 +605,7 @@ std::string SHA256AutoDetect()
(void)have_avx; (void)have_avx;
(void)have_xsave; (void)have_xsave;
(void)have_avx2; (void)have_avx2;
(void)have_shani; (void)have_x86_shani;
(void)enabled_avx; (void)enabled_avx;
uint32_t eax, ebx, ecx, edx; uint32_t eax, ebx, ecx, edx;
@ -599,15 +619,15 @@ std::string SHA256AutoDetect()
if (have_sse4) { if (have_sse4) {
cpuid(7, 0, eax, ebx, ecx, edx); cpuid(7, 0, eax, ebx, ecx, edx);
have_avx2 = (ebx >> 5) & 1; have_avx2 = (ebx >> 5) & 1;
have_shani = (ebx >> 29) & 1; have_x86_shani = (ebx >> 29) & 1;
} }
#if defined(ENABLE_SHANI) && !defined(BUILD_BITCOIN_INTERNAL) #if defined(ENABLE_X86_SHANI) && !defined(BUILD_BITCOIN_INTERNAL)
if (have_shani) { if (have_x86_shani) {
Transform = sha256_shani::Transform; Transform = sha256_x86_shani::Transform;
TransformD64 = TransformD64Wrapper<sha256_shani::Transform>; TransformD64 = TransformD64Wrapper<sha256_x86_shani::Transform>;
TransformD64_2way = sha256d64_shani::Transform_2way; TransformD64_2way = sha256d64_x86_shani::Transform_2way;
ret = "shani(1way,2way)"; ret = "x86_shani(1way,2way)";
have_sse4 = false; // Disable SSE4/AVX2; have_sse4 = false; // Disable SSE4/AVX2;
have_avx2 = false; have_avx2 = false;
} }
@ -633,6 +653,38 @@ std::string SHA256AutoDetect()
#endif #endif
#endif #endif
#if defined(ENABLE_ARM_SHANI) && !defined(BUILD_BITCOIN_INTERNAL)
bool have_arm_shani = false;
#if defined(__linux__)
#if defined(__arm__) // 32-bit
if (getauxval(AT_HWCAP2) & HWCAP2_SHA2) {
have_arm_shani = true;
}
#endif
#if defined(__aarch64__) // 64-bit
if (getauxval(AT_HWCAP) & HWCAP_SHA2) {
have_arm_shani = true;
}
#endif
#endif
#if defined(MAC_OSX)
int val = 0;
size_t len = sizeof(val);
if (sysctlbyname("hw.optional.arm.FEAT_SHA256", &val, &len, nullptr, 0) == 0) {
have_arm_shani = val != 0;
}
#endif
if (have_arm_shani) {
Transform = sha256_arm_shani::Transform;
TransformD64 = TransformD64Wrapper<sha256_arm_shani::Transform>;
TransformD64_2way = sha256d64_arm_shani::Transform_2way;
ret = "arm_shani(1way,2way)";
}
#endif
assert(SelfTest()); assert(SelfTest());
return ret; return ret;
} }

View File

@ -0,0 +1,899 @@
// Copyright (c) 2022 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
//
// Based on https://github.com/noloader/SHA-Intrinsics/blob/master/sha256-arm.c,
// Written and placed in public domain by Jeffrey Walton.
// Based on code from ARM, and by Johannes Schneiders, Skip Hovsmith and
// Barry O'Rourke for the mbedTLS project.
// Variant specialized for 64-byte inputs added by Pieter Wuille.
#ifdef ENABLE_ARM_SHANI
#include <array>
#include <cstdint>
#include <cstddef>
#include <arm_acle.h>
#include <arm_neon.h>
namespace {
alignas(uint32x4_t) static constexpr std::array<uint32_t, 64> K =
{
0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5,
0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5,
0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3,
0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174,
0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC,
0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA,
0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7,
0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967,
0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13,
0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85,
0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3,
0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070,
0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5,
0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3,
0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208,
0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2,
};
}
namespace sha256_arm_shani {
void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks)
{
uint32x4_t STATE0, STATE1, ABEF_SAVE, CDGH_SAVE;
uint32x4_t MSG0, MSG1, MSG2, MSG3;
uint32x4_t TMP0, TMP2;
// Load state
STATE0 = vld1q_u32(&s[0]);
STATE1 = vld1q_u32(&s[4]);
while (blocks--)
{
// Save state
ABEF_SAVE = STATE0;
CDGH_SAVE = STATE1;
// Load and convert input chunk to Big Endian
MSG0 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(chunk + 0)));
MSG1 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(chunk + 16)));
MSG2 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(chunk + 32)));
MSG3 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(chunk + 48)));
chunk += 64;
// Original implemenation preloaded message and constant addition which was 1-3% slower.
// Now included as first step in quad round code saving one Q Neon register
// "TMP0 = vaddq_u32(MSG0, vld1q_u32(&K[0]));"
// Rounds 1-4
TMP0 = vaddq_u32(MSG0, vld1q_u32(&K[0]));
TMP2 = STATE0;
MSG0 = vsha256su0q_u32(MSG0, MSG1);
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
MSG0 = vsha256su1q_u32(MSG0, MSG2, MSG3);
// Rounds 5-8
TMP0 = vaddq_u32(MSG1, vld1q_u32(&K[4]));
TMP2 = STATE0;
MSG1 = vsha256su0q_u32(MSG1, MSG2);
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
MSG1 = vsha256su1q_u32(MSG1, MSG3, MSG0);
// Rounds 9-12
TMP0 = vaddq_u32(MSG2, vld1q_u32(&K[8]));
TMP2 = STATE0;
MSG2 = vsha256su0q_u32(MSG2, MSG3);
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
MSG2 = vsha256su1q_u32(MSG2, MSG0, MSG1);
// Rounds 13-16
TMP0 = vaddq_u32(MSG3, vld1q_u32(&K[12]));
TMP2 = STATE0;
MSG3 = vsha256su0q_u32(MSG3, MSG0);
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
MSG3 = vsha256su1q_u32(MSG3, MSG1, MSG2);
// Rounds 17-20
TMP0 = vaddq_u32(MSG0, vld1q_u32(&K[16]));
TMP2 = STATE0;
MSG0 = vsha256su0q_u32(MSG0, MSG1);
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
MSG0 = vsha256su1q_u32(MSG0, MSG2, MSG3);
// Rounds 21-24
TMP0 = vaddq_u32(MSG1, vld1q_u32(&K[20]));
TMP2 = STATE0;
MSG1 = vsha256su0q_u32(MSG1, MSG2);
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
MSG1 = vsha256su1q_u32(MSG1, MSG3, MSG0);
// Rounds 25-28
TMP0 = vaddq_u32(MSG2, vld1q_u32(&K[24]));
TMP2 = STATE0;
MSG2 = vsha256su0q_u32(MSG2, MSG3);
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
MSG2 = vsha256su1q_u32(MSG2, MSG0, MSG1);
// Rounds 29-32
TMP0 = vaddq_u32(MSG3, vld1q_u32(&K[28]));
TMP2 = STATE0;
MSG3 = vsha256su0q_u32(MSG3, MSG0);
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
MSG3 = vsha256su1q_u32(MSG3, MSG1, MSG2);
// Rounds 33-36
TMP0 = vaddq_u32(MSG0, vld1q_u32(&K[32]));
TMP2 = STATE0;
MSG0 = vsha256su0q_u32(MSG0, MSG1);
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
MSG0 = vsha256su1q_u32(MSG0, MSG2, MSG3);
// Rounds 37-40
TMP0 = vaddq_u32(MSG1, vld1q_u32(&K[36]));
TMP2 = STATE0;
MSG1 = vsha256su0q_u32(MSG1, MSG2);
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
MSG1 = vsha256su1q_u32(MSG1, MSG3, MSG0);
// Rounds 41-44
TMP0 = vaddq_u32(MSG2, vld1q_u32(&K[40]));
TMP2 = STATE0;
MSG2 = vsha256su0q_u32(MSG2, MSG3);
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
MSG2 = vsha256su1q_u32(MSG2, MSG0, MSG1);
// Rounds 45-48
TMP0 = vaddq_u32(MSG3, vld1q_u32(&K[44]));
TMP2 = STATE0;
MSG3 = vsha256su0q_u32(MSG3, MSG0);
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
MSG3 = vsha256su1q_u32(MSG3, MSG1, MSG2);
// Rounds 49-52
TMP0 = vaddq_u32(MSG0, vld1q_u32(&K[48]));
TMP2 = STATE0;
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
// Rounds 53-56
TMP0 = vaddq_u32(MSG1, vld1q_u32(&K[52]));
TMP2 = STATE0;
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
// Rounds 57-60
TMP0 = vaddq_u32(MSG2, vld1q_u32(&K[56]));
TMP2 = STATE0;
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
// Rounds 61-64
TMP0 = vaddq_u32(MSG3, vld1q_u32(&K[60]));
TMP2 = STATE0;
STATE0 = vsha256hq_u32(STATE0, STATE1, TMP0);
STATE1 = vsha256h2q_u32(STATE1, TMP2, TMP0);
// Update state
STATE0 = vaddq_u32(STATE0, ABEF_SAVE);
STATE1 = vaddq_u32(STATE1, CDGH_SAVE);
}
// Save final state
vst1q_u32(&s[0], STATE0);
vst1q_u32(&s[4], STATE1);
}
}
namespace sha256d64_arm_shani {
void Transform_2way(unsigned char* output, const unsigned char* input)
{
/* Initial state. */
alignas(uint32x4_t) static constexpr std::array<uint32_t, 8> INIT = {
0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19
};
/* Precomputed message schedule for the 2nd transform. */
alignas(uint32x4_t) static constexpr std::array<uint32_t, 64> MIDS = {
0xc28a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf374,
0x649b69c1, 0xf0fe4786, 0x0fe1edc6, 0x240cf254,
0x4fe9346f, 0x6cc984be, 0x61b9411e, 0x16f988fa,
0xf2c65152, 0xa88e5a6d, 0xb019fc65, 0xb9d99ec7,
0x9a1231c3, 0xe70eeaa0, 0xfdb1232b, 0xc7353eb0,
0x3069bad5, 0xcb976d5f, 0x5a0f118f, 0xdc1eeefd,
0x0a35b689, 0xde0b7a04, 0x58f4ca9d, 0xe15d5b16,
0x007f3e86, 0x37088980, 0xa507ea32, 0x6fab9537,
0x17406110, 0x0d8cd6f1, 0xcdaa3b6d, 0xc0bbbe37,
0x83613bda, 0xdb48a363, 0x0b02e931, 0x6fd15ca7,
0x521afaca, 0x31338431, 0x6ed41a95, 0x6d437890,
0xc39c91f2, 0x9eccabbd, 0xb5c9a0e6, 0x532fb63c,
0xd2c741c6, 0x07237ea3, 0xa4954b68, 0x4c191d76
};
/* A few precomputed message schedule values for the 3rd transform. */
alignas(uint32x4_t) static constexpr std::array<uint32_t, 12> FINS = {
0x5807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
0x80000000, 0x00000000, 0x00000000, 0x00000000,
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf274
};
/* Padding processed in the 3rd transform (byteswapped). */
alignas(uint32x4_t) static constexpr std::array<uint32_t, 8> FINAL = {0x80000000, 0, 0, 0, 0, 0, 0, 0x100};
uint32x4_t STATE0A, STATE0B, STATE1A, STATE1B, ABEF_SAVEA, ABEF_SAVEB, CDGH_SAVEA, CDGH_SAVEB;
uint32x4_t MSG0A, MSG0B, MSG1A, MSG1B, MSG2A, MSG2B, MSG3A, MSG3B;
uint32x4_t TMP0A, TMP0B, TMP2A, TMP2B, TMP;
// Transform 1: Load state
STATE0A = vld1q_u32(&INIT[0]);
STATE0B = STATE0A;
STATE1A = vld1q_u32(&INIT[4]);
STATE1B = STATE1A;
// Transform 1: Load and convert input chunk to Big Endian
MSG0A = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 0)));
MSG1A = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 16)));
MSG2A = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 32)));
MSG3A = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 48)));
MSG0B = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 64)));
MSG1B = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 80)));
MSG2B = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 96)));
MSG3B = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(input + 112)));
// Transform 1: Rounds 1-4
TMP = vld1q_u32(&K[0]);
TMP0A = vaddq_u32(MSG0A, TMP);
TMP0B = vaddq_u32(MSG0B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG0A = vsha256su0q_u32(MSG0A, MSG1A);
MSG0B = vsha256su0q_u32(MSG0B, MSG1B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A);
MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B);
// Transform 1: Rounds 5-8
TMP = vld1q_u32(&K[4]);
TMP0A = vaddq_u32(MSG1A, TMP);
TMP0B = vaddq_u32(MSG1B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG1A = vsha256su0q_u32(MSG1A, MSG2A);
MSG1B = vsha256su0q_u32(MSG1B, MSG2B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A);
MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B);
// Transform 1: Rounds 9-12
TMP = vld1q_u32(&K[8]);
TMP0A = vaddq_u32(MSG2A, TMP);
TMP0B = vaddq_u32(MSG2B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG2A = vsha256su0q_u32(MSG2A, MSG3A);
MSG2B = vsha256su0q_u32(MSG2B, MSG3B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A);
MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B);
// Transform 1: Rounds 13-16
TMP = vld1q_u32(&K[12]);
TMP0A = vaddq_u32(MSG3A, TMP);
TMP0B = vaddq_u32(MSG3B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG3A = vsha256su0q_u32(MSG3A, MSG0A);
MSG3B = vsha256su0q_u32(MSG3B, MSG0B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A);
MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B);
// Transform 1: Rounds 17-20
TMP = vld1q_u32(&K[16]);
TMP0A = vaddq_u32(MSG0A, TMP);
TMP0B = vaddq_u32(MSG0B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG0A = vsha256su0q_u32(MSG0A, MSG1A);
MSG0B = vsha256su0q_u32(MSG0B, MSG1B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A);
MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B);
// Transform 1: Rounds 21-24
TMP = vld1q_u32(&K[20]);
TMP0A = vaddq_u32(MSG1A, TMP);
TMP0B = vaddq_u32(MSG1B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG1A = vsha256su0q_u32(MSG1A, MSG2A);
MSG1B = vsha256su0q_u32(MSG1B, MSG2B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A);
MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B);
// Transform 1: Rounds 25-28
TMP = vld1q_u32(&K[24]);
TMP0A = vaddq_u32(MSG2A, TMP);
TMP0B = vaddq_u32(MSG2B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG2A = vsha256su0q_u32(MSG2A, MSG3A);
MSG2B = vsha256su0q_u32(MSG2B, MSG3B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A);
MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B);
// Transform 1: Rounds 29-32
TMP = vld1q_u32(&K[28]);
TMP0A = vaddq_u32(MSG3A, TMP);
TMP0B = vaddq_u32(MSG3B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG3A = vsha256su0q_u32(MSG3A, MSG0A);
MSG3B = vsha256su0q_u32(MSG3B, MSG0B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A);
MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B);
// Transform 1: Rounds 33-36
TMP = vld1q_u32(&K[32]);
TMP0A = vaddq_u32(MSG0A, TMP);
TMP0B = vaddq_u32(MSG0B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG0A = vsha256su0q_u32(MSG0A, MSG1A);
MSG0B = vsha256su0q_u32(MSG0B, MSG1B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A);
MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B);
// Transform 1: Rounds 37-40
TMP = vld1q_u32(&K[36]);
TMP0A = vaddq_u32(MSG1A, TMP);
TMP0B = vaddq_u32(MSG1B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG1A = vsha256su0q_u32(MSG1A, MSG2A);
MSG1B = vsha256su0q_u32(MSG1B, MSG2B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A);
MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B);
// Transform 1: Rounds 41-44
TMP = vld1q_u32(&K[40]);
TMP0A = vaddq_u32(MSG2A, TMP);
TMP0B = vaddq_u32(MSG2B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG2A = vsha256su0q_u32(MSG2A, MSG3A);
MSG2B = vsha256su0q_u32(MSG2B, MSG3B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A);
MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B);
// Transform 1: Rounds 45-48
TMP = vld1q_u32(&K[44]);
TMP0A = vaddq_u32(MSG3A, TMP);
TMP0B = vaddq_u32(MSG3B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG3A = vsha256su0q_u32(MSG3A, MSG0A);
MSG3B = vsha256su0q_u32(MSG3B, MSG0B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A);
MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B);
// Transform 1: Rounds 49-52
TMP = vld1q_u32(&K[48]);
TMP0A = vaddq_u32(MSG0A, TMP);
TMP0B = vaddq_u32(MSG0B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
// Transform 1: Rounds 53-56
TMP = vld1q_u32(&K[52]);
TMP0A = vaddq_u32(MSG1A, TMP);
TMP0B = vaddq_u32(MSG1B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
// Transform 1: Rounds 57-60
TMP = vld1q_u32(&K[56]);
TMP0A = vaddq_u32(MSG2A, TMP);
TMP0B = vaddq_u32(MSG2B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
// Transform 1: Rounds 61-64
TMP = vld1q_u32(&K[60]);
TMP0A = vaddq_u32(MSG3A, TMP);
TMP0B = vaddq_u32(MSG3B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
// Transform 1: Update state
TMP = vld1q_u32(&INIT[0]);
STATE0A = vaddq_u32(STATE0A, TMP);
STATE0B = vaddq_u32(STATE0B, TMP);
TMP = vld1q_u32(&INIT[4]);
STATE1A = vaddq_u32(STATE1A, TMP);
STATE1B = vaddq_u32(STATE1B, TMP);
// Transform 2: Save state
ABEF_SAVEA = STATE0A;
ABEF_SAVEB = STATE0B;
CDGH_SAVEA = STATE1A;
CDGH_SAVEB = STATE1B;
// Transform 2: Rounds 1-4
TMP = vld1q_u32(&MIDS[0]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Rounds 5-8
TMP = vld1q_u32(&MIDS[4]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Rounds 9-12
TMP = vld1q_u32(&MIDS[8]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Rounds 13-16
TMP = vld1q_u32(&MIDS[12]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Rounds 17-20
TMP = vld1q_u32(&MIDS[16]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Rounds 21-24
TMP = vld1q_u32(&MIDS[20]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Rounds 25-28
TMP = vld1q_u32(&MIDS[24]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Rounds 29-32
TMP = vld1q_u32(&MIDS[28]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Rounds 33-36
TMP = vld1q_u32(&MIDS[32]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Rounds 37-40
TMP = vld1q_u32(&MIDS[36]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Rounds 41-44
TMP = vld1q_u32(&MIDS[40]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Rounds 45-48
TMP = vld1q_u32(&MIDS[44]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Rounds 49-52
TMP = vld1q_u32(&MIDS[48]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Rounds 53-56
TMP = vld1q_u32(&MIDS[52]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Rounds 57-60
TMP = vld1q_u32(&MIDS[56]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Rounds 61-64
TMP = vld1q_u32(&MIDS[60]);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
// Transform 2: Update state
STATE0A = vaddq_u32(STATE0A, ABEF_SAVEA);
STATE0B = vaddq_u32(STATE0B, ABEF_SAVEB);
STATE1A = vaddq_u32(STATE1A, CDGH_SAVEA);
STATE1B = vaddq_u32(STATE1B, CDGH_SAVEB);
// Transform 3: Pad previous output
MSG0A = STATE0A;
MSG0B = STATE0B;
MSG1A = STATE1A;
MSG1B = STATE1B;
MSG2A = vld1q_u32(&FINAL[0]);
MSG2B = MSG2A;
MSG3A = vld1q_u32(&FINAL[4]);
MSG3B = MSG3A;
// Transform 3: Load state
STATE0A = vld1q_u32(&INIT[0]);
STATE0B = STATE0A;
STATE1A = vld1q_u32(&INIT[4]);
STATE1B = STATE1A;
// Transform 3: Rounds 1-4
TMP = vld1q_u32(&K[0]);
TMP0A = vaddq_u32(MSG0A, TMP);
TMP0B = vaddq_u32(MSG0B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG0A = vsha256su0q_u32(MSG0A, MSG1A);
MSG0B = vsha256su0q_u32(MSG0B, MSG1B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A);
MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B);
// Transform 3: Rounds 5-8
TMP = vld1q_u32(&K[4]);
TMP0A = vaddq_u32(MSG1A, TMP);
TMP0B = vaddq_u32(MSG1B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG1A = vsha256su0q_u32(MSG1A, MSG2A);
MSG1B = vsha256su0q_u32(MSG1B, MSG2B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A);
MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B);
// Transform 3: Rounds 9-12
TMP = vld1q_u32(&FINS[0]);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG2A = vld1q_u32(&FINS[4]);
MSG2B = MSG2A;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A);
MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B);
// Transform 3: Rounds 13-16
TMP = vld1q_u32(&FINS[8]);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG3A = vsha256su0q_u32(MSG3A, MSG0A);
MSG3B = vsha256su0q_u32(MSG3B, MSG0B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP);
MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A);
MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B);
// Transform 3: Rounds 17-20
TMP = vld1q_u32(&K[16]);
TMP0A = vaddq_u32(MSG0A, TMP);
TMP0B = vaddq_u32(MSG0B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG0A = vsha256su0q_u32(MSG0A, MSG1A);
MSG0B = vsha256su0q_u32(MSG0B, MSG1B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A);
MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B);
// Transform 3: Rounds 21-24
TMP = vld1q_u32(&K[20]);
TMP0A = vaddq_u32(MSG1A, TMP);
TMP0B = vaddq_u32(MSG1B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG1A = vsha256su0q_u32(MSG1A, MSG2A);
MSG1B = vsha256su0q_u32(MSG1B, MSG2B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A);
MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B);
// Transform 3: Rounds 25-28
TMP = vld1q_u32(&K[24]);
TMP0A = vaddq_u32(MSG2A, TMP);
TMP0B = vaddq_u32(MSG2B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG2A = vsha256su0q_u32(MSG2A, MSG3A);
MSG2B = vsha256su0q_u32(MSG2B, MSG3B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A);
MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B);
// Transform 3: Rounds 29-32
TMP = vld1q_u32(&K[28]);
TMP0A = vaddq_u32(MSG3A, TMP);
TMP0B = vaddq_u32(MSG3B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG3A = vsha256su0q_u32(MSG3A, MSG0A);
MSG3B = vsha256su0q_u32(MSG3B, MSG0B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A);
MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B);
// Transform 3: Rounds 33-36
TMP = vld1q_u32(&K[32]);
TMP0A = vaddq_u32(MSG0A, TMP);
TMP0B = vaddq_u32(MSG0B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG0A = vsha256su0q_u32(MSG0A, MSG1A);
MSG0B = vsha256su0q_u32(MSG0B, MSG1B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG0A = vsha256su1q_u32(MSG0A, MSG2A, MSG3A);
MSG0B = vsha256su1q_u32(MSG0B, MSG2B, MSG3B);
// Transform 3: Rounds 37-40
TMP = vld1q_u32(&K[36]);
TMP0A = vaddq_u32(MSG1A, TMP);
TMP0B = vaddq_u32(MSG1B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG1A = vsha256su0q_u32(MSG1A, MSG2A);
MSG1B = vsha256su0q_u32(MSG1B, MSG2B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG1A = vsha256su1q_u32(MSG1A, MSG3A, MSG0A);
MSG1B = vsha256su1q_u32(MSG1B, MSG3B, MSG0B);
// Transform 3: Rounds 41-44
TMP = vld1q_u32(&K[40]);
TMP0A = vaddq_u32(MSG2A, TMP);
TMP0B = vaddq_u32(MSG2B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG2A = vsha256su0q_u32(MSG2A, MSG3A);
MSG2B = vsha256su0q_u32(MSG2B, MSG3B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG2A = vsha256su1q_u32(MSG2A, MSG0A, MSG1A);
MSG2B = vsha256su1q_u32(MSG2B, MSG0B, MSG1B);
// Transform 3: Rounds 45-48
TMP = vld1q_u32(&K[44]);
TMP0A = vaddq_u32(MSG3A, TMP);
TMP0B = vaddq_u32(MSG3B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
MSG3A = vsha256su0q_u32(MSG3A, MSG0A);
MSG3B = vsha256su0q_u32(MSG3B, MSG0B);
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
MSG3A = vsha256su1q_u32(MSG3A, MSG1A, MSG2A);
MSG3B = vsha256su1q_u32(MSG3B, MSG1B, MSG2B);
// Transform 3: Rounds 49-52
TMP = vld1q_u32(&K[48]);
TMP0A = vaddq_u32(MSG0A, TMP);
TMP0B = vaddq_u32(MSG0B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
// Transform 3: Rounds 53-56
TMP = vld1q_u32(&K[52]);
TMP0A = vaddq_u32(MSG1A, TMP);
TMP0B = vaddq_u32(MSG1B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
// Transform 3: Rounds 57-60
TMP = vld1q_u32(&K[56]);
TMP0A = vaddq_u32(MSG2A, TMP);
TMP0B = vaddq_u32(MSG2B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
// Transform 3: Rounds 61-64
TMP = vld1q_u32(&K[60]);
TMP0A = vaddq_u32(MSG3A, TMP);
TMP0B = vaddq_u32(MSG3B, TMP);
TMP2A = STATE0A;
TMP2B = STATE0B;
STATE0A = vsha256hq_u32(STATE0A, STATE1A, TMP0A);
STATE0B = vsha256hq_u32(STATE0B, STATE1B, TMP0B);
STATE1A = vsha256h2q_u32(STATE1A, TMP2A, TMP0A);
STATE1B = vsha256h2q_u32(STATE1B, TMP2B, TMP0B);
// Transform 3: Update state
TMP = vld1q_u32(&INIT[0]);
STATE0A = vaddq_u32(STATE0A, TMP);
STATE0B = vaddq_u32(STATE0B, TMP);
TMP = vld1q_u32(&INIT[4]);
STATE1A = vaddq_u32(STATE1A, TMP);
STATE1B = vaddq_u32(STATE1B, TMP);
// Store result
vst1q_u8(output, vrev32q_u8(vreinterpretq_u8_u32(STATE0A)));
vst1q_u8(output + 16, vrev32q_u8(vreinterpretq_u8_u32(STATE1A)));
vst1q_u8(output + 32, vrev32q_u8(vreinterpretq_u8_u32(STATE0B)));
vst1q_u8(output + 48, vrev32q_u8(vreinterpretq_u8_u32(STATE1B)));
}
}
#endif

View File

@ -6,7 +6,7 @@
// Written and placed in public domain by Jeffrey Walton. // Written and placed in public domain by Jeffrey Walton.
// Based on code from Intel, and by Sean Gulley for the miTLS project. // Based on code from Intel, and by Sean Gulley for the miTLS project.
#ifdef ENABLE_SHANI #ifdef ENABLE_X86_SHANI
#include <stdint.h> #include <stdint.h>
#include <immintrin.h> #include <immintrin.h>
@ -74,7 +74,7 @@ void inline __attribute__((always_inline)) Save(unsigned char* out, __m128i s)
} }
} }
namespace sha256_shani { namespace sha256_x86_shani {
void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks)
{ {
__m128i m0, m1, m2, m3, s0, s1, so0, so1; __m128i m0, m1, m2, m3, s0, s1, so0, so1;
@ -139,7 +139,7 @@ void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks)
} }
} }
namespace sha256d64_shani { namespace sha256d64_x86_shani {
void Transform_2way(unsigned char* out, const unsigned char* in) void Transform_2way(unsigned char* out, const unsigned char* in)
{ {