Merge #10821: Add SSE4 optimized SHA256

6b8d872 Protect SSE4 code behind a compile-time flag (Pieter Wuille)
fa9be90 Add selftest for SHA256 transform (Pieter Wuille)
c1ccb15 Add SSE4 based SHA256 (Pieter Wuille)
2991c91 Add SHA256 dispatcher (Pieter Wuille)
4d50f38 Support multi-block SHA256 transforms (Pieter Wuille)

Pull request description:

  This adds an SSE4 assembly version of the SHA256 transform by Intel, and uses it at run time if SSE4 instructions are available, and use a fallback C++ implementation otherwise. Nearly every x86_64 CPU supports SSE4. The feature is only enabled when compiled with `--enable-experimental-asm`.

  In order to avoid build dependencies and other complications, the original Intel YASM code was translated to GCC extended asm syntax.

  This gives around a 50% speedup on the SHA256 benchmark for me.

  It is based on an earlier patch by @laanwj, though only includes a single assembly version (for now), and removes the YASM dependency.

Tree-SHA512: d31c50695ceb45264291537b93c0d7497670be38edf021ca5402eaa7d4e1e0e1ae492326e28d4e93979d066168129e62d1825e0384b1b906d36f85d93dfcb43c
This commit is contained in:
Wladimir J. van der Laan 2017-07-20 20:16:28 +02:00 committed by Pasta
parent d587fe3f0e
commit c7dcf79f02
No known key found for this signature in database
GPG Key ID: D362C9F7142766AE
7 changed files with 1674 additions and 83 deletions

View File

@ -178,6 +178,16 @@ AC_ARG_ENABLE([glibc-back-compat],
[use_glibc_compat=$enableval],
[use_glibc_compat=no])
AC_ARG_ENABLE([experimental-asm],
[AS_HELP_STRING([--enable-experimental-asm],
[Enable experimental assembly routines (default is no)])],
[experimental_asm=$enableval],
[experimental_asm=no])
if test "x$experimental_asm" = xyes; then
AC_DEFINE(EXPERIMENTAL_ASM, 1, [Define this symbol to build in experimental assembly routines])
fi
AC_ARG_WITH([system-univalue],
[AS_HELP_STRING([--with-system-univalue],
[Build with system UniValue (default is no)])],
@ -1218,6 +1228,7 @@ AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes])
AM_CONDITIONAL([GLIBC_BACK_COMPAT],[test x$use_glibc_compat = xyes])
AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes])
AM_CONDITIONAL([ENABLE_HWCRC32],[test x$enable_hwcrc32 = xyes])
AM_CONDITIONAL([EXPERIMENTAL_ASM],[test x$experimental_asm = xyes])
AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version])
AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version])

View File

@ -385,6 +385,10 @@ crypto_libdash_crypto_a_SOURCES = \
crypto/sha512.cpp \
crypto/sha512.h
if EXPERIMENTAL_ASM
crypto_libbitcoin_crypto_a_SOURCES += crypto/sha256_sse4.cpp
endif
# x11
crypto_libdash_crypto_a_SOURCES += \
crypto/blake.c \

View File

@ -3,10 +3,21 @@
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include "crypto/sha256.h"
#include "crypto/common.h"
#include <assert.h>
#include <string.h>
#include <atomic>
#if defined(__x86_64__) || defined(__amd64__)
#if defined(EXPERIMENTAL_ASM)
#include <cpuid.h>
namespace sha256_sse4
{
void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks);
}
#endif
#endif
// Internal implementation code.
namespace
@ -43,9 +54,10 @@ void inline Initialize(uint32_t* s)
s[7] = 0x5be0cd19ul;
}
/** Perform one SHA-256 transformation, processing a 64-byte chunk. */
void Transform(uint32_t* s, const unsigned char* chunk)
/** Perform a number of SHA-256 transformations, processing 64-byte chunks. */
void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks)
{
while (blocks--) {
uint32_t a = s[0], b = s[1], c = s[2], d = s[3], e = s[4], f = s[5], g = s[6], h = s[7];
uint32_t w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15;
@ -125,11 +137,59 @@ void Transform(uint32_t* s, const unsigned char* chunk)
s[5] += f;
s[6] += g;
s[7] += h;
chunk += 64;
}
}
} // namespace sha256
typedef void (*TransformType)(uint32_t*, const unsigned char*, size_t);
bool SelfTest(TransformType tr) {
static const unsigned char in1[65] = {0, 0x80};
static const unsigned char in2[129] = {
0,
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0
};
static const uint32_t init[8] = {0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, 0x1f83d9abul, 0x5be0cd19ul};
static const uint32_t out1[8] = {0xe3b0c442ul, 0x98fc1c14ul, 0x9afbf4c8ul, 0x996fb924ul, 0x27ae41e4ul, 0x649b934cul, 0xa495991bul, 0x7852b855ul};
static const uint32_t out2[8] = {0xce4153b0ul, 0x147c2a86ul, 0x3ed4298eul, 0xe0676bc8ul, 0x79fc77a1ul, 0x2abe1f49ul, 0xb2b055dful, 0x1069523eul};
uint32_t buf[8];
memcpy(buf, init, sizeof(buf));
// Process nothing, and check we remain in the initial state.
tr(buf, nullptr, 0);
if (memcmp(buf, init, sizeof(buf))) return false;
// Process the padded empty string (unaligned)
tr(buf, in1 + 1, 1);
if (memcmp(buf, out1, sizeof(buf))) return false;
// Process 64 spaces (unaligned)
memcpy(buf, init, sizeof(buf));
tr(buf, in2 + 1, 2);
if (memcmp(buf, out2, sizeof(buf))) return false;
return true;
}
TransformType Transform = sha256::Transform;
} // namespace
std::string SHA256AutoDetect()
{
#if defined(EXPERIMENTAL_ASM) && (defined(__x86_64__) || defined(__amd64__))
uint32_t eax, ebx, ecx, edx;
if (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx >> 19) & 1) {
Transform = sha256_sse4::Transform;
assert(SelfTest(Transform));
return "sse4";
}
#endif
assert(SelfTest(Transform));
return "standard";
}
////// SHA-256
@ -147,14 +207,14 @@ CSHA256& CSHA256::Write(const unsigned char* data, size_t len)
memcpy(buf + bufsize, data, 64 - bufsize);
bytes += 64 - bufsize;
data += 64 - bufsize;
sha256::Transform(s, buf);
Transform(s, buf, 1);
bufsize = 0;
}
while (end >= data + 64) {
// Process full chunks directly from the source.
sha256::Transform(s, data);
bytes += 64;
data += 64;
if (end - data >= 64) {
size_t blocks = (end - data) / 64;
Transform(s, data, blocks);
data += 64 * blocks;
bytes += 64 * blocks;
}
if (end > data) {
// Fill the buffer with what remains.

View File

@ -7,6 +7,7 @@
#include <stdint.h>
#include <stdlib.h>
#include <string>
/** A hasher class for SHA-256. */
class CSHA256
@ -25,4 +26,9 @@ public:
CSHA256& Reset();
};
/** Autodetect the best available SHA256 implementation.
* Returns the name of the implementation.
*/
std::string SHA256AutoDetect();
#endif // BITCOIN_CRYPTO_SHA256_H

1506
src/crypto/sha256_sse4.cpp Normal file

File diff suppressed because it is too large Load Diff

View File

@ -1441,6 +1441,8 @@ bool AppInitSanityChecks()
// ********************************************************* Step 4: sanity checks
// Initialize elliptic curve code
std::string sha256_algo = SHA256AutoDetect();
LogPrintf("Using the '%s' SHA256 implementation\n", sha256_algo);
RandomInit();
ECC_Start();
globalVerifyHandle.reset(new ECCVerifyHandle());

View File

@ -7,6 +7,7 @@
#include "chainparams.h"
#include "consensus/consensus.h"
#include "consensus/validation.h"
#include "crypto/sha256.h"
#include "fs.h"
#include "key.h"
#include "validation.h"
@ -38,6 +39,7 @@ extern void noui_connect();
BasicTestingSetup::BasicTestingSetup(const std::string& chainName)
{
SHA256AutoDetect();
RandomInit();
ECC_Start();
BLSInit();