Merge #8173: Use SipHash for node eviction (cont'd)

eebc232 test: Add more test vectors for siphash (Wladimir J. van der Laan)
8884830 Use C++11 thread-safe static initializers (Pieter Wuille)
c31b24f Use 64-bit SipHash of netgroups in eviction (Pieter Wuille)
9bf156b Support SipHash with arbitrary byte writes (Pieter Wuille)
053930f Avoid recalculating vchKeyedNetGroup in eviction logic. (Patrick Strateman)
This commit is contained in:
Pieter Wuille 2016-06-08 18:05:01 +02:00 committed by Alexander Block
parent fc7b5778f1
commit 361d26037b
6 changed files with 159 additions and 81 deletions

View File

@ -100,12 +100,15 @@ CSipHasher::CSipHasher(uint64_t k0, uint64_t k1)
v[2] = 0x6c7967656e657261ULL ^ k0;
v[3] = 0x7465646279746573ULL ^ k1;
count = 0;
tmp = 0;
}
CSipHasher& CSipHasher::Write(uint64_t data)
{
uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
assert(count % 8 == 0);
v3 ^= data;
SIPROUND;
SIPROUND;
@ -116,7 +119,35 @@ CSipHasher& CSipHasher::Write(uint64_t data)
v[2] = v2;
v[3] = v3;
count++;
count += 8;
return *this;
}
CSipHasher& CSipHasher::Write(const unsigned char* data, size_t size)
{
uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
uint64_t t = tmp;
int c = count;
while (size--) {
t |= ((uint64_t)(*(data++))) << (8 * (c % 8));
c++;
if ((c & 7) == 0) {
v3 ^= t;
SIPROUND;
SIPROUND;
v0 ^= t;
t = 0;
}
}
v[0] = v0;
v[1] = v1;
v[2] = v2;
v[3] = v3;
count = c;
tmp = t;
return *this;
}
@ -124,10 +155,12 @@ uint64_t CSipHasher::Finalize() const
{
uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3];
v3 ^= ((uint64_t)count) << 59;
uint64_t t = tmp | (((uint64_t)count) << 56);
v3 ^= t;
SIPROUND;
SIPROUND;
v0 ^= ((uint64_t)count) << 59;
v0 ^= t;
v2 ^= 0xFF;
SIPROUND;
SIPROUND;

View File

@ -313,19 +313,38 @@ unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector<unsigned char
void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char header, const unsigned char data[32], unsigned char output[64]);
/** SipHash-2-4, using a uint64_t-based (rather than byte-based) interface */
/** SipHash-2-4 */
class CSipHasher
{
private:
uint64_t v[4];
uint64_t tmp;
int count;
public:
/** Construct a SipHash calculator initialized with 128-bit key (k0, k1) */
CSipHasher(uint64_t k0, uint64_t k1);
/** Hash a 64-bit integer worth of data
* It is treated as if this was the little-endian interpretation of 8 bytes.
* This function can only be used when a multiple of 8 bytes have been written so far.
*/
CSipHasher& Write(uint64_t data);
/** Hash arbitrary bytes. */
CSipHasher& Write(const unsigned char* data, size_t size);
/** Compute the 64-bit SipHash-2-4 of the data written so far. The object remains untouched. */
uint64_t Finalize() const;
};
/** Optimized SipHash-2-4 implementation for uint256.
*
* It is identical to:
* SipHasher(k0, k1)
* .Write(val.GetUint64(0))
* .Write(val.GetUint64(1))
* .Write(val.GetUint64(2))
* .Write(val.GetUint64(3))
* .Finalize()
*/
uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val);
uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra);

View File

@ -15,6 +15,7 @@
#include "clientversion.h"
#include "consensus/consensus.h"
#include "crypto/common.h"
#include "crypto/sha256.h"
#include "hash.h"
#include "primitives/transaction.h"
#include "netbase.h"
@ -292,8 +293,6 @@ bool IsReachable(const CNetAddr& addr)
}
std::vector<unsigned char> CNode::vchSecretKey;
CNode* CConnman::FindNode(const CNetAddr& ip)
{
LOCK(cs_vNodes);
@ -861,11 +860,10 @@ struct NodeEvictionCandidate
fNetworkNode(pnode->fNetworkNode),
fRelayTxes(pnode->fRelayTxes),
fBloomFilter(pnode->pfilter != NULL),
vchNetGroup(pnode->addr.GetGroup()),
vchKeyedNetGroup(pnode->vchKeyedNetGroup)
nKeyedNetGroup(pnode->nKeyedNetGroup)
{}
int id;
NodeId id;
int64_t nTimeConnected;
int64_t nMinPingUsecTime;
int64_t nLastBlockTime;
@ -873,8 +871,7 @@ struct NodeEvictionCandidate
bool fNetworkNode;
bool fRelayTxes;
bool fBloomFilter;
std::vector<unsigned char> vchNetGroup;
std::vector<unsigned char> vchKeyedNetGroup;
uint64_t nKeyedNetGroup;
};
static bool ReverseCompareNodeMinPingTime(const NodeEvictionCandidate& a, const NodeEvictionCandidate& b)
@ -887,10 +884,9 @@ static bool ReverseCompareNodeTimeConnected(const NodeEvictionCandidate& a, cons
return a.nTimeConnected > b.nTimeConnected;
}
static bool CompareKeyedNetGroup(const NodeEvictionCandidate& a, const NodeEvictionCandidate& b)
{
return a.vchKeyedNetGroup < b.vchKeyedNetGroup;
}
static bool CompareNetGroupKeyed(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) {
return a.nKeyedNetGroup < b.nKeyedNetGroup;
};
static bool CompareNodeBlockTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b)
{
@ -940,8 +936,8 @@ bool CConnman::AttemptToEvictConnection()
// Protect connections with certain characteristics
// Deterministically select 4 peers to protect by netgroup.
// An attacker cannot predict which netgroups will be protected.
std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareKeyedNetGroup);
// An attacker cannot predict which netgroups will be protected
std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNetGroupKeyed);
vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast<int>(vEvictionCandidates.size())), vEvictionCandidates.end());
if (vEvictionCandidates.empty()) return false;
@ -976,44 +972,34 @@ bool CConnman::AttemptToEvictConnection()
// Identify the network group with the most connections and youngest member.
// (vEvictionCandidates is already sorted by reverse connect time)
std::vector<unsigned char> naMostConnections;
uint64_t naMostConnections;
unsigned int nMostConnections = 0;
int64_t nMostConnectionsTime = 0;
std::map<std::vector<unsigned char>, std::vector<NodeEvictionCandidate> > mapAddrCounts;
for(size_t i = 0; i < vEvictionCandidates.size(); ++i) {
const NodeEvictionCandidate& candidate = vEvictionCandidates[i];
mapAddrCounts[candidate.vchNetGroup].push_back(candidate);
int64_t grouptime = mapAddrCounts[candidate.vchNetGroup][0].nTimeConnected;
size_t groupsize = mapAddrCounts[candidate.vchNetGroup].size();
std::map<uint64_t, std::vector<NodeEvictionCandidate> > mapAddrCounts;
BOOST_FOREACH(const NodeEvictionCandidate &node, vEvictionCandidates) {
mapAddrCounts[node.nKeyedNetGroup].push_back(node);
int64_t grouptime = mapAddrCounts[node.nKeyedNetGroup][0].nTimeConnected;
size_t groupsize = mapAddrCounts[node.nKeyedNetGroup].size();
if (groupsize > nMostConnections || (groupsize == nMostConnections && grouptime > nMostConnectionsTime)) {
nMostConnections = groupsize;
nMostConnectionsTime = grouptime;
naMostConnections = candidate.vchNetGroup;
naMostConnections = node.nKeyedNetGroup;
}
}
// Reduce to the network group with the most connections
std::vector<NodeEvictionCandidate> vEvictionNodes = mapAddrCounts[naMostConnections];
// Do not disconnect peers if there is only 1 connection from their network group
if(vEvictionNodes.empty()) {
return false;
}
vEvictionCandidates = std::move(mapAddrCounts[naMostConnections]);
// Disconnect from the network group with the most connections
int nEvictionId = vEvictionNodes[0].id;
{
LOCK(cs_vNodes);
for(size_t i = 0; i < vNodes.size(); ++i) {
CNode* pnode = vNodes[i];
if(pnode->id == nEvictionId) {
pnode->fDisconnect = true;
return true;
}
NodeId evicted = vEvictionCandidates.front().id;
LOCK(cs_vNodes);
for(std::vector<CNode*>::const_iterator it(vNodes.begin()); it != vNodes.end(); ++it) {
if ((*it)->GetId() == evicted) {
(*it)->fDisconnect = true;
return true;
}
}
return false;
}
@ -2622,6 +2608,8 @@ unsigned int CConnman::GetReceiveFloodSize() const { return nReceiveFloodSize; }
unsigned int CConnman::GetSendBufferSize() const{ return nSendBufferMaxSize; }
CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNameIn, bool fInboundIn, bool fNetworkNodeIn) :
addr(addrIn),
nKeyedNetGroup(CalculateKeyedNetGroup(addrIn)),
addrKnown(5000, 0.001),
filterInventoryKnown(50000, 0.000001),
nSendVersion(0)
@ -2636,7 +2624,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn
nRecvBytes = 0;
nTimeConnected = GetSystemTimeInSeconds();
nTimeOffset = 0;
addr = addrIn;
addrName = addrNameIn == "" ? addr.ToStringIPPort() : addrNameIn;
nVersion = 0;
nNumWarningsSkipped = 0;
@ -2676,7 +2663,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn
minFeeFilter = 0;
lastSentFeeFilter = 0;
nextSendTimeFeeFilter = 0;
vchKeyedNetGroup = CalculateKeyedNetGroup(addr);
id = idIn;
nLocalServices = nLocalServicesIn;
fPauseRecv = false;
@ -2758,27 +2744,6 @@ bool CConnman::NodeFullyConnected(const CNode* pnode)
return pnode && pnode->fSuccessfullyConnected && !pnode->fDisconnect;
}
std::vector<unsigned char> CNode::CalculateKeyedNetGroup(CAddress& address)
{
if(vchSecretKey.size() == 0) {
vchSecretKey.resize(32, 0);
GetRandBytes(vchSecretKey.data(), vchSecretKey.size());
}
std::vector<unsigned char> vchGroup;
CSHA256 hash;
std::vector<unsigned char> vch(32);
vchGroup = address.GetGroup();
hash.Write(begin_ptr(vchGroup), vchGroup.size());
hash.Write(begin_ptr(vchSecretKey), vchSecretKey.size());
hash.Finalize(begin_ptr(vch));
return vch;
}
CDataStream CConnman::BeginMessage(CNode* pnode, int nVersion, int flags, const std::string& sCommand)
{
return {SER_NETWORK, (nVersion ? nVersion : pnode->GetSendVersion()) | flags, CMessageHeader(Params().MessageStart(), sCommand.c_str(), 0) };
@ -2878,3 +2843,13 @@ void CConnman::ReleaseNodeVector(const std::vector<CNode*>& vecNodes)
pnode->Release();
}
}
/* static */ uint64_t CNode::CalculateKeyedNetGroup(const CAddress& ad)
{
static const uint64_t k0 = GetRand(std::numeric_limits<uint64_t>::max());
static const uint64_t k1 = GetRand(std::numeric_limits<uint64_t>::max());
std::vector<unsigned char> vchNetGroup(ad.GetGroup());
return CSipHasher(k0, k1).Write(&vchNetGroup[0], vchNetGroup.size()).Finalize();
}

View File

@ -683,7 +683,7 @@ public:
int64_t nTimeConnected;
int64_t nTimeOffset;
int64_t nLastWarningTime;
CAddress addr;
const CAddress addr;
std::string addrName;
CService addrLocal;
int nNumWarningsSkipped;
@ -716,6 +716,8 @@ public:
int nRefCount;
NodeId id;
const uint64_t nKeyedNetGroup;
std::atomic_bool fPauseRecv;
std::atomic_bool fPauseSend;
protected:
@ -782,20 +784,17 @@ public:
CAmount lastSentFeeFilter;
int64_t nextSendTimeFeeFilter;
std::vector<unsigned char> vchKeyedNetGroup;
CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, const std::string &addrNameIn = "", bool fInboundIn = false, bool fNetworkNodeIn = false);
~CNode();
private:
// Secret key for computing keyed net groups
static std::vector<unsigned char> vchSecretKey;
CCriticalSection cs_nRefCount;
CNode(const CNode&);
void operator=(const CNode&);
static uint64_t CalculateKeyedNetGroup(const CAddress& ad);
uint64_t nLocalHostNonce;
ServiceFlags nLocalServices;
int nMyStartingHeight;
@ -918,8 +917,6 @@ public:
{
return nLocalServices;
}
static std::vector<unsigned char> CalculateKeyedNetGroup(CAddress& address);
};
class CExplicitNetCleanup

View File

@ -766,11 +766,8 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman& connma
// Relay to a limited number of other nodes
// Use deterministic randomness to send to the same nodes for 24 hours
// at a time so the addrKnowns of the chosen nodes prevent repeats
static uint64_t salt0 = 0, salt1 = 0;
while (salt0 == 0 && salt1 == 0) {
GetRandBytes((unsigned char*)&salt0, sizeof(salt0));
GetRandBytes((unsigned char*)&salt1, sizeof(salt1));
}
static const uint64_t salt0 = GetRand(std::numeric_limits<uint64_t>::max());
static const uint64_t salt1 = GetRand(std::numeric_limits<uint64_t>::max());
uint64_t hashAddr = addr.GetHash();
multimap<uint64_t, CNode*> mapMix;
const CSipHasher hasher = CSipHasher(salt0, salt1).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60));

View File

@ -47,17 +47,58 @@ BOOST_AUTO_TEST_CASE(murmurhash3)
#undef T
}
/*
SipHash-2-4 output with
k = 00 01 02 ...
and
in = (empty string)
in = 00 (1 byte)
in = 00 01 (2 bytes)
in = 00 01 02 (3 bytes)
...
in = 00 01 02 ... 3e (63 bytes)
from: https://131002.net/siphash/siphash24.c
*/
uint64_t siphash_4_2_testvec[] = {
0x726fdb47dd0e0e31, 0x74f839c593dc67fd, 0x0d6c8009d9a94f5a, 0x85676696d7fb7e2d,
0xcf2794e0277187b7, 0x18765564cd99a68d, 0xcbc9466e58fee3ce, 0xab0200f58b01d137,
0x93f5f5799a932462, 0x9e0082df0ba9e4b0, 0x7a5dbbc594ddb9f3, 0xf4b32f46226bada7,
0x751e8fbc860ee5fb, 0x14ea5627c0843d90, 0xf723ca908e7af2ee, 0xa129ca6149be45e5,
0x3f2acc7f57c29bdb, 0x699ae9f52cbe4794, 0x4bc1b3f0968dd39c, 0xbb6dc91da77961bd,
0xbed65cf21aa2ee98, 0xd0f2cbb02e3b67c7, 0x93536795e3a33e88, 0xa80c038ccd5ccec8,
0xb8ad50c6f649af94, 0xbce192de8a85b8ea, 0x17d835b85bbb15f3, 0x2f2e6163076bcfad,
0xde4daaaca71dc9a5, 0xa6a2506687956571, 0xad87a3535c49ef28, 0x32d892fad841c342,
0x7127512f72f27cce, 0xa7f32346f95978e3, 0x12e0b01abb051238, 0x15e034d40fa197ae,
0x314dffbe0815a3b4, 0x027990f029623981, 0xcadcd4e59ef40c4d, 0x9abfd8766a33735c,
0x0e3ea96b5304a7d0, 0xad0c42d6fc585992, 0x187306c89bc215a9, 0xd4a60abcf3792b95,
0xf935451de4f21df2, 0xa9538f0419755787, 0xdb9acddff56ca510, 0xd06c98cd5c0975eb,
0xe612a3cb9ecba951, 0xc766e62cfcadaf96, 0xee64435a9752fe72, 0xa192d576b245165a,
0x0a8787bf8ecb74b2, 0x81b3e73d20b49b6f, 0x7fa8220ba3b2ecea, 0x245731c13ca42499,
0xb78dbfaf3a8d83bd, 0xea1ad565322a1a0b, 0x60e61c23a3795013, 0x6606d7e446282b93,
0x6ca4ecb15c5f91e1, 0x9f626da15c9625f3, 0xe51b38608ef25f57, 0x958a324ceb064572
};
BOOST_AUTO_TEST_CASE(siphash)
{
CSipHasher hasher(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x726fdb47dd0e0e31ull);
hasher.Write(0x0706050403020100ULL);
static const unsigned char t0[1] = {0};
hasher.Write(t0, 1);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x74f839c593dc67fdull);
static const unsigned char t1[7] = {1,2,3,4,5,6,7};
hasher.Write(t1, 7);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x93f5f5799a932462ull);
hasher.Write(0x0F0E0D0C0B0A0908ULL);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x3f2acc7f57c29bdbull);
hasher.Write(0x1716151413121110ULL);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0xb8ad50c6f649af94ull);
hasher.Write(0x1F1E1D1C1B1A1918ULL);
static const unsigned char t2[2] = {16,17};
hasher.Write(t2, 2);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x4bc1b3f0968dd39cull);
static const unsigned char t3[9] = {18,19,20,21,22,23,24,25,26};
hasher.Write(t3, 9);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x2f2e6163076bcfadull);
static const unsigned char t4[5] = {27,28,29,30,31};
hasher.Write(t4, 5);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x7127512f72f27cceull);
hasher.Write(0x2726252423222120ULL);
BOOST_CHECK_EQUAL(hasher.Finalize(), 0x0e3ea96b5304a7d0ull);
@ -66,6 +107,22 @@ BOOST_AUTO_TEST_CASE(siphash)
BOOST_CHECK_EQUAL(SipHashUint256(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL, uint256S("1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403020100")), 0x7127512f72f27cceull);
// Check test vectors from spec, one byte at a time
CSipHasher hasher2(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL);
for (uint8_t x=0; x<ARRAYLEN(siphash_4_2_testvec); ++x)
{
BOOST_CHECK_EQUAL(hasher2.Finalize(), siphash_4_2_testvec[x]);
hasher2.Write(&x, 1);
}
// Check test vectors from spec, eight bytes at a time
CSipHasher hasher3(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL);
for (uint8_t x=0; x<ARRAYLEN(siphash_4_2_testvec); x+=8)
{
BOOST_CHECK_EQUAL(hasher3.Finalize(), siphash_4_2_testvec[x]);
hasher3.Write(uint64_t(x)|(uint64_t(x+1)<<8)|(uint64_t(x+2)<<16)|(uint64_t(x+3)<<24)|
(uint64_t(x+4)<<32)|(uint64_t(x+5)<<40)|(uint64_t(x+6)<<48)|(uint64_t(x+7)<<56));
}
// Check consistency between CSipHasher and SipHashUint256[Extra].
// TODO reenable when backporting Bitcoin #10321
/*FastRandomContext ctx;