mirror of
https://github.com/dashpay/dash.git
synced 2024-12-25 03:52:49 +01:00
Compressed headers implementation. (#4497)
* Compressed headers implementation. First header is always compressed in a headers2 msg Version is uncompressed if it’s not matched within the last 7 unique versions to be sent in the current msg Service flag to signal that the peer supports compressed headers If compressed headers services is active, the peer will receive headers compressed If both sendheaders and sendheaders2 are sent, the peer will respond with compressed headers Functional tests as for uncompressed headers Updates regarding the existing functional tests to use the compressed headers if the NODE_HEADERS_COMPRESSED service flag is active * style: add missing comma Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com> Co-authored-by: PastaPastaPasta <6443210+PastaPastaPasta@users.noreply.github.com> Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>
This commit is contained in:
gabriel-bjg
GitHub
parent
afd098cbd1
commit
52ddf5c453
@ -1200,7 +1200,7 @@ namespace { // Variables internal to initialization process only
|
||||
int nMaxConnections;
|
||||
int nUserMaxConnections;
|
||||
int nFD;
|
||||
ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK | NODE_NETWORK_LIMITED);
|
||||
ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK | NODE_NETWORK_LIMITED | NODE_HEADERS_COMPRESSED);
|
||||
int64_t peer_connect_timeout;
|
||||
std::set<BlockFilterType> g_enabled_filter_types;
|
||||
|
||||
|
||||
@ -31,6 +31,7 @@
|
||||
#include <util/strencodings.h>
|
||||
#include <util/validation.h>
|
||||
|
||||
#include <list>
|
||||
#include <memory>
|
||||
|
||||
#include <spork.h>
|
||||
@ -298,6 +299,8 @@ struct CNodeState {
|
||||
bool fPreferredDownload;
|
||||
//! Whether this peer wants invs or headers (when possible) for block announcements.
|
||||
bool fPreferHeaders;
|
||||
//! Whether this peer wants invs or compressed headers (when possible) for block announcements.
|
||||
bool fPreferHeadersCompressed;
|
||||
//! Whether this peer wants invs or cmpctblocks (when possible) for block announcements.
|
||||
bool fPreferHeaderAndIDs;
|
||||
//! Whether this peer will send us cmpctblocks if we request them
|
||||
@ -418,6 +421,7 @@ struct CNodeState {
|
||||
nBlocksInFlightValidHeaders = 0;
|
||||
fPreferredDownload = false;
|
||||
fPreferHeaders = false;
|
||||
fPreferHeadersCompressed = false;
|
||||
fPreferHeaderAndIDs = false;
|
||||
fProvidesHeaderAndIDs = false;
|
||||
fSupportsDesiredCmpctVersion = false;
|
||||
@ -1868,10 +1872,12 @@ bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::ve
|
||||
// nUnconnectingHeaders gets reset back to 0.
|
||||
if (!LookupBlockIndex(headers[0].hashPrevBlock) && nCount < MAX_BLOCKS_TO_ANNOUNCE) {
|
||||
nodestate->nUnconnectingHeaders++;
|
||||
connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256()));
|
||||
LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n",
|
||||
std::string strCommand = (pfrom->nServices & NODE_HEADERS_COMPRESSED) ? NetMsgType::GETHEADERS2 : NetMsgType::GETHEADERS;
|
||||
connman->PushMessage(pfrom, msgMaker.Make(strCommand, ::ChainActive().GetLocator(pindexBestHeader), uint256()));
|
||||
LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending %s (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n",
|
||||
headers[0].GetHash().ToString(),
|
||||
headers[0].hashPrevBlock.ToString(),
|
||||
strCommand,
|
||||
pindexBestHeader->nHeight,
|
||||
pfrom->GetId(), nodestate->nUnconnectingHeaders);
|
||||
// Set hashLastUnknownBlock for this peer, so that if we
|
||||
@ -1973,8 +1979,9 @@ bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::ve
|
||||
// Headers message had its maximum size; the peer may have more headers.
|
||||
// TODO: optimize: if pindexLast is an ancestor of ::ChainActive().Tip or pindexBestHeader, continue
|
||||
// from there instead.
|
||||
LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight);
|
||||
connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexLast), uint256()));
|
||||
std::string strCommand = (pfrom->nServices & NODE_HEADERS_COMPRESSED) ? NetMsgType::GETHEADERS2 : NetMsgType::GETHEADERS;
|
||||
LogPrint(BCLog::NET, "more %s (%d) to end to peer=%d (startheight:%d)\n", strCommand, pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight);
|
||||
connman->PushMessage(pfrom, msgMaker.Make(strCommand, ::ChainActive().GetLocator(pindexLast), uint256()));
|
||||
}
|
||||
|
||||
bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus());
|
||||
@ -2640,7 +2647,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr
|
||||
// We send this to non-NODE NETWORK peers as well, because even
|
||||
// non-NODE NETWORK peers can announce blocks (such as pruning
|
||||
// nodes)
|
||||
connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::SENDHEADERS));
|
||||
connman->PushMessage(pfrom, msgMaker.Make((pfrom->nServices & NODE_HEADERS_COMPRESSED) ? NetMsgType::SENDHEADERS2 : NetMsgType::SENDHEADERS));
|
||||
|
||||
if (pfrom->CanRelay()) {
|
||||
// Tell our peer we are willing to provide version-1 cmpctblocks
|
||||
@ -2762,7 +2769,13 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr
|
||||
return true;
|
||||
}
|
||||
|
||||
if (strCommand == NetMsgType::SENDCMPCT) {
|
||||
if (strCommand == NetMsgType::SENDHEADERS2) {
|
||||
LOCK(cs_main);
|
||||
State(pfrom->GetId())->fPreferHeadersCompressed = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (strCommand == NetMsgType::SENDCMPCT) {
|
||||
bool fAnnounceUsingCMPCTBLOCK = false;
|
||||
uint64_t nCMPCTBLOCKVersion = 1;
|
||||
vRecv >> fAnnounceUsingCMPCTBLOCK >> nCMPCTBLOCKVersion;
|
||||
@ -2851,8 +2864,9 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr
|
||||
// fell back to inv we probably have a reorg which we should get the headers for first,
|
||||
// we now only provide a getheaders response here. When we receive the headers, we will
|
||||
// then ask for the blocks we need.
|
||||
connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), inv.hash));
|
||||
LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->GetId());
|
||||
std::string strCommand = (pfrom->nServices & NODE_HEADERS_COMPRESSED) ? NetMsgType::GETHEADERS2 : NetMsgType::GETHEADERS;
|
||||
connman->PushMessage(pfrom, msgMaker.Make(strCommand, ::ChainActive().GetLocator(pindexBestHeader), inv.hash));
|
||||
LogPrint(BCLog::NET, "%s (%d) %s to peer=%d\n", strCommand, pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->GetId());
|
||||
}
|
||||
}
|
||||
else
|
||||
@ -3016,20 +3030,20 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr
|
||||
return true;
|
||||
}
|
||||
|
||||
if (strCommand == NetMsgType::GETHEADERS) {
|
||||
if (strCommand == NetMsgType::GETHEADERS || strCommand == NetMsgType::GETHEADERS2) {
|
||||
CBlockLocator locator;
|
||||
uint256 hashStop;
|
||||
vRecv >> locator >> hashStop;
|
||||
|
||||
if (locator.vHave.size() > MAX_LOCATOR_SZ) {
|
||||
LogPrint(BCLog::NET, "getheaders locator size %lld > %d, disconnect peer=%d\n", locator.vHave.size(), MAX_LOCATOR_SZ, pfrom->GetId());
|
||||
LogPrint(BCLog::NET, "%s locator size %lld > %d, disconnect peer=%d\n", strCommand, locator.vHave.size(), MAX_LOCATOR_SZ, pfrom->GetId());
|
||||
pfrom->fDisconnect = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
LOCK(cs_main);
|
||||
if (::ChainstateActive().IsInitialBlockDownload() && !pfrom->HasPermission(PF_NOBAN)) {
|
||||
LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom->GetId());
|
||||
LogPrint(BCLog::NET, "Ignoring %s from peer=%d because node is in initial block download\n", strCommand, pfrom->GetId());
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -3056,30 +3070,46 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr
|
||||
pindex = ::ChainActive().Next(pindex);
|
||||
}
|
||||
|
||||
// we must use CBlocks, as CBlockHeaders won't include the 0x00 nTx count at the end
|
||||
std::vector<CBlock> vHeaders;
|
||||
int nLimit = MAX_HEADERS_RESULTS;
|
||||
LogPrint(BCLog::NET, "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom->GetId());
|
||||
for (; pindex; pindex = ::ChainActive().Next(pindex))
|
||||
{
|
||||
vHeaders.push_back(pindex->GetBlockHeader());
|
||||
if (--nLimit <= 0 || pindex->GetBlockHash() == hashStop)
|
||||
break;
|
||||
const auto send_headers = [&hashStop, &pindex, &nodestate, &connman, &pfrom, &msgMaker](auto msg_type, auto& v_headers, auto callback) {
|
||||
int nLimit = MAX_HEADERS_RESULTS;
|
||||
for (; pindex; pindex = ::ChainActive().Next(pindex)) {
|
||||
v_headers.push_back(callback(pindex));
|
||||
|
||||
if (--nLimit <= 0 || pindex->GetBlockHash() == hashStop)
|
||||
break;
|
||||
}
|
||||
// pindex can be nullptr either if we sent ::ChainActive().Tip() OR
|
||||
// if our peer has ::ChainActive().Tip() (and thus we are sending an empty
|
||||
// headers message). In both cases it's safe to update
|
||||
// pindexBestHeaderSent to be our tip.
|
||||
//
|
||||
// It is important that we simply reset the BestHeaderSent value here,
|
||||
// and not max(BestHeaderSent, newHeaderSent). We might have announced
|
||||
// the currently-being-connected tip using a compact block, which
|
||||
// resulted in the peer sending a headers request, which we respond to
|
||||
// without the new block. By resetting the BestHeaderSent, we ensure we
|
||||
// will re-announce the new block via headers (or compact blocks again)
|
||||
// in the SendMessages logic.
|
||||
nodestate->pindexBestHeaderSent = pindex ? pindex : ::ChainActive().Tip();
|
||||
connman->PushMessage(pfrom, msgMaker.Make(msg_type, v_headers));
|
||||
};
|
||||
|
||||
LogPrint(BCLog::NET, "%s %d to %s from peer=%d\n", strCommand, (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom->GetId());
|
||||
if (strCommand == NetMsgType::GETHEADERS) {
|
||||
// we must use CBlocks, as CBlockHeaders won't include the 0x00 nTx count at the end
|
||||
std::vector<CBlock> v_headers;
|
||||
send_headers(NetMsgType::HEADERS, v_headers, [](const auto block_pindex) { return block_pindex->GetBlockHeader(); });
|
||||
} else if (strCommand == NetMsgType::GETHEADERS2) {
|
||||
// Keeps track of the last 7 unique version blocks
|
||||
std::list<int32_t> last_unique_versions;
|
||||
std::vector<CompressibleBlockHeader> v_headers;
|
||||
|
||||
send_headers(NetMsgType::HEADERS2, v_headers, [&v_headers, &last_unique_versions](const auto block_pindex) {
|
||||
CompressibleBlockHeader compressible_header{block_pindex->GetBlockHeader()};
|
||||
if (!v_headers.empty()) compressible_header.Compress(v_headers, last_unique_versions); // first block is always uncompressed
|
||||
return compressible_header;
|
||||
});
|
||||
}
|
||||
// pindex can be nullptr either if we sent ::ChainActive().Tip() OR
|
||||
// if our peer has ::ChainActive().Tip() (and thus we are sending an empty
|
||||
// headers message). In both cases it's safe to update
|
||||
// pindexBestHeaderSent to be our tip.
|
||||
//
|
||||
// It is important that we simply reset the BestHeaderSent value here,
|
||||
// and not max(BestHeaderSent, newHeaderSent). We might have announced
|
||||
// the currently-being-connected tip using a compact block, which
|
||||
// resulted in the peer sending a headers request, which we respond to
|
||||
// without the new block. By resetting the BestHeaderSent, we ensure we
|
||||
// will re-announce the new block via headers (or compact blocks again)
|
||||
// in the SendMessages logic.
|
||||
nodestate->pindexBestHeaderSent = pindex ? pindex : ::ChainActive().Tip();
|
||||
connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders));
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -3318,7 +3348,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr
|
||||
if (!LookupBlockIndex(cmpctblock.header.hashPrevBlock)) {
|
||||
// Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers
|
||||
if (!::ChainstateActive().IsInitialBlockDownload())
|
||||
connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256()));
|
||||
connman->PushMessage(pfrom, msgMaker.Make((pfrom->nServices & NODE_HEADERS_COMPRESSED) ? NetMsgType::GETHEADERS2 : NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256()));
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -3601,8 +3631,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr
|
||||
return true;
|
||||
}
|
||||
|
||||
if (strCommand == NetMsgType::HEADERS)
|
||||
{
|
||||
if (strCommand == NetMsgType::HEADERS || strCommand == NetMsgType::HEADERS2) {
|
||||
// Ignore headers received while importing
|
||||
if (fImporting || fReindex) {
|
||||
LogPrint(BCLog::NET, "Unexpected headers message received from peer %d\n", pfrom->GetId());
|
||||
@ -3618,10 +3647,21 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr
|
||||
Misbehaving(pfrom->GetId(), 20, strprintf("headers message size = %u", nCount));
|
||||
return false;
|
||||
}
|
||||
headers.resize(nCount);
|
||||
for (unsigned int n = 0; n < nCount; n++) {
|
||||
vRecv >> headers[n];
|
||||
ReadCompactSize(vRecv); // ignore tx count; assume it is 0.
|
||||
|
||||
if (strCommand == NetMsgType::HEADERS) {
|
||||
headers.resize(nCount);
|
||||
for (unsigned int n = 0; n < nCount; n++) {
|
||||
vRecv >> headers[n];
|
||||
ReadCompactSize(vRecv); // ignore tx count; assume it is 0.
|
||||
}
|
||||
} else if (strCommand == NetMsgType::HEADERS2) {
|
||||
std::list<int32_t> last_unique_versions;
|
||||
for (unsigned int n = 0; n < nCount; n++) {
|
||||
CompressibleBlockHeader block_header_compressed;
|
||||
vRecv >> block_header_compressed;
|
||||
block_header_compressed.Uncompress(headers, last_unique_versions);
|
||||
headers.push_back(block_header_compressed);
|
||||
}
|
||||
}
|
||||
|
||||
// Headers received via a HEADERS message should be valid, and reflect
|
||||
@ -4149,8 +4189,13 @@ void PeerLogicValidation::ConsiderEviction(CNode *pto, int64_t time_in_seconds)
|
||||
pto->fDisconnect = true;
|
||||
} else {
|
||||
assert(state.m_chain_sync.m_work_header);
|
||||
LogPrint(BCLog::NET, "sending getheaders to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n", pto->GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>", state.m_chain_sync.m_work_header->GetBlockHash().ToString());
|
||||
connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(state.m_chain_sync.m_work_header->pprev), uint256()));
|
||||
std::string strCommand = (pto->nServices & NODE_HEADERS_COMPRESSED) ? NetMsgType::GETHEADERS2 : NetMsgType::GETHEADERS;
|
||||
LogPrint(BCLog::NET, "sending %s to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n",
|
||||
strCommand,
|
||||
pto->GetId(),
|
||||
state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>",
|
||||
state.m_chain_sync.m_work_header->GetBlockHash().ToString());
|
||||
connman->PushMessage(pto, msgMaker.Make(strCommand, ::ChainActive().GetLocator(state.m_chain_sync.m_work_header->pprev), uint256()));
|
||||
state.m_chain_sync.m_sent_getheaders = true;
|
||||
constexpr int64_t HEADERS_RESPONSE_TIME = 120; // 2 minutes
|
||||
// Bump the timeout to allow a response, which could clear the timeout
|
||||
@ -4375,8 +4420,9 @@ bool PeerLogicValidation::SendMessages(CNode* pto)
|
||||
got back an empty response. */
|
||||
if (pindexStart->pprev)
|
||||
pindexStart = pindexStart->pprev;
|
||||
LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), pto->nStartingHeight);
|
||||
connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexStart), uint256()));
|
||||
std::string strCommand = (pto->nServices & NODE_HEADERS_COMPRESSED) ? NetMsgType::GETHEADERS2 : NetMsgType::GETHEADERS;
|
||||
LogPrint(BCLog::NET, "initial %s (%d) to peer=%d (startheight:%d)\n", strCommand, pindexStart->nHeight, pto->GetId(), pto->nStartingHeight);
|
||||
connman->PushMessage(pto, msgMaker.Make(strCommand, ::ChainActive().GetLocator(pindexStart), uint256()));
|
||||
}
|
||||
}
|
||||
|
||||
@ -4393,7 +4439,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto)
|
||||
// add all to the inv queue.
|
||||
LOCK(pto->cs_inventory);
|
||||
std::vector<CBlock> vHeaders;
|
||||
bool fRevertToInv = ((!state.fPreferHeaders &&
|
||||
bool fRevertToInv = ((!state.fPreferHeaders && !state.fPreferHeadersCompressed &&
|
||||
(!state.fPreferHeaderAndIDs || pto->vBlockHashesToAnnounce.size() > 1)) ||
|
||||
pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE);
|
||||
const CBlockIndex *pBestIndex = nullptr; // last header queued for delivery
|
||||
@ -4478,6 +4524,20 @@ bool PeerLogicValidation::SendMessages(CNode* pto)
|
||||
connman->PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock));
|
||||
}
|
||||
state.pindexBestHeaderSent = pBestIndex;
|
||||
} else if (state.fPreferHeadersCompressed) {
|
||||
std::vector<CompressibleBlockHeader> vHeadersCompressed;
|
||||
std::list<int32_t> last_unique_versions;
|
||||
|
||||
// Save other headers compressed
|
||||
std::for_each(vHeaders.cbegin(), vHeaders.cend(), [&vHeadersCompressed, &last_unique_versions](const auto& block) {
|
||||
CompressibleBlockHeader compressible_header{block.GetBlockHeader()};
|
||||
compressible_header.Compress(vHeadersCompressed, last_unique_versions);
|
||||
vHeadersCompressed.push_back(compressible_header);
|
||||
});
|
||||
|
||||
// Push message to peer
|
||||
connman->PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS2, vHeadersCompressed));
|
||||
state.pindexBestHeaderSent = pBestIndex;
|
||||
} else if (state.fPreferHeaders) {
|
||||
if (vHeaders.size() > 1) {
|
||||
LogPrint(BCLog::NET, "%s: %u headers, range (%s, %s), to peer=%d\n", __func__,
|
||||
|
||||
@ -32,3 +32,102 @@ std::string CBlock::ToString() const
|
||||
}
|
||||
return s.str();
|
||||
}
|
||||
|
||||
static void MarkVersionAsMostRecent(std::list<int32_t>& last_unique_versions, std::list<int32_t>::const_iterator version_it)
|
||||
{
|
||||
if (version_it != last_unique_versions.cbegin()) {
|
||||
// Move the found version to the front of the list
|
||||
last_unique_versions.splice(last_unique_versions.begin(), last_unique_versions, version_it, std::next(version_it));
|
||||
}
|
||||
}
|
||||
|
||||
static void SaveVersionAsMostRecent(std::list<int32_t>& last_unique_versions, const int32_t version)
|
||||
{
|
||||
last_unique_versions.push_front(version);
|
||||
|
||||
// Always keep the last 7 unique versions
|
||||
constexpr std::size_t max_backwards_look_ups = 7;
|
||||
if (last_unique_versions.size() > max_backwards_look_ups) {
|
||||
// Evict the oldest version
|
||||
last_unique_versions.pop_back();
|
||||
}
|
||||
}
|
||||
|
||||
void CompressibleBlockHeader::Compress(const std::vector<CompressibleBlockHeader>& previous_blocks, std::list<int32_t>& last_unique_versions)
|
||||
{
|
||||
if (previous_blocks.empty()) {
|
||||
// Previous block not available, we have to send the block completely uncompressed
|
||||
SaveVersionAsMostRecent(last_unique_versions, nVersion);
|
||||
return;
|
||||
}
|
||||
|
||||
// Try to compress version
|
||||
const auto version_it = std::find(last_unique_versions.cbegin(), last_unique_versions.cend(), nVersion);
|
||||
if (version_it != last_unique_versions.cend()) {
|
||||
// Version is found in the last 7 unique blocks.
|
||||
bit_field.SetVersionOffset(static_cast<uint8_t>(std::distance(last_unique_versions.cbegin(), version_it) + 1));
|
||||
|
||||
// Mark the version as the most recent one
|
||||
MarkVersionAsMostRecent(last_unique_versions, version_it);
|
||||
} else {
|
||||
// Save the version as the most recent one
|
||||
SaveVersionAsMostRecent(last_unique_versions, nVersion);
|
||||
}
|
||||
|
||||
// Previous block is available
|
||||
const auto& last_block = previous_blocks.back();
|
||||
bit_field.MarkAsCompressed(CompressedHeaderBitField::Flag::PREV_BLOCK_HASH);
|
||||
|
||||
// Compute compressed time diff
|
||||
const int64_t time_diff = nTime - last_block.nTime;
|
||||
if (time_diff <= std::numeric_limits<int16_t>::max() && time_diff >= std::numeric_limits<int16_t>::min()) {
|
||||
time_offset = static_cast<int16_t>(time_diff);
|
||||
bit_field.MarkAsCompressed(CompressedHeaderBitField::Flag::TIMESTAMP);
|
||||
}
|
||||
|
||||
// If n_bits matches previous block, it can be compressed (not sent at all)
|
||||
if (nBits == last_block.nBits) {
|
||||
bit_field.MarkAsCompressed(CompressedHeaderBitField::Flag::NBITS);
|
||||
}
|
||||
}
|
||||
|
||||
void CompressibleBlockHeader::Uncompress(const std::vector<CBlockHeader>& previous_blocks, std::list<int32_t>& last_unique_versions)
|
||||
{
|
||||
if (previous_blocks.empty()) {
|
||||
// First block in chain is always uncompressed
|
||||
SaveVersionAsMostRecent(last_unique_versions, nVersion);
|
||||
return;
|
||||
}
|
||||
|
||||
// We have the previous block
|
||||
const auto& last_block = previous_blocks.back();
|
||||
|
||||
// Uncompress version
|
||||
if (bit_field.IsVersionCompressed()) {
|
||||
const auto version_offset = bit_field.GetVersionOffset();
|
||||
if (version_offset <= last_unique_versions.size()) {
|
||||
auto version_it = last_unique_versions.begin();
|
||||
std::advance(version_it, version_offset - 1);
|
||||
nVersion = *version_it;
|
||||
MarkVersionAsMostRecent(last_unique_versions, version_it);
|
||||
}
|
||||
} else {
|
||||
// Save the version as the most recent one
|
||||
SaveVersionAsMostRecent(last_unique_versions, nVersion);
|
||||
}
|
||||
|
||||
// Uncompress prev block hash
|
||||
if (bit_field.IsCompressed(CompressedHeaderBitField::Flag::PREV_BLOCK_HASH)) {
|
||||
hashPrevBlock = last_block.GetHash();
|
||||
}
|
||||
|
||||
// Uncompress timestamp
|
||||
if (bit_field.IsCompressed(CompressedHeaderBitField::Flag::TIMESTAMP)) {
|
||||
nTime = last_block.nTime + time_offset;
|
||||
}
|
||||
|
||||
// Uncompress n_bits
|
||||
if (bit_field.IsCompressed(CompressedHeaderBitField::Flag::NBITS)) {
|
||||
nBits = last_block.nBits;
|
||||
}
|
||||
}
|
||||
|
||||
@ -6,9 +6,12 @@
|
||||
#ifndef BITCOIN_PRIMITIVES_BLOCK_H
|
||||
#define BITCOIN_PRIMITIVES_BLOCK_H
|
||||
|
||||
#include <list>
|
||||
#include <primitives/transaction.h>
|
||||
#include <serialize.h>
|
||||
#include <uint256.h>
|
||||
#include <cstddef>
|
||||
#include <type_traits>
|
||||
|
||||
/** Nodes collect new transactions into a block, hash them into a hash tree,
|
||||
* and scan through nonce values to make the block's hash satisfy proof-of-work
|
||||
@ -58,6 +61,127 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
class CompressedHeaderBitField
|
||||
{
|
||||
std::byte bit_field{0};
|
||||
|
||||
public:
|
||||
enum class Flag : std::underlying_type_t<std::byte> {
|
||||
VERSION_BIT_0 = (1 << 0),
|
||||
VERSION_BIT_1 = (1 << 1),
|
||||
VERSION_BIT_2 = (1 << 2),
|
||||
PREV_BLOCK_HASH = (1 << 3),
|
||||
TIMESTAMP = (1 << 4),
|
||||
NBITS = (1 << 5),
|
||||
};
|
||||
|
||||
inline bool IsCompressed(Flag flag) const
|
||||
{
|
||||
return (bit_field & to_byte(flag)) == to_byte(0);
|
||||
}
|
||||
|
||||
inline void MarkAsUncompressed(Flag flag)
|
||||
{
|
||||
bit_field |= to_byte(flag);
|
||||
}
|
||||
|
||||
inline void MarkAsCompressed(Flag flag)
|
||||
{
|
||||
bit_field &= ~to_byte(flag);
|
||||
}
|
||||
|
||||
inline bool IsVersionCompressed() const
|
||||
{
|
||||
return GetVersionOffset() != 0;
|
||||
}
|
||||
|
||||
inline void SetVersionOffset(uint8_t version)
|
||||
{
|
||||
bit_field &= ~VERSION_BIT_MASK;
|
||||
bit_field |= to_byte(version) & VERSION_BIT_MASK;
|
||||
}
|
||||
|
||||
inline uint8_t GetVersionOffset() const
|
||||
{
|
||||
return to_uint8(bit_field & VERSION_BIT_MASK);
|
||||
}
|
||||
|
||||
template <typename Stream>
|
||||
void Serialize(Stream& s) const
|
||||
{
|
||||
::Serialize(s, to_uint8(bit_field));
|
||||
}
|
||||
|
||||
template <typename Stream>
|
||||
void Unserialize(Stream& s)
|
||||
{
|
||||
uint8_t new_bit_field_value;
|
||||
::Unserialize(s, new_bit_field_value);
|
||||
bit_field = to_byte(new_bit_field_value);
|
||||
}
|
||||
|
||||
private:
|
||||
static constexpr uint8_t to_uint8(const std::byte value)
|
||||
{
|
||||
return std::to_integer<uint8_t>(value);
|
||||
}
|
||||
|
||||
static constexpr std::byte to_byte(const uint8_t value)
|
||||
{
|
||||
return std::byte{value};
|
||||
}
|
||||
|
||||
static constexpr std::byte to_byte(const Flag flag)
|
||||
{
|
||||
return std::byte{flag};
|
||||
}
|
||||
|
||||
static constexpr std::byte VERSION_BIT_MASK = std::byte{Flag::VERSION_BIT_0} | std::byte{Flag::VERSION_BIT_1} | std::byte{Flag::VERSION_BIT_2};
|
||||
};
|
||||
|
||||
struct CompressibleBlockHeader : CBlockHeader {
|
||||
CompressedHeaderBitField bit_field;
|
||||
int16_t time_offset{0};
|
||||
|
||||
CompressibleBlockHeader() = default;
|
||||
|
||||
explicit CompressibleBlockHeader(CBlockHeader&& block_header)
|
||||
{
|
||||
static_assert(std::is_trivially_copyable_v<CBlockHeader>, "If CBlockHeader is not trivially copyable, please consider using std::move on the next line");
|
||||
*static_cast<CBlockHeader*>(this) = block_header;
|
||||
|
||||
// When we create this from a block header, mark everything as uncompressed
|
||||
bit_field.SetVersionOffset(0);
|
||||
bit_field.MarkAsUncompressed(CompressedHeaderBitField::Flag::PREV_BLOCK_HASH);
|
||||
bit_field.MarkAsUncompressed(CompressedHeaderBitField::Flag::TIMESTAMP);
|
||||
bit_field.MarkAsUncompressed(CompressedHeaderBitField::Flag::NBITS);
|
||||
}
|
||||
|
||||
SERIALIZE_METHODS(CompressibleBlockHeader, obj)
|
||||
{
|
||||
READWRITE(obj.bit_field);
|
||||
if (!obj.bit_field.IsVersionCompressed()) {
|
||||
READWRITE(obj.nVersion);
|
||||
}
|
||||
if (!obj.bit_field.IsCompressed(CompressedHeaderBitField::Flag::PREV_BLOCK_HASH)) {
|
||||
READWRITE(obj.hashPrevBlock);
|
||||
}
|
||||
READWRITE(obj.hashMerkleRoot);
|
||||
if (!obj.bit_field.IsCompressed(CompressedHeaderBitField::Flag::TIMESTAMP)) {
|
||||
READWRITE(obj.nTime);
|
||||
} else {
|
||||
READWRITE(obj.time_offset);
|
||||
}
|
||||
if (!obj.bit_field.IsCompressed(CompressedHeaderBitField::Flag::NBITS)) {
|
||||
READWRITE(obj.nBits);
|
||||
}
|
||||
READWRITE(obj.nNonce);
|
||||
}
|
||||
|
||||
void Compress(const std::vector<CompressibleBlockHeader>& previous_blocks, std::list<int32_t>& last_unique_versions);
|
||||
|
||||
void Uncompress(const std::vector<CBlockHeader>& previous_blocks, std::list<int32_t>& last_unique_versions);
|
||||
};
|
||||
|
||||
class CBlock : public CBlockHeader
|
||||
{
|
||||
|
||||
@ -86,6 +86,9 @@ MAKE_MSG(CLSIG, "clsig");
|
||||
MAKE_MSG(ISLOCK, "islock");
|
||||
MAKE_MSG(ISDLOCK, "isdlock");
|
||||
MAKE_MSG(MNAUTH, "mnauth");
|
||||
MAKE_MSG(GETHEADERS2, "getheaders2");
|
||||
MAKE_MSG(SENDHEADERS2, "sendheaders2");
|
||||
MAKE_MSG(HEADERS2, "headers2");
|
||||
}; // namespace NetMsgType
|
||||
|
||||
/** All known message types. Keep this in the same order as the list of
|
||||
@ -164,7 +167,9 @@ const static std::string allNetMessageTypes[] = {
|
||||
NetMsgType::ISLOCK,
|
||||
NetMsgType::ISDLOCK,
|
||||
NetMsgType::MNAUTH,
|
||||
};
|
||||
NetMsgType::GETHEADERS2,
|
||||
NetMsgType::SENDHEADERS2,
|
||||
NetMsgType::HEADERS2};
|
||||
const static std::vector<std::string> allNetMessageTypesVec(allNetMessageTypes, allNetMessageTypes+ARRAYLEN(allNetMessageTypes));
|
||||
|
||||
CMessageHeader::CMessageHeader(const MessageStartChars& pchMessageStartIn)
|
||||
@ -345,6 +350,7 @@ static std::string serviceFlagToStr(size_t bit)
|
||||
case NODE_BLOOM: return "BLOOM";
|
||||
case NODE_COMPACT_FILTERS: return "COMPACT_FILTERS";
|
||||
case NODE_NETWORK_LIMITED: return "NETWORK_LIMITED";
|
||||
case NODE_HEADERS_COMPRESSED: return "HEADERS_COMPRESSED";
|
||||
// Not using default, so we get warned when a case is missing
|
||||
}
|
||||
|
||||
|
||||
@ -296,6 +296,9 @@ extern const char *CLSIG;
|
||||
extern const char *ISLOCK;
|
||||
extern const char *ISDLOCK;
|
||||
extern const char *MNAUTH;
|
||||
extern const char *GETHEADERS2;
|
||||
extern const char *SENDHEADERS2;
|
||||
extern const char *HEADERS2;
|
||||
};
|
||||
|
||||
/* Get a vector of all valid message types (see above) */
|
||||
@ -324,6 +327,8 @@ enum ServiceFlags : uint64_t {
|
||||
// serving the last 288 blocks
|
||||
// See BIP159 for details on how this is implemented.
|
||||
NODE_NETWORK_LIMITED = (1 << 10),
|
||||
// description will be provided
|
||||
NODE_HEADERS_COMPRESSED = (1 << 11),
|
||||
|
||||
// Bits 24-31 are reserved for temporary experiments. Just pick a bit that
|
||||
// isn't getting used, or one not being used much, and notify the
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
import random
|
||||
|
||||
from test_framework.blocktools import create_block, create_coinbase
|
||||
from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxOut, FromHex, HeaderAndShortIDs, msg_block, msg_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, ToHex
|
||||
from test_framework.messages import BlockTransactions, BlockTransactionsRequest, calculate_shortid, CBlock, CBlockHeader, CInv, COutPoint, CTransaction, CTxIn, CTxOut, FromHex, HeaderAndShortIDs, msg_block, msg_blocktxn, msg_cmpctblock, msg_getblocktxn, msg_getdata, msg_getheaders, msg_headers, msg_inv, msg_sendcmpct, msg_sendheaders, msg_tx, NODE_NETWORK, P2PHeaderAndShortIDs, PrefilledTransaction, ToHex, NODE_HEADERS_COMPRESSED
|
||||
from test_framework.mininode import mininode_lock, P2PInterface
|
||||
from test_framework.script import CScript, OP_TRUE, OP_DROP
|
||||
from test_framework.test_framework import BitcoinTestFramework
|
||||
@ -344,7 +344,8 @@ class CompactBlocksTest(BitcoinTestFramework):
|
||||
|
||||
if announce == "inv":
|
||||
test_node.send_message(msg_inv([CInv(2, block.sha256)]))
|
||||
wait_until(lambda: "getheaders" in test_node.last_message, timeout=30, lock=mininode_lock)
|
||||
getheaders_key = "getheaders2" if test_node.nServices & NODE_HEADERS_COMPRESSED else "getheaders"
|
||||
wait_until(lambda: getheaders_key in test_node.last_message, timeout=30, lock=mininode_lock)
|
||||
test_node.send_header_for_blocks([block])
|
||||
else:
|
||||
test_node.send_header_for_blocks([block])
|
||||
@ -720,8 +721,8 @@ class CompactBlocksTest(BitcoinTestFramework):
|
||||
def run_test(self):
|
||||
# Setup the p2p connections
|
||||
self.test_node = self.nodes[0].add_p2p_connection(TestP2PConn())
|
||||
self.second_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK)
|
||||
self.old_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK)
|
||||
self.second_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK | NODE_HEADERS_COMPRESSED)
|
||||
self.old_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK | NODE_HEADERS_COMPRESSED)
|
||||
|
||||
# We will need UTXOs to construct transactions in later tests.
|
||||
self.make_utxos()
|
||||
|
||||
@ -8,7 +8,7 @@ Tests that a node configured with -prune=550 signals NODE_NETWORK_LIMITED correc
|
||||
and that it responds to getdata requests for blocks correctly:
|
||||
- send a block within 288 + 2 of the tip
|
||||
- disconnect peers who request blocks older than that."""
|
||||
from test_framework.messages import CInv, msg_getdata, NODE_BLOOM, NODE_NETWORK_LIMITED, msg_verack
|
||||
from test_framework.messages import CInv, msg_getdata, NODE_BLOOM, NODE_NETWORK_LIMITED, NODE_HEADERS_COMPRESSED, msg_verack
|
||||
from test_framework.mininode import P2PInterface, mininode_lock
|
||||
from test_framework.test_framework import BitcoinTestFramework
|
||||
from test_framework.util import assert_equal, disconnect_nodes, connect_nodes, sync_blocks, wait_until
|
||||
@ -49,7 +49,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework):
|
||||
def run_test(self):
|
||||
node = self.nodes[0].add_p2p_connection(P2PIgnoreInv())
|
||||
|
||||
expected_services = NODE_BLOOM | NODE_NETWORK_LIMITED
|
||||
expected_services = NODE_BLOOM | NODE_NETWORK_LIMITED | NODE_HEADERS_COMPRESSED
|
||||
|
||||
self.log.info("Check that node has signalled expected services.")
|
||||
assert_equal(node.nServices, expected_services)
|
||||
@ -77,7 +77,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework):
|
||||
|
||||
node1.wait_for_addr()
|
||||
#must relay address with NODE_NETWORK_LIMITED
|
||||
assert_equal(node1.firstAddrnServices, 1028) # Not 1036 like bitcoin, because NODE_WITNESS = 1 << 3 = 8
|
||||
assert_equal(node1.firstAddrnServices, expected_services)
|
||||
|
||||
self.nodes[0].disconnect_p2ps()
|
||||
node1.wait_for_disconnect()
|
||||
|
||||
@ -86,7 +86,7 @@ e. Announce one more that doesn't connect.
|
||||
Expect: disconnect.
|
||||
"""
|
||||
from test_framework.blocktools import create_block, create_coinbase
|
||||
from test_framework.messages import CInv
|
||||
from test_framework.messages import CInv, NODE_HEADERS_COMPRESSED
|
||||
from test_framework.mininode import (
|
||||
CBlockHeader,
|
||||
P2PInterface,
|
||||
@ -240,7 +240,7 @@ class SendHeadersTest(BitcoinTestFramework):
|
||||
inv_node = self.nodes[0].add_p2p_connection(BaseNode())
|
||||
# Make sure NODE_NETWORK is not set for test_node, so no block download
|
||||
# will occur outside of direct fetching
|
||||
test_node = self.nodes[0].add_p2p_connection(BaseNode(), services=0)
|
||||
test_node = self.nodes[0].add_p2p_connection(BaseNode(), services=NODE_HEADERS_COMPRESSED)
|
||||
|
||||
# Ensure verack's have been processed by our peer
|
||||
inv_node.sync_with_ping()
|
||||
@ -547,7 +547,7 @@ class SendHeadersTest(BitcoinTestFramework):
|
||||
height += 1
|
||||
# Send the header of the second block -> this won't connect.
|
||||
with mininode_lock:
|
||||
test_node.last_message.pop("getheaders", None)
|
||||
test_node.last_message.pop("getheaders2" if test_node.nServices & NODE_HEADERS_COMPRESSED else "getheaders", None)
|
||||
test_node.send_header_for_blocks([blocks[1]])
|
||||
test_node.wait_for_getheaders()
|
||||
test_node.send_header_for_blocks(blocks)
|
||||
@ -570,7 +570,7 @@ class SendHeadersTest(BitcoinTestFramework):
|
||||
for i in range(1, MAX_UNCONNECTING_HEADERS):
|
||||
# Send a header that doesn't connect, check that we get a getheaders.
|
||||
with mininode_lock:
|
||||
test_node.last_message.pop("getheaders", None)
|
||||
test_node.last_message.pop("getheaders2" if test_node.nServices & NODE_HEADERS_COMPRESSED else "getheaders", None)
|
||||
test_node.send_header_for_blocks([blocks[i]])
|
||||
test_node.wait_for_getheaders()
|
||||
|
||||
@ -585,7 +585,7 @@ class SendHeadersTest(BitcoinTestFramework):
|
||||
for i in range(5 * MAX_UNCONNECTING_HEADERS - 1):
|
||||
# Send a header that doesn't connect, check that we get a getheaders.
|
||||
with mininode_lock:
|
||||
test_node.last_message.pop("getheaders", None)
|
||||
test_node.last_message.pop("getheaders2" if test_node.nServices & NODE_HEADERS_COMPRESSED else "getheaders", None)
|
||||
test_node.send_header_for_blocks([blocks[i % len(blocks)]])
|
||||
test_node.wait_for_getheaders()
|
||||
|
||||
|
||||
602
test/functional/p2p_sendheaders_compressed.py
Normal file
602
test/functional/p2p_sendheaders_compressed.py
Normal file
@ -0,0 +1,602 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2021 The Dash Core developers
|
||||
# Distributed under the MIT software license, see the accompanying
|
||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
"""Test behavior of headers messages to announce blocks.
|
||||
|
||||
Setup:
|
||||
|
||||
- Two nodes:
|
||||
- node0 is the node-under-test. We create two p2p connections to it. The
|
||||
first p2p connection is a control and should only ever receive inv's. The
|
||||
second p2p connection tests the headers sending logic.
|
||||
- node1 is used to create reorgs.
|
||||
"""
|
||||
from test_framework.blocktools import create_block, create_coinbase
|
||||
from test_framework.messages import CInv, NODE_HEADERS_COMPRESSED
|
||||
from test_framework.mininode import (
|
||||
CompressibleBlockHeader,
|
||||
P2PInterface,
|
||||
mininode_lock,
|
||||
msg_block,
|
||||
msg_getblocks,
|
||||
msg_getdata,
|
||||
msg_getheaders2,
|
||||
msg_headers2,
|
||||
msg_inv,
|
||||
msg_sendheaders2,
|
||||
)
|
||||
from test_framework.test_framework import BitcoinTestFramework
|
||||
from test_framework.util import (
|
||||
assert_equal,
|
||||
wait_until,
|
||||
)
|
||||
|
||||
DIRECT_FETCH_RESPONSE_TIME = 0.05
|
||||
|
||||
|
||||
class BaseNode(P2PInterface):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
self.block_announced = False
|
||||
self.last_blockhash_announced = None
|
||||
self.recent_headers_announced = []
|
||||
|
||||
def send_get_data(self, block_hashes):
|
||||
"""Request data for a list of block hashes."""
|
||||
msg = msg_getdata()
|
||||
for block_hash in block_hashes:
|
||||
msg.inv.append(CInv(2, block_hash))
|
||||
self.send_message(msg)
|
||||
|
||||
def send_get_headers(self, locator, hashstop):
|
||||
msg = msg_getheaders2()
|
||||
msg.locator.vHave = locator
|
||||
msg.hashstop = hashstop
|
||||
self.send_message(msg)
|
||||
|
||||
def send_block_inv(self, blockhash):
|
||||
msg = msg_inv()
|
||||
msg.inv = [CInv(2, blockhash)]
|
||||
self.send_message(msg)
|
||||
|
||||
def send_header_for_blocks(self, new_blocks):
|
||||
new_blocks_compressible = [CompressibleBlockHeader(h) for h in new_blocks] if new_blocks else None
|
||||
headers_message = msg_headers2(new_blocks_compressible)
|
||||
self.send_message(headers_message)
|
||||
|
||||
def send_getblocks(self, locator):
|
||||
getblocks_message = msg_getblocks()
|
||||
getblocks_message.locator.vHave = locator
|
||||
self.send_message(getblocks_message)
|
||||
|
||||
def wait_for_getdata(self, hash_list, timeout=60):
|
||||
if not hash_list:
|
||||
return
|
||||
|
||||
test_function = lambda: "getdata" in self.last_message and [inv.hash for inv in self.last_message["getdata"].inv] == hash_list
|
||||
wait_until(test_function, timeout=timeout, lock=mininode_lock)
|
||||
|
||||
def wait_for_block_announcement(self, block_hash, timeout=60):
|
||||
test_function = lambda: self.last_blockhash_announced == block_hash
|
||||
wait_until(test_function, timeout=timeout, lock=mininode_lock)
|
||||
|
||||
def on_inv(self, message):
|
||||
self.block_announced = True
|
||||
self.last_blockhash_announced = message.inv[-1].hash
|
||||
|
||||
def on_headers2(self, message):
|
||||
if len(message.headers):
|
||||
self.block_announced = True
|
||||
for header in message.headers:
|
||||
header.calc_sha256()
|
||||
# append because headers may be announced over multiple messages.
|
||||
self.recent_headers_announced.append(header.sha256)
|
||||
self.last_blockhash_announced = message.headers[-1].sha256
|
||||
|
||||
def clear_block_announcements(self):
|
||||
with mininode_lock:
|
||||
self.block_announced = False
|
||||
self.last_message.pop("inv", None)
|
||||
self.last_message.pop("headers2", None)
|
||||
self.recent_headers_announced = []
|
||||
|
||||
def check_last_headers_announcement(self, headers):
|
||||
"""Test whether the last headers announcements received are right.
|
||||
Headers may be announced across more than one message."""
|
||||
test_function = lambda: (len(self.recent_headers_announced) >= len(headers))
|
||||
wait_until(test_function, timeout=60, lock=mininode_lock)
|
||||
with mininode_lock:
|
||||
assert_equal(self.recent_headers_announced, headers)
|
||||
self.block_announced = False
|
||||
self.last_message.pop("headers2", None)
|
||||
self.recent_headers_announced = []
|
||||
|
||||
def check_last_inv_announcement(self, inv):
|
||||
"""Test whether the last announcement received had the right inv.
|
||||
inv should be a list of block hashes."""
|
||||
|
||||
test_function = lambda: self.block_announced
|
||||
wait_until(test_function, timeout=60, lock=mininode_lock)
|
||||
|
||||
with mininode_lock:
|
||||
compare_inv = []
|
||||
if "inv" in self.last_message:
|
||||
compare_inv = [inv.hash for inv in self.last_message["inv"].inv]
|
||||
assert_equal(compare_inv, inv)
|
||||
self.block_announced = False
|
||||
self.last_message.pop("inv", None)
|
||||
|
||||
|
||||
class SendHeadersTest(BitcoinTestFramework):
|
||||
def set_test_params(self):
|
||||
self.setup_clean_chain = True
|
||||
self.num_nodes = 2
|
||||
|
||||
def mine_blocks(self, count):
|
||||
"""Mine count blocks and return the new tip."""
|
||||
|
||||
# Clear out block announcements from each p2p listener
|
||||
[p2p.clear_block_announcements() for p2p in self.nodes[0].p2ps]
|
||||
self.nodes[0].generate(count)
|
||||
return int(self.nodes[0].getbestblockhash(), 16)
|
||||
|
||||
def mine_reorg(self, length):
|
||||
"""Mine a reorg that invalidates length blocks (replacing them with # length+1 blocks).
|
||||
|
||||
Note: we clear the state of our p2p connections after the
|
||||
to-be-reorged-out blocks are mined, so that we don't break later tests.
|
||||
return the list of block hashes newly mined."""
|
||||
|
||||
self.nodes[0].generate(length) # make sure all invalidated blocks are node0's
|
||||
self.sync_blocks(self.nodes, wait=0.1)
|
||||
for p2p in self.nodes[0].p2ps:
|
||||
p2p.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16))
|
||||
p2p.clear_block_announcements()
|
||||
|
||||
tip_height = self.nodes[1].getblockcount()
|
||||
hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1))
|
||||
self.nodes[1].invalidateblock(hash_to_invalidate)
|
||||
all_hashes = self.nodes[1].generate(length + 1) # Must be longer than the orig chain
|
||||
self.sync_blocks(self.nodes, wait=0.1)
|
||||
return [int(hash_value, 16) for hash_value in all_hashes]
|
||||
|
||||
def run_test(self):
|
||||
# Setup the p2p connections
|
||||
inv_node = self.nodes[0].add_p2p_connection(BaseNode())
|
||||
# Make sure NODE_NETWORK is not set for test_node, so no block download
|
||||
# will occur outside of direct fetching
|
||||
test_node = self.nodes[0].add_p2p_connection(BaseNode(), services=NODE_HEADERS_COMPRESSED)
|
||||
|
||||
# Ensure verack's have been processed by our peer
|
||||
inv_node.sync_with_ping()
|
||||
test_node.sync_with_ping()
|
||||
|
||||
self.test_null_locators(test_node, inv_node)
|
||||
self.test_nonnull_locators(test_node, inv_node)
|
||||
|
||||
def test_null_locators(self, test_node, inv_node):
|
||||
"""
|
||||
Sends two getheaders requests with null locator values. First request's hashstop
|
||||
value refers to validated block, while second request's hashstop value refers to
|
||||
a block which hasn't been validated. Verifies only the first request returns
|
||||
headers.
|
||||
"""
|
||||
tip = self.nodes[0].getblockheader(self.nodes[0].generate(1)[0])
|
||||
tip_hash = int(tip["hash"], 16)
|
||||
|
||||
inv_node.check_last_inv_announcement(inv=[tip_hash])
|
||||
test_node.check_last_inv_announcement(inv=[tip_hash])
|
||||
|
||||
self.log.info("Verify getheaders with null locator and valid hashstop returns headers.")
|
||||
test_node.clear_block_announcements()
|
||||
test_node.send_get_headers(locator=[], hashstop=tip_hash)
|
||||
test_node.check_last_headers_announcement(headers=[tip_hash])
|
||||
|
||||
self.log.info("Verify getheaders with null locator and invalid hashstop does not return headers.")
|
||||
block = create_block(int(tip["hash"], 16), create_coinbase(tip["height"] + 1), tip["mediantime"] + 1)
|
||||
block.solve()
|
||||
test_node.send_header_for_blocks([block])
|
||||
test_node.clear_block_announcements()
|
||||
test_node.send_get_headers(locator=[], hashstop=int(block.hash, 16))
|
||||
test_node.sync_with_ping()
|
||||
assert_equal(test_node.block_announced, False)
|
||||
inv_node.clear_block_announcements()
|
||||
test_node.send_message(msg_block(block))
|
||||
inv_node.check_last_inv_announcement(inv=[int(block.hash, 16)])
|
||||
|
||||
def test_nonnull_locators(self, test_node, inv_node):
|
||||
"""
|
||||
Part 1: No headers announcements before "sendheaders"
|
||||
a. node mines a block [expect: inv]
|
||||
send getdata for the block [expect: block]
|
||||
b. node mines another block [expect: inv]
|
||||
send getheaders and getdata [expect: headers, then block]
|
||||
c. node mines another block [expect: inv]
|
||||
peer mines a block, announces with header [expect: getdata]
|
||||
d. node mines another block [expect: inv]
|
||||
|
||||
Part 2: After "sendheaders", headers announcements should generally work.
|
||||
a. peer sends sendheaders [expect: no response]
|
||||
peer sends getheaders with current tip [expect: no response]
|
||||
b. node mines a block [expect: tip header]
|
||||
c. for N in 1, ..., 10:
|
||||
* for announce-type in {inv, header}
|
||||
- peer mines N blocks, announces with announce-type
|
||||
[ expect: getheaders/getdata or getdata, deliver block(s) ]
|
||||
- node mines a block [ expect: 1 header ]
|
||||
|
||||
Part 3: Headers announcements stop after large reorg and resume after getheaders or inv from peer.
|
||||
- For response-type in {inv, getheaders}
|
||||
* node mines a 7 block reorg [ expect: headers announcement of 8 blocks ]
|
||||
* node mines an 8-block reorg [ expect: inv at tip ]
|
||||
* peer responds with getblocks/getdata [expect: inv, blocks ]
|
||||
* node mines another block [ expect: inv at tip, peer sends getdata, expect: block ]
|
||||
* node mines another block at tip [ expect: inv ]
|
||||
* peer responds with getheaders with an old hashstop more than 8 blocks back [expect: headers]
|
||||
* peer requests block [ expect: block ]
|
||||
* node mines another block at tip [ expect: inv, peer sends getdata, expect: block ]
|
||||
* peer sends response-type [expect headers if getheaders, getheaders/getdata if mining new block]
|
||||
* node mines 1 block [expect: 1 header, peer responds with getdata]
|
||||
|
||||
Part 4: Test direct fetch behavior
|
||||
a. Announce 2 old block headers.
|
||||
Expect: no getdata requests.
|
||||
b. Announce 3 new blocks via 1 headers message.
|
||||
Expect: one getdata request for all 3 blocks.
|
||||
(Send blocks.)
|
||||
c. Announce 1 header that forks off the last two blocks.
|
||||
Expect: no response.
|
||||
d. Announce 1 more header that builds on that fork.
|
||||
Expect: one getdata request for two blocks.
|
||||
e. Announce 16 more headers that build on that fork.
|
||||
Expect: getdata request for 14 more blocks.
|
||||
f. Announce 1 more header that builds on that fork.
|
||||
Expect: no response.
|
||||
|
||||
Part 5: Test handling of headers that don't connect.
|
||||
a. Repeat 10 times:
|
||||
1. Announce a header that doesn't connect.
|
||||
Expect: getheaders message
|
||||
2. Send headers chain.
|
||||
Expect: getdata for the missing blocks, tip update.
|
||||
b. Then send 9 more headers that don't connect.
|
||||
Expect: getheaders message each time.
|
||||
c. Announce a header that does connect.
|
||||
Expect: no response.
|
||||
d. Announce 49 headers that don't connect.
|
||||
Expect: getheaders message each time.
|
||||
e. Announce one more that doesn't connect.
|
||||
Expect: disconnect.
|
||||
"""
|
||||
tip = int(self.nodes[0].getbestblockhash(), 16)
|
||||
|
||||
# PART 1
|
||||
# 1. Mine a block; expect inv announcements each time
|
||||
self.log.info("Part 1: headers don't start before sendheaders message...")
|
||||
for i in range(4):
|
||||
self.log.debug("Part 1.{}: starting...".format(i))
|
||||
old_tip = tip
|
||||
tip = self.mine_blocks(1)
|
||||
inv_node.check_last_inv_announcement(inv=[tip])
|
||||
test_node.check_last_inv_announcement(inv=[tip])
|
||||
# Try a few different responses; none should affect next announcement
|
||||
if i == 0:
|
||||
# first request the block
|
||||
test_node.send_get_data([tip])
|
||||
test_node.wait_for_block(tip)
|
||||
elif i == 1:
|
||||
# next try requesting header and block
|
||||
test_node.send_get_headers(locator=[old_tip], hashstop=tip)
|
||||
test_node.send_get_data([tip])
|
||||
test_node.wait_for_block(tip)
|
||||
test_node.clear_block_announcements() # since we requested headers...
|
||||
elif i == 2:
|
||||
# this time announce own block via headers
|
||||
inv_node.clear_block_announcements()
|
||||
height = self.nodes[0].getblockcount()
|
||||
last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time']
|
||||
block_time = last_time + 1
|
||||
new_block = create_block(tip, create_coinbase(height + 1), block_time)
|
||||
new_block.solve()
|
||||
test_node.send_header_for_blocks([new_block])
|
||||
test_node.wait_for_getdata([new_block.sha256])
|
||||
test_node.send_message(msg_block(new_block))
|
||||
test_node.sync_with_ping() # make sure this block is processed
|
||||
wait_until(lambda: inv_node.block_announced, timeout=60, lock=mininode_lock)
|
||||
inv_node.clear_block_announcements()
|
||||
test_node.clear_block_announcements()
|
||||
|
||||
self.log.info("Part 1: success!")
|
||||
self.log.info("Part 2: announce blocks with headers after sendheaders message...")
|
||||
# PART 2
|
||||
# 2. Send a sendheaders message and test that headers announcements
|
||||
# commence and keep working.
|
||||
test_node.send_message(msg_sendheaders2())
|
||||
prev_tip = int(self.nodes[0].getbestblockhash(), 16)
|
||||
test_node.send_get_headers(locator=[prev_tip], hashstop=0)
|
||||
test_node.sync_with_ping()
|
||||
|
||||
# Now that we've synced headers, headers announcements should work
|
||||
tip = self.mine_blocks(1)
|
||||
inv_node.check_last_inv_announcement(inv=[tip])
|
||||
test_node.check_last_headers_announcement(headers=[tip])
|
||||
|
||||
height = self.nodes[0].getblockcount() + 1
|
||||
block_time += 10 # Advance far enough ahead
|
||||
for i in range(10):
|
||||
self.log.debug("Part 2.{}: starting...".format(i))
|
||||
# Mine i blocks, and alternate announcing either via
|
||||
# inv (of tip) or via headers. After each, new blocks
|
||||
# mined by the node should successfully be announced
|
||||
# with block header, even though the blocks are never requested
|
||||
for j in range(2):
|
||||
self.log.debug("Part 2.{}.{}: starting...".format(i, j))
|
||||
blocks = []
|
||||
for _ in range(i + 1):
|
||||
blocks.append(create_block(tip, create_coinbase(height), block_time))
|
||||
blocks[-1].solve()
|
||||
tip = blocks[-1].sha256
|
||||
block_time += 1
|
||||
height += 1
|
||||
if j == 0:
|
||||
# Announce via inv
|
||||
test_node.send_block_inv(tip)
|
||||
test_node.wait_for_getheaders()
|
||||
# Should have received a getheaders now
|
||||
test_node.send_header_for_blocks(blocks)
|
||||
# Test that duplicate inv's won't result in duplicate
|
||||
# getdata requests, or duplicate headers announcements
|
||||
[inv_node.send_block_inv(block.sha256) for block in blocks]
|
||||
test_node.wait_for_getdata([block.sha256 for block in blocks])
|
||||
inv_node.sync_with_ping()
|
||||
else:
|
||||
# Announce via headers
|
||||
test_node.send_header_for_blocks(blocks)
|
||||
test_node.wait_for_getdata([block.sha256 for block in blocks])
|
||||
# Test that duplicate headers won't result in duplicate
|
||||
# getdata requests (the check is further down)
|
||||
inv_node.send_header_for_blocks(blocks)
|
||||
inv_node.sync_with_ping()
|
||||
[test_node.send_message(msg_block(block)) for block in blocks]
|
||||
test_node.sync_with_ping()
|
||||
inv_node.sync_with_ping()
|
||||
# This block should not be announced to the inv node (since it also
|
||||
# broadcast it)
|
||||
assert "inv" not in inv_node.last_message
|
||||
assert "headers2" not in inv_node.last_message
|
||||
tip = self.mine_blocks(1)
|
||||
inv_node.check_last_inv_announcement(inv=[tip])
|
||||
test_node.check_last_headers_announcement(headers=[tip])
|
||||
height += 1
|
||||
block_time += 1
|
||||
|
||||
self.log.info("Part 2: success!")
|
||||
|
||||
self.log.info("Part 3: headers announcements can stop after large reorg, and resume after headers/inv from peer...")
|
||||
|
||||
# PART 3. Headers announcements can stop after large reorg, and resume after
|
||||
# getheaders or inv from peer.
|
||||
for j in range(2):
|
||||
self.log.debug("Part 3.{}: starting...".format(j))
|
||||
# First try mining a reorg that can propagate with header announcement
|
||||
new_block_hashes = self.mine_reorg(length=7)
|
||||
tip = new_block_hashes[-1]
|
||||
inv_node.check_last_inv_announcement(inv=[tip])
|
||||
test_node.check_last_headers_announcement(headers=new_block_hashes)
|
||||
|
||||
block_time += 8
|
||||
|
||||
# Mine a too-large reorg, which should be announced with a single inv
|
||||
new_block_hashes = self.mine_reorg(length=8)
|
||||
tip = new_block_hashes[-1]
|
||||
inv_node.check_last_inv_announcement(inv=[tip])
|
||||
test_node.check_last_inv_announcement(inv=[tip])
|
||||
|
||||
block_time += 9
|
||||
|
||||
fork_point = self.nodes[0].getblock("%02x" % new_block_hashes[0])["previousblockhash"]
|
||||
fork_point = int(fork_point, 16)
|
||||
|
||||
# Use getblocks/getdata
|
||||
test_node.send_getblocks(locator=[fork_point])
|
||||
test_node.check_last_inv_announcement(inv=new_block_hashes)
|
||||
test_node.send_get_data(new_block_hashes)
|
||||
test_node.wait_for_block(new_block_hashes[-1])
|
||||
|
||||
for i in range(3):
|
||||
self.log.debug("Part 3.{}.{}: starting...".format(j, i))
|
||||
|
||||
# Mine another block, still should get only an inv
|
||||
tip = self.mine_blocks(1)
|
||||
inv_node.check_last_inv_announcement(inv=[tip])
|
||||
test_node.check_last_inv_announcement(inv=[tip])
|
||||
if i == 0:
|
||||
# Just get the data -- shouldn't cause headers announcements to resume
|
||||
test_node.send_get_data([tip])
|
||||
test_node.wait_for_block(tip)
|
||||
elif i == 1:
|
||||
# Send a getheaders message that shouldn't trigger headers announcements
|
||||
# to resume (best header sent will be too old)
|
||||
test_node.send_get_headers(locator=[fork_point], hashstop=new_block_hashes[1])
|
||||
test_node.send_get_data([tip])
|
||||
test_node.wait_for_block(tip)
|
||||
elif i == 2:
|
||||
# This time, try sending either a getheaders to trigger resumption
|
||||
# of headers announcements, or mine a new block and inv it, also
|
||||
# triggering resumption of headers announcements.
|
||||
test_node.send_get_data([tip])
|
||||
test_node.wait_for_block(tip)
|
||||
if j == 0:
|
||||
test_node.send_get_headers(locator=[tip], hashstop=0)
|
||||
test_node.sync_with_ping()
|
||||
else:
|
||||
test_node.send_block_inv(tip)
|
||||
test_node.sync_with_ping()
|
||||
# New blocks should now be announced with header
|
||||
tip = self.mine_blocks(1)
|
||||
inv_node.check_last_inv_announcement(inv=[tip])
|
||||
test_node.check_last_headers_announcement(headers=[tip])
|
||||
|
||||
self.log.info("Part 3: success!")
|
||||
|
||||
self.log.info("Part 4: Testing direct fetch behavior...")
|
||||
tip = self.mine_blocks(1)
|
||||
height = self.nodes[0].getblockcount() + 1
|
||||
last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time']
|
||||
block_time = last_time + 1
|
||||
|
||||
# Create 2 blocks. Send the blocks, then send the headers.
|
||||
blocks = []
|
||||
for _ in range(2):
|
||||
blocks.append(create_block(tip, create_coinbase(height), block_time))
|
||||
blocks[-1].solve()
|
||||
tip = blocks[-1].sha256
|
||||
block_time += 1
|
||||
height += 1
|
||||
inv_node.send_message(msg_block(blocks[-1]))
|
||||
|
||||
inv_node.sync_with_ping() # Make sure blocks are processed
|
||||
test_node.last_message.pop("getdata", None)
|
||||
test_node.send_header_for_blocks(blocks)
|
||||
test_node.sync_with_ping()
|
||||
# should not have received any getdata messages
|
||||
with mininode_lock:
|
||||
assert "getdata" not in test_node.last_message
|
||||
|
||||
# This time, direct fetch should work
|
||||
blocks = []
|
||||
for _ in range(3):
|
||||
blocks.append(create_block(tip, create_coinbase(height), block_time))
|
||||
blocks[-1].solve()
|
||||
tip = blocks[-1].sha256
|
||||
block_time += 1
|
||||
height += 1
|
||||
|
||||
test_node.send_header_for_blocks(blocks)
|
||||
test_node.sync_with_ping()
|
||||
test_node.wait_for_getdata([block.sha256 for block in blocks], timeout=DIRECT_FETCH_RESPONSE_TIME)
|
||||
|
||||
[test_node.send_message(msg_block(block)) for block in blocks]
|
||||
|
||||
test_node.sync_with_ping()
|
||||
|
||||
# Now announce a header that forks the last two blocks
|
||||
tip = blocks[0].sha256
|
||||
height -= 2
|
||||
blocks = []
|
||||
|
||||
# Create extra blocks for later
|
||||
for _ in range(20):
|
||||
blocks.append(create_block(tip, create_coinbase(height), block_time))
|
||||
blocks[-1].solve()
|
||||
tip = blocks[-1].sha256
|
||||
block_time += 1
|
||||
height += 1
|
||||
|
||||
# Announcing one block on fork should not trigger direct fetch
|
||||
# (less work than tip)
|
||||
test_node.last_message.pop("getdata", None)
|
||||
test_node.send_header_for_blocks(blocks[0:1])
|
||||
test_node.sync_with_ping()
|
||||
with mininode_lock:
|
||||
assert "getdata" not in test_node.last_message
|
||||
|
||||
# Announcing one more block on fork should trigger direct fetch for
|
||||
# both blocks (same work as tip)
|
||||
test_node.send_header_for_blocks(blocks[1:2])
|
||||
test_node.sync_with_ping()
|
||||
test_node.wait_for_getdata([block.sha256 for block in blocks[0:2]], timeout=DIRECT_FETCH_RESPONSE_TIME)
|
||||
|
||||
# Announcing 16 more headers should trigger direct fetch for 14 more
|
||||
# blocks
|
||||
test_node.send_header_for_blocks(blocks[2:18])
|
||||
test_node.sync_with_ping()
|
||||
test_node.wait_for_getdata([block.sha256 for block in blocks[2:16]], timeout=DIRECT_FETCH_RESPONSE_TIME)
|
||||
|
||||
# Announcing 1 more header should not trigger any response
|
||||
test_node.last_message.pop("getdata", None)
|
||||
test_node.send_header_for_blocks(blocks[18:19])
|
||||
test_node.sync_with_ping()
|
||||
with mininode_lock:
|
||||
assert "getdata" not in test_node.last_message
|
||||
|
||||
self.log.info("Part 4: success!")
|
||||
|
||||
# Now deliver all those blocks we announced.
|
||||
[test_node.send_message(msg_block(block)) for block in blocks]
|
||||
|
||||
self.log.info("Part 5: Testing handling of unconnecting headers")
|
||||
# First we test that receipt of an unconnecting header doesn't prevent
|
||||
# chain sync.
|
||||
for i in range(10):
|
||||
self.log.debug("Part 5.{}: starting...".format(i))
|
||||
test_node.last_message.pop("getdata", None)
|
||||
blocks = []
|
||||
# Create two more blocks.
|
||||
for _ in range(2):
|
||||
blocks.append(create_block(tip, create_coinbase(height), block_time))
|
||||
blocks[-1].solve()
|
||||
tip = blocks[-1].sha256
|
||||
block_time += 1
|
||||
height += 1
|
||||
# Send the header of the second block -> this won't connect.
|
||||
with mininode_lock:
|
||||
test_node.last_message.pop("getheaders2", None)
|
||||
test_node.send_header_for_blocks([blocks[1]])
|
||||
test_node.wait_for_getheaders()
|
||||
test_node.send_header_for_blocks(blocks)
|
||||
test_node.wait_for_getdata([block.sha256 for block in blocks])
|
||||
[test_node.send_message(msg_block(block)) for block in blocks]
|
||||
test_node.sync_with_ping()
|
||||
assert_equal(int(self.nodes[0].getbestblockhash(), 16), blocks[1].sha256)
|
||||
|
||||
blocks = []
|
||||
# Now we test that if we repeatedly don't send connecting headers, we
|
||||
# don't go into an infinite loop trying to get them to connect.
|
||||
MAX_UNCONNECTING_HEADERS = 10
|
||||
for _ in range(MAX_UNCONNECTING_HEADERS + 1):
|
||||
blocks.append(create_block(tip, create_coinbase(height), block_time))
|
||||
blocks[-1].solve()
|
||||
tip = blocks[-1].sha256
|
||||
block_time += 1
|
||||
height += 1
|
||||
|
||||
for i in range(1, MAX_UNCONNECTING_HEADERS):
|
||||
# Send a header that doesn't connect, check that we get a getheaders.
|
||||
with mininode_lock:
|
||||
test_node.last_message.pop("getheaders2", None)
|
||||
test_node.send_header_for_blocks([blocks[i]])
|
||||
test_node.wait_for_getheaders()
|
||||
|
||||
# Next header will connect, should re-set our count:
|
||||
test_node.send_header_for_blocks([blocks[0]])
|
||||
|
||||
# Remove the first two entries (blocks[1] would connect):
|
||||
blocks = blocks[2:]
|
||||
|
||||
# Now try to see how many unconnecting headers we can send
|
||||
# before we get disconnected. Should be 5*MAX_UNCONNECTING_HEADERS
|
||||
for i in range(5 * MAX_UNCONNECTING_HEADERS - 1):
|
||||
# Send a header that doesn't connect, check that we get a getheaders.
|
||||
with mininode_lock:
|
||||
test_node.last_message.pop("getheaders2", None)
|
||||
test_node.send_header_for_blocks([blocks[i % len(blocks)]])
|
||||
test_node.wait_for_getheaders()
|
||||
|
||||
# Eventually this stops working.
|
||||
test_node.send_header_for_blocks([blocks[-1]])
|
||||
|
||||
# Should get disconnected
|
||||
test_node.wait_for_disconnect()
|
||||
|
||||
self.log.info("Part 5: success!")
|
||||
|
||||
# Finally, check that the inv node never received a getdata request,
|
||||
# throughout the test
|
||||
assert "getdata" not in inv_node.last_message
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
SendHeadersTest().main()
|
||||
@ -24,6 +24,7 @@ from test_framework.messages import (
|
||||
NODE_NETWORK,
|
||||
NODE_GETUTXO,NODE_BLOOM,
|
||||
NODE_NETWORK_LIMITED,
|
||||
NODE_HEADERS_COMPRESSED,
|
||||
)
|
||||
|
||||
|
||||
@ -42,6 +43,8 @@ def assert_net_servicesnames(servicesflag, servicenames):
|
||||
assert "BLOOM" in servicenames
|
||||
if servicesflag & NODE_NETWORK_LIMITED:
|
||||
assert "NETWORK_LIMITED" in servicenames
|
||||
if servicesflag & NODE_HEADERS_COMPRESSED:
|
||||
assert "HEADERS_COMPRESSED" in servicenames
|
||||
|
||||
|
||||
class NetTest(BitcoinTestFramework):
|
||||
@ -126,7 +129,7 @@ class NetTest(BitcoinTestFramework):
|
||||
# check the `servicesnames` field
|
||||
network_info = [node.getnetworkinfo() for node in self.nodes]
|
||||
for info in network_info:
|
||||
assert_net_servicesnames(int(info["localservices"]), info["localservicesnames"])
|
||||
assert_net_servicesnames(int(info["localservices"], 16), info["localservicesnames"])
|
||||
|
||||
def _test_getaddednodeinfo(self):
|
||||
assert_equal(self.nodes[0].getaddednodeinfo(), [])
|
||||
@ -148,7 +151,7 @@ class NetTest(BitcoinTestFramework):
|
||||
assert_equal(peer_info[1][0]['addrbind'], peer_info[0][0]['addr'])
|
||||
# check the `servicesnames` field
|
||||
for info in peer_info:
|
||||
assert_net_servicesnames(int(info[0]["services"]), info[0]["servicesnames"])
|
||||
assert_net_servicesnames(int(info[0]["services"], 16), info[0]["servicesnames"])
|
||||
|
||||
def test_service_flags(self):
|
||||
self.nodes[0].add_p2p_connection(P2PInterface(), services=(1 << 4) | (1 << 63))
|
||||
|
||||
@ -48,6 +48,7 @@ NODE_GETUTXO = (1 << 1)
|
||||
NODE_BLOOM = (1 << 2)
|
||||
NODE_COMPACT_FILTERS = (1 << 6)
|
||||
NODE_NETWORK_LIMITED = (1 << 10)
|
||||
NODE_HEADERS_COMPRESSED = (1 << 11)
|
||||
|
||||
FILTER_TYPE_BASIC = 0
|
||||
|
||||
@ -630,6 +631,177 @@ class CBlock(CBlockHeader):
|
||||
% (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot,
|
||||
time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx))
|
||||
|
||||
|
||||
class CompressibleBlockHeader:
|
||||
__slots__ = ("bitfield", "timeOffset", "nVersion", "hashPrevBlock", "hashMerkleRoot", "nTime", "nBits", "nNonce",
|
||||
"hash", "sha256")
|
||||
|
||||
FLAG_VERSION_BIT_0 = 1 << 0
|
||||
FLAG_VERSION_BIT_1 = 1 << 1
|
||||
FLAG_VERSION_BIT_2 = 1 << 2
|
||||
FLAG_PREV_BLOCK_HASH = 1 << 3
|
||||
FLAG_TIMESTAMP = 1 << 4
|
||||
FLAG_NBITS = 1 << 5
|
||||
|
||||
BITMASK_VERSION = FLAG_VERSION_BIT_0 | FLAG_VERSION_BIT_1 | FLAG_VERSION_BIT_2
|
||||
|
||||
def __init__(self, header=None):
|
||||
if header is None:
|
||||
self.set_null()
|
||||
else:
|
||||
self.bitfield = 0
|
||||
self.timeOffset = 0
|
||||
self.nVersion = header.nVersion
|
||||
self.hashPrevBlock = header.hashPrevBlock
|
||||
self.hashMerkleRoot = header.hashMerkleRoot
|
||||
self.nTime = header.nTime
|
||||
self.nBits = header.nBits
|
||||
self.nNonce = header.nNonce
|
||||
self.hash = None
|
||||
self.sha256 = None
|
||||
self.calc_sha256()
|
||||
|
||||
def set_null(self):
|
||||
self.bitfield = 0
|
||||
self.timeOffset = 0
|
||||
self.nVersion = 0
|
||||
self.hashPrevBlock = 0
|
||||
self.hashMerkleRoot = 0
|
||||
self.nTime = 0
|
||||
self.nBits = 0
|
||||
self.nNonce = 0
|
||||
self.hash = None
|
||||
self.sha256 = None
|
||||
|
||||
def deserialize(self, f):
|
||||
self.bitfield = struct.unpack("<B", f.read(1))[0]
|
||||
if self.bitfield & self.BITMASK_VERSION == 0:
|
||||
self.nVersion = struct.unpack("<i", f.read(4))[0]
|
||||
if self.bitfield & self.FLAG_PREV_BLOCK_HASH:
|
||||
self.hashPrevBlock = deser_uint256(f)
|
||||
self.hashMerkleRoot = deser_uint256(f)
|
||||
if self.bitfield & self.FLAG_TIMESTAMP:
|
||||
self.nTime = struct.unpack("<I", f.read(4))[0]
|
||||
else:
|
||||
self.timeOffset = struct.unpack("<h", f.read(2))[0]
|
||||
if self.bitfield & self.FLAG_NBITS:
|
||||
self.nBits = struct.unpack("<I", f.read(4))[0]
|
||||
self.nNonce = struct.unpack("<I", f.read(4))[0]
|
||||
self.rehash()
|
||||
|
||||
def serialize(self):
|
||||
r = b""
|
||||
r += struct.pack("<B", self.bitfield)
|
||||
if not self.bitfield & self.BITMASK_VERSION:
|
||||
r += struct.pack("<i", self.nVersion)
|
||||
if self.bitfield & self.FLAG_PREV_BLOCK_HASH:
|
||||
r += ser_uint256(self.hashPrevBlock)
|
||||
r += ser_uint256(self.hashMerkleRoot)
|
||||
r += struct.pack("<I", self.nTime) if self.bitfield & self.FLAG_TIMESTAMP else struct.pack("<h", self.timeOffset)
|
||||
if self.bitfield & self.FLAG_NBITS:
|
||||
r += struct.pack("<I", self.nBits)
|
||||
r += struct.pack("<I", self.nNonce)
|
||||
return r
|
||||
|
||||
def calc_sha256(self):
|
||||
if self.sha256 is None:
|
||||
r = b""
|
||||
r += struct.pack("<i", self.nVersion)
|
||||
r += ser_uint256(self.hashPrevBlock)
|
||||
r += ser_uint256(self.hashMerkleRoot)
|
||||
r += struct.pack("<I", self.nTime)
|
||||
r += struct.pack("<I", self.nBits)
|
||||
r += struct.pack("<I", self.nNonce)
|
||||
self.sha256 = uint256_from_str(dashhash(r))
|
||||
self.hash = int(encode(dashhash(r)[::-1], 'hex_codec'), 16)
|
||||
|
||||
def rehash(self):
|
||||
self.sha256 = None
|
||||
self.calc_sha256()
|
||||
return self.sha256
|
||||
|
||||
def __repr__(self):
|
||||
return "BlockHeaderCompressed(bitfield=%064x, nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s " \
|
||||
"nBits=%08x nNonce=%08x timeOffset=%i)" % \
|
||||
(self.bitfield, self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, time.ctime(self.nTime), self.nBits, self.nNonce, self.timeOffset)
|
||||
|
||||
def __save_version_as_most_recent(self, last_unique_versions):
|
||||
last_unique_versions.insert(0, self.nVersion)
|
||||
|
||||
# Evict the oldest version
|
||||
if len(last_unique_versions) > 7:
|
||||
last_unique_versions.pop()
|
||||
|
||||
@staticmethod
|
||||
def __mark_version_as_most_recent(last_unique_versions, version_idx):
|
||||
# Move version to the front of the list
|
||||
last_unique_versions.insert(0, last_unique_versions.pop(version_idx))
|
||||
|
||||
def compress(self, last_blocks, last_unique_versions):
|
||||
if not last_blocks:
|
||||
# First block, everything must be uncompressed
|
||||
self.bitfield &= (~CompressibleBlockHeader.BITMASK_VERSION)
|
||||
self.bitfield |= CompressibleBlockHeader.FLAG_PREV_BLOCK_HASH
|
||||
self.bitfield |= CompressibleBlockHeader.FLAG_TIMESTAMP
|
||||
self.bitfield |= CompressibleBlockHeader.FLAG_NBITS
|
||||
self.__save_version_as_most_recent(last_unique_versions)
|
||||
return
|
||||
|
||||
# Compress version
|
||||
try:
|
||||
version_idx = last_unique_versions.index(self.nVersion)
|
||||
version_offset = len(last_unique_versions) - version_idx
|
||||
self.bitfield &= (~CompressibleBlockHeader.BITMASK_VERSION)
|
||||
self.bitfield |= (version_offset & CompressibleBlockHeader.BITMASK_VERSION)
|
||||
self.__mark_version_as_most_recent(last_unique_versions, version_idx)
|
||||
except ValueError:
|
||||
self.__save_version_as_most_recent(last_unique_versions)
|
||||
|
||||
# We have the previous block
|
||||
last_block = last_blocks[-1]
|
||||
|
||||
# Compress time
|
||||
self.timeOffset = self.nTime - last_block.nTime
|
||||
if self.timeOffset > 32767 or self.timeOffset < -32768:
|
||||
# Time diff overflows, we have to send it as 4 bytes (uncompressed)
|
||||
self.bitfield |= CompressibleBlockHeader.FLAG_TIMESTAMP
|
||||
|
||||
# If nBits doesn't match previous block, we have to send it
|
||||
if self.nBits != last_block.nBits:
|
||||
self.bitfield |= CompressibleBlockHeader.FLAG_NBITS
|
||||
|
||||
def uncompress(self, last_compressed_blocks, last_unique_versions):
|
||||
if not last_compressed_blocks:
|
||||
# First block header is always uncompressed
|
||||
self.__save_version_as_most_recent(last_unique_versions)
|
||||
return
|
||||
|
||||
previous_block = last_compressed_blocks[-1]
|
||||
|
||||
# Uncompress version
|
||||
version_idx = self.bitfield & self.BITMASK_VERSION
|
||||
if version_idx != 0:
|
||||
if version_idx <= len(last_unique_versions):
|
||||
self.nVersion = last_unique_versions[version_idx - 1]
|
||||
self.__mark_version_as_most_recent(last_unique_versions, version_idx - 1)
|
||||
else:
|
||||
self.__save_version_as_most_recent(last_unique_versions)
|
||||
|
||||
# Uncompress prev block hash
|
||||
if not self.bitfield & self.FLAG_PREV_BLOCK_HASH:
|
||||
self.hashPrevBlock = previous_block.hash
|
||||
|
||||
# Uncompress time
|
||||
if not self.bitfield & self.FLAG_TIMESTAMP:
|
||||
self.nTime = previous_block.nTime + self.timeOffset
|
||||
|
||||
# Uncompress time bits
|
||||
if not self.bitfield & self.FLAG_NBITS:
|
||||
self.nBits = previous_block.nBits
|
||||
|
||||
self.rehash()
|
||||
|
||||
|
||||
class PrefilledTransaction:
|
||||
__slots__ = ("index", "tx")
|
||||
|
||||
@ -1492,6 +1664,23 @@ class msg_sendheaders:
|
||||
return "msg_sendheaders()"
|
||||
|
||||
|
||||
class msg_sendheaders2:
|
||||
__slots__ = ()
|
||||
command = b"sendheaders2"
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def deserialize(self, f):
|
||||
pass
|
||||
|
||||
def serialize(self):
|
||||
return b""
|
||||
|
||||
def __repr__(self):
|
||||
return "msg_sendheaders2()"
|
||||
|
||||
|
||||
# getheaders message has
|
||||
# number of entries
|
||||
# vector of hashes
|
||||
@ -1520,6 +1709,31 @@ class msg_getheaders:
|
||||
% (repr(self.locator), self.hashstop)
|
||||
|
||||
|
||||
# same as msg_getheaders, but to request the headers compressed
|
||||
class msg_getheaders2:
|
||||
__slots__ = ("hashstop", "locator",)
|
||||
command = b"getheaders2"
|
||||
|
||||
def __init__(self):
|
||||
self.locator = CBlockLocator()
|
||||
self.hashstop = 0
|
||||
|
||||
def deserialize(self, f):
|
||||
self.locator = CBlockLocator()
|
||||
self.locator.deserialize(f)
|
||||
self.hashstop = deser_uint256(f)
|
||||
|
||||
def serialize(self):
|
||||
r = b""
|
||||
r += self.locator.serialize()
|
||||
r += ser_uint256(self.hashstop)
|
||||
return r
|
||||
|
||||
def __repr__(self):
|
||||
return "msg_getheaders2(locator=%s, stop=%064x)" \
|
||||
% (repr(self.locator), self.hashstop)
|
||||
|
||||
|
||||
# headers message has
|
||||
# <count> <vector of block headers>
|
||||
class msg_headers:
|
||||
@ -1543,6 +1757,31 @@ class msg_headers:
|
||||
return "msg_headers(headers=%s)" % repr(self.headers)
|
||||
|
||||
|
||||
# headers message has
|
||||
# <count> <vector of compressed block headers>
|
||||
class msg_headers2:
|
||||
__slots__ = ("headers",)
|
||||
command = b"headers2"
|
||||
|
||||
def __init__(self, headers=None):
|
||||
self.headers = headers if headers is not None else []
|
||||
|
||||
def deserialize(self, f):
|
||||
self.headers = deser_vector(f, CompressibleBlockHeader)
|
||||
last_unique_versions = []
|
||||
for idx in range(len(self.headers)):
|
||||
self.headers[idx].uncompress(self.headers[:idx], last_unique_versions)
|
||||
|
||||
def serialize(self):
|
||||
last_unique_versions = []
|
||||
for idx in range(len(self.headers)):
|
||||
self.headers[idx].compress(self.headers[:idx], last_unique_versions)
|
||||
return ser_vector(self.headers)
|
||||
|
||||
def __repr__(self):
|
||||
return "msg_headers2(headers=%s)" % repr(self.headers)
|
||||
|
||||
|
||||
class msg_reject:
|
||||
__slots__ = ("code", "data", "message", "reason")
|
||||
command = b"reject"
|
||||
|
||||
@ -25,7 +25,9 @@ import threading
|
||||
|
||||
from test_framework.messages import (
|
||||
CBlockHeader,
|
||||
CompressibleBlockHeader,
|
||||
MIN_VERSION_SUPPORTED,
|
||||
NODE_HEADERS_COMPRESSED,
|
||||
msg_addr,
|
||||
msg_addrv2,
|
||||
msg_block,
|
||||
@ -40,8 +42,10 @@ from test_framework.messages import (
|
||||
msg_getblocktxn,
|
||||
msg_getdata,
|
||||
msg_getheaders,
|
||||
msg_getheaders2,
|
||||
msg_getmnlistd,
|
||||
msg_headers,
|
||||
msg_headers2,
|
||||
msg_inv,
|
||||
msg_islock,
|
||||
msg_isdlock,
|
||||
@ -56,6 +60,7 @@ from test_framework.messages import (
|
||||
msg_sendaddrv2,
|
||||
msg_sendcmpct,
|
||||
msg_sendheaders,
|
||||
msg_sendheaders2,
|
||||
msg_tx,
|
||||
msg_verack,
|
||||
msg_version,
|
||||
@ -85,7 +90,9 @@ MESSAGEMAP = {
|
||||
b"getblocktxn": msg_getblocktxn,
|
||||
b"getdata": msg_getdata,
|
||||
b"getheaders": msg_getheaders,
|
||||
b"getheaders2": msg_getheaders2,
|
||||
b"headers": msg_headers,
|
||||
b"headers2": msg_headers2,
|
||||
b"inv": msg_inv,
|
||||
b"mempool": msg_mempool,
|
||||
b"ping": msg_ping,
|
||||
@ -94,6 +101,7 @@ MESSAGEMAP = {
|
||||
b"sendaddrv2": msg_sendaddrv2,
|
||||
b"sendcmpct": msg_sendcmpct,
|
||||
b"sendheaders": msg_sendheaders,
|
||||
b"sendheaders2": msg_sendheaders2,
|
||||
b"tx": msg_tx,
|
||||
b"verack": msg_verack,
|
||||
b"version": msg_version,
|
||||
@ -330,7 +338,7 @@ class P2PInterface(P2PConnection):
|
||||
|
||||
self.support_addrv2 = support_addrv2
|
||||
|
||||
def peer_connect(self, *args, services=NODE_NETWORK, send_version=True, **kwargs):
|
||||
def peer_connect(self, *args, services=NODE_NETWORK | NODE_HEADERS_COMPRESSED, send_version=True, **kwargs):
|
||||
create_conn = super().peer_connect(*args, **kwargs)
|
||||
|
||||
if send_version:
|
||||
@ -386,7 +394,9 @@ class P2PInterface(P2PConnection):
|
||||
def on_getblocktxn(self, message): pass
|
||||
def on_getdata(self, message): pass
|
||||
def on_getheaders(self, message): pass
|
||||
def on_getheaders2(self, message): pass
|
||||
def on_headers(self, message): pass
|
||||
def on_headers2(self, message): pass
|
||||
def on_mempool(self, message): pass
|
||||
def on_notfound(self, message): pass
|
||||
def on_pong(self, message): pass
|
||||
@ -394,6 +404,7 @@ class P2PInterface(P2PConnection):
|
||||
def on_sendaddrv2(self, message): pass
|
||||
def on_sendcmpct(self, message): pass
|
||||
def on_sendheaders(self, message): pass
|
||||
def on_sendheaders2(self, message): pass
|
||||
def on_tx(self, message): pass
|
||||
|
||||
def on_inv(self, message):
|
||||
@ -485,7 +496,8 @@ class P2PInterface(P2PConnection):
|
||||
|
||||
def test_function():
|
||||
assert self.is_connected
|
||||
return self.last_message.get("getheaders")
|
||||
return self.last_message.get("getheaders2") if self.nServices & NODE_HEADERS_COMPRESSED \
|
||||
else self.last_message.get("getheaders")
|
||||
|
||||
wait_until(test_function, timeout=timeout, lock=mininode_lock)
|
||||
|
||||
@ -582,11 +594,7 @@ class P2PDataStore(P2PInterface):
|
||||
else:
|
||||
logger.debug('getdata message type {} received.'.format(hex(inv.type)))
|
||||
|
||||
def on_getheaders(self, message):
|
||||
"""Search back through our block store for the locator, and reply with a headers message if found."""
|
||||
|
||||
locator, hash_stop = message.locator, message.hashstop
|
||||
|
||||
def _compute_requested_block_headers(self, locator, hash_stop):
|
||||
# Assume that the most recent block added is the tip
|
||||
if not self.block_store:
|
||||
return
|
||||
@ -609,6 +617,23 @@ class P2PDataStore(P2PInterface):
|
||||
|
||||
# Truncate the list if there are too many headers
|
||||
headers_list = headers_list[:-maxheaders - 1:-1]
|
||||
|
||||
return headers_list
|
||||
|
||||
def on_getheaders2(self, message):
|
||||
"""Search back through our block store for the locator, and reply with a compressed headers message if found."""
|
||||
|
||||
headers_list = self._compute_requested_block_headers(message.locator, message.hashstop)
|
||||
compressible_headers_list = [CompressibleBlockHeader(h) for h in headers_list] if headers_list else None
|
||||
response = msg_headers2(compressible_headers_list)
|
||||
|
||||
if response is not None:
|
||||
self.send_message(response)
|
||||
|
||||
def on_getheaders(self, message):
|
||||
"""Search back through our block store for the locator, and reply with a headers message if found."""
|
||||
|
||||
headers_list = self._compute_requested_block_headers(message.locator, message.hashstop)
|
||||
response = msg_headers(headers_list)
|
||||
|
||||
if response is not None:
|
||||
|
||||
@ -109,6 +109,7 @@ BASE_SCRIPTS = [
|
||||
'feature_dip4_coinbasemerkleroots.py', # NOTE: needs dash_hash to pass
|
||||
# vv Tests less than 60s vv
|
||||
'p2p_sendheaders.py', # NOTE: needs dash_hash to pass
|
||||
'p2p_sendheaders_compressed.py', # NOTE: needs dash_hash to pass
|
||||
'wallet_zapwallettxes.py',
|
||||
'wallet_importmulti.py',
|
||||
'mempool_limit.py',
|
||||
|
||||
Loading…
Reference in New Issue
Block a user