diff --git a/qa/pull-tester/run-bitcoind-for-test.sh.in b/qa/pull-tester/run-bitcoind-for-test.sh.in index 210fc3c42f..15363d09a6 100755 --- a/qa/pull-tester/run-bitcoind-for-test.sh.in +++ b/qa/pull-tester/run-bitcoind-for-test.sh.in @@ -10,7 +10,7 @@ touch "$DATADIR/regtest/debug.log" tail -q -n 1 -F "$DATADIR/regtest/debug.log" | grep -m 1 -q "Done loading" & WAITER=$! PORT=`expr 10000 + $$ % 55536` -"@abs_top_builddir@/src/bitcoind@EXEEXT@" -connect=0.0.0.0 -datadir="$DATADIR" -rpcuser=user -rpcpassword=pass -listen -keypool=3 -debug -debug=net -logtimestamps -port=$PORT -whitelist=127.0.0.1 -regtest -rpcport=`expr $PORT + 1` & +"@abs_top_builddir@/src/bitcoind@EXEEXT@" -connect=0.0.0.0 -datadir="$DATADIR" -rpcuser=user -rpcpassword=pass -listen -keypool=3 -debug -debug=net -logtimestamps -checkmempool=0 -port=$PORT -whitelist=127.0.0.1 -regtest -rpcport=`expr $PORT + 1` & BITCOIND=$! #Install a watchdog. diff --git a/src/chain.h b/src/chain.h index 0aafb40b98..4e6a466c6a 100644 --- a/src/chain.h +++ b/src/chain.h @@ -49,12 +49,29 @@ struct CDiskBlockPos }; enum BlockStatus { + // Unused. BLOCK_VALID_UNKNOWN = 0, - BLOCK_VALID_HEADER = 1, // parsed, version ok, hash satisfies claimed PoW, 1 <= vtx count <= max, timestamp not in future - BLOCK_VALID_TREE = 2, // parent found, difficulty matches, timestamp >= median previous, checkpoint - BLOCK_VALID_TRANSACTIONS = 3, // only first tx is coinbase, 2 <= coinbase input script length <= 100, transactions valid, no duplicate txids, sigops, size, merkle root - BLOCK_VALID_CHAIN = 4, // outputs do not overspend inputs, no double spends, coinbase output ok, immature coinbase spends, BIP30 - BLOCK_VALID_SCRIPTS = 5, // scripts/signatures ok + + // Parsed, version ok, hash satisfies claimed PoW, 1 <= vtx count <= max, timestamp not in future + BLOCK_VALID_HEADER = 1, + + // All parent headers found, difficulty matches, timestamp >= median previous, checkpoint. Implies all parents + // are also at least TREE. + BLOCK_VALID_TREE = 2, + + // Only first tx is coinbase, 2 <= coinbase input script length <= 100, transactions valid, no duplicate txids, + // sigops, size, merkle root. Implies all parents are at least TREE but not necessarily TRANSACTIONS. When all + // parent blocks also have TRANSACTIONS, CBlockIndex::nChainTx will be set. + BLOCK_VALID_TRANSACTIONS = 3, + + // Outputs do not overspend inputs, no double spends, coinbase output ok, immature coinbase spends, BIP30. + // Implies all parents are also at least CHAIN. + BLOCK_VALID_CHAIN = 4, + + // Scripts & signatures ok. Implies all parents are also at least SCRIPTS. + BLOCK_VALID_SCRIPTS = 5, + + // All validity bits. BLOCK_VALID_MASK = BLOCK_VALID_HEADER | BLOCK_VALID_TREE | BLOCK_VALID_TRANSACTIONS | BLOCK_VALID_CHAIN | BLOCK_VALID_SCRIPTS, @@ -103,7 +120,8 @@ public: // Note: in a potential headers-first mode, this number cannot be relied upon unsigned int nTx; - // (memory only) Number of transactions in the chain up to and including this block + // (memory only) Number of transactions in the chain up to and including this block. + // This value will be non-zero only if and only if transactions for this block and all its parents are available. unsigned int nChainTx; // change to 64-bit type when necessary; won't happen before 2030 // Verification status of this block. See enum BlockStatus @@ -146,7 +164,7 @@ public: SetNull(); } - CBlockIndex(CBlockHeader& block) + CBlockIndex(const CBlockHeader& block) { SetNull(); diff --git a/src/init.cpp b/src/init.cpp index 8772be5539..55991ac9b3 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -974,7 +974,7 @@ bool AppInit2(boost::thread_group& threadGroup) // If the loaded chain has a wrong genesis, bail out immediately // (we're likely using a testnet datadir, or the other way around). - if (!mapBlockIndex.empty() && chainActive.Genesis() == NULL) + if (!mapBlockIndex.empty() && mapBlockIndex.count(Params().HashGenesisBlock()) == 0) return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?")); // Initialize the block index (no-op if non-empty database was already loaded) diff --git a/src/main.cpp b/src/main.cpp index fc8167e40e..f5fd7561c6 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -41,6 +41,7 @@ CCriticalSection cs_main; BlockMap mapBlockIndex; CChain chainActive; +CBlockIndex *pindexBestHeader = NULL; int64_t nTimeBestReceived = 0; CWaitableCriticalSection csBestBlock; CConditionVariable cvBlockChange; @@ -51,19 +52,12 @@ bool fTxIndex = false; bool fIsBareMultisigStd = true; unsigned int nCoinCacheSize = 5000; + /** Fees smaller than this (in satoshi) are considered zero fee (for relaying and mining) */ CFeeRate minRelayTxFee = CFeeRate(1000); CTxMemPool mempool(::minRelayTxFee); -struct COrphanBlock { - uint256 hashBlock; - uint256 hashPrev; - vector vchBlock; -}; -map mapOrphanBlocks; -multimap mapOrphanBlocksByPrev; - struct COrphanTx { CTransaction tx; NodeId fromPeer; @@ -105,7 +99,11 @@ namespace { // The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS or better that are at least // as good as our current tip. Entries may be failed, though. - set setBlockIndexValid; + set setBlockIndexCandidates; + // Number of nodes with fSyncStarted. + int nSyncStarted = 0; + // All pairs A->B, where A (or one if its ancestors) misses transactions, but B has transactions. + multimap mapBlocksUnlinked; CCriticalSection cs_LastBlockFile; CBlockFileInfo infoLastBlockFile; @@ -125,11 +123,10 @@ namespace { // Protected by cs_main. struct QueuedBlock { uint256 hash; + CBlockIndex *pindex; // Optional. int64_t nTime; // Time of "getdata" request in microseconds. - int nQueuedBefore; // Number of blocks in flight at the time of request. }; map::iterator> > mapBlocksInFlight; - map::iterator> > mapBlocksToDownload; } // anon namespace @@ -220,22 +217,24 @@ struct CNodeState { CBlockIndex *pindexBestKnownBlock; // The hash of the last unknown block this peer has announced. uint256 hashLastUnknownBlock; + // The last full block we both have. + CBlockIndex *pindexLastCommonBlock; + // Whether we've started headers synchronization with this peer. + bool fSyncStarted; + // Since when we're stalling block download progress (in microseconds), or 0. + int64_t nStallingSince; list vBlocksInFlight; int nBlocksInFlight; - list vBlocksToDownload; - int nBlocksToDownload; - int64_t nLastBlockReceive; - int64_t nLastBlockProcess; CNodeState() { nMisbehavior = 0; fShouldBan = false; pindexBestKnownBlock = NULL; hashLastUnknownBlock = uint256(0); - nBlocksToDownload = 0; + pindexLastCommonBlock = NULL; + fSyncStarted = false; + nStallingSince = 0; nBlocksInFlight = 0; - nLastBlockReceive = 0; - nLastBlockProcess = 0; } }; @@ -266,64 +265,37 @@ void FinalizeNode(NodeId nodeid) { LOCK(cs_main); CNodeState *state = State(nodeid); + if (state->fSyncStarted) + nSyncStarted--; + BOOST_FOREACH(const QueuedBlock& entry, state->vBlocksInFlight) mapBlocksInFlight.erase(entry.hash); - BOOST_FOREACH(const uint256& hash, state->vBlocksToDownload) - mapBlocksToDownload.erase(hash); EraseOrphansFor(nodeid); mapNodeState.erase(nodeid); } // Requires cs_main. -void MarkBlockAsReceived(const uint256 &hash, NodeId nodeFrom = -1) { - map::iterator> >::iterator itToDownload = mapBlocksToDownload.find(hash); - if (itToDownload != mapBlocksToDownload.end()) { - CNodeState *state = State(itToDownload->second.first); - state->vBlocksToDownload.erase(itToDownload->second.second); - state->nBlocksToDownload--; - mapBlocksToDownload.erase(itToDownload); - } - +void MarkBlockAsReceived(const uint256& hash) { map::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); if (itInFlight != mapBlocksInFlight.end()) { CNodeState *state = State(itInFlight->second.first); state->vBlocksInFlight.erase(itInFlight->second.second); state->nBlocksInFlight--; - if (itInFlight->second.first == nodeFrom) - state->nLastBlockReceive = GetTimeMicros(); + state->nStallingSince = 0; mapBlocksInFlight.erase(itInFlight); } } // Requires cs_main. -bool AddBlockToQueue(NodeId nodeid, const uint256 &hash) { - if (mapBlocksToDownload.count(hash) || mapBlocksInFlight.count(hash)) - return false; - - CNodeState *state = State(nodeid); - if (state == NULL) - return false; - - list::iterator it = state->vBlocksToDownload.insert(state->vBlocksToDownload.end(), hash); - state->nBlocksToDownload++; - if (state->nBlocksToDownload > 5000) - Misbehaving(nodeid, 10); - mapBlocksToDownload[hash] = std::make_pair(nodeid, it); - return true; -} - -// Requires cs_main. -void MarkBlockAsInFlight(NodeId nodeid, const uint256 &hash) { +void MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, CBlockIndex *pindex = NULL) { CNodeState *state = State(nodeid); assert(state != NULL); // Make sure it's not listed somewhere already. MarkBlockAsReceived(hash); - QueuedBlock newentry = {hash, GetTimeMicros(), state->nBlocksInFlight}; - if (state->nBlocksInFlight == 0) - state->nLastBlockReceive = newentry.nTime; // Reset when a first request is sent. + QueuedBlock newentry = {hash, pindex, GetTimeMicros()}; list::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), newentry); state->nBlocksInFlight++; mapBlocksInFlight[hash] = std::make_pair(nodeid, it); @@ -362,6 +334,104 @@ void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) { } } +/** Find the last common ancestor two blocks have. + * Both pa and pb must be non-NULL. */ +CBlockIndex* LastCommonAncestor(CBlockIndex* pa, CBlockIndex* pb) { + if (pa->nHeight > pb->nHeight) { + pa = pa->GetAncestor(pb->nHeight); + } else if (pb->nHeight > pa->nHeight) { + pb = pb->GetAncestor(pa->nHeight); + } + + while (pa != pb && pa && pb) { + pa = pa->pprev; + pb = pb->pprev; + } + + // Eventually all chain branches meet at the genesis block. + assert(pa == pb); + return pa; +} + +/** Update pindexLastCommonBlock and add not-in-flight missing successors to vBlocks, until it has + * at most count entries. */ +void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector& vBlocks, NodeId& nodeStaller) { + if (count == 0) + return; + + vBlocks.reserve(vBlocks.size() + count); + CNodeState *state = State(nodeid); + assert(state != NULL); + + // Make sure pindexBestKnownBlock is up to date, we'll need it. + ProcessBlockAvailability(nodeid); + + if (state->pindexBestKnownBlock == NULL || state->pindexBestKnownBlock->nChainWork < chainActive.Tip()->nChainWork) { + // This peer has nothing interesting. + return; + } + + if (state->pindexLastCommonBlock == NULL) { + // Bootstrap quickly by guessing a parent of our best tip is the forking point. + // Guessing wrong in either direction is not a problem. + state->pindexLastCommonBlock = chainActive[std::min(state->pindexBestKnownBlock->nHeight, chainActive.Height())]; + } + + // If the peer reorganized, our previous pindexLastCommonBlock may not be an ancestor + // of their current tip anymore. Go back enough to fix that. + state->pindexLastCommonBlock = LastCommonAncestor(state->pindexLastCommonBlock, state->pindexBestKnownBlock); + if (state->pindexLastCommonBlock == state->pindexBestKnownBlock) + return; + + std::vector vToFetch; + CBlockIndex *pindexWalk = state->pindexLastCommonBlock; + // Never fetch further than the best block we know the peer has, or more than BLOCK_DOWNLOAD_WINDOW + 1 beyond the last + // linked block we have in common with this peer. The +1 is so we can detect stalling, namely if we would be able to + // download that next block if the window were 1 larger. + int nWindowEnd = state->pindexLastCommonBlock->nHeight + BLOCK_DOWNLOAD_WINDOW; + int nMaxHeight = std::min(state->pindexBestKnownBlock->nHeight, nWindowEnd + 1); + NodeId waitingfor = -1; + while (pindexWalk->nHeight < nMaxHeight) { + // Read up to 128 (or more, if more blocks than that are needed) successors of pindexWalk (towards + // pindexBestKnownBlock) into vToFetch. We fetch 128, because CBlockIndex::GetAncestor may be as expensive + // as iterating over ~100 CBlockIndex* entries anyway. + int nToFetch = std::min(nMaxHeight - pindexWalk->nHeight, std::max(count - vBlocks.size(), 128)); + vToFetch.resize(nToFetch); + pindexWalk = state->pindexBestKnownBlock->GetAncestor(pindexWalk->nHeight + nToFetch); + vToFetch[nToFetch - 1] = pindexWalk; + for (unsigned int i = nToFetch - 1; i > 0; i--) { + vToFetch[i - 1] = vToFetch[i]->pprev; + } + + // Iterate over those blocks in vToFetch (in forward direction), adding the ones that + // are not yet downloaded and not in flight to vBlocks. In the mean time, update + // pindexLastCommonBlock as long as all ancestors are already downloaded. + BOOST_FOREACH(CBlockIndex* pindex, vToFetch) { + if (pindex->nStatus & BLOCK_HAVE_DATA) { + if (pindex->nChainTx) + state->pindexLastCommonBlock = pindex; + } else if (mapBlocksInFlight.count(pindex->GetBlockHash()) == 0) { + // The block is not already downloaded, and not yet in flight. + if (pindex->nHeight > nWindowEnd) { + // We reached the end of the window. + if (vBlocks.size() == 0 && waitingfor != nodeid) { + // We aren't able to fetch anything, but we would be if the download window was one larger. + nodeStaller = waitingfor; + } + return; + } + vBlocks.push_back(pindex); + if (vBlocks.size() == count) { + return; + } + } else if (waitingfor == -1) { + // This is the first already-in-flight block. + waitingfor = mapBlocksInFlight[pindex->GetBlockHash()].first; + } + } + } +} + } // anon namespace bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { @@ -371,6 +441,11 @@ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { return false; stats.nMisbehavior = state->nMisbehavior; stats.nSyncHeight = state->pindexBestKnownBlock ? state->pindexBestKnownBlock->nHeight : -1; + stats.nCommonHeight = state->pindexLastCommonBlock ? state->pindexLastCommonBlock->nHeight : -1; + BOOST_FOREACH(const QueuedBlock& queue, state->vBlocksInFlight) { + if (queue.pindex) + stats.vHeightInFlight.push_back(queue.pindex->nHeight); + } return true; } @@ -1086,46 +1161,6 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex) return true; } -uint256 static GetOrphanRoot(const uint256& hash) -{ - map::iterator it = mapOrphanBlocks.find(hash); - if (it == mapOrphanBlocks.end()) - return hash; - - // Work back to the first block in the orphan chain - do { - map::iterator it2 = mapOrphanBlocks.find(it->second->hashPrev); - if (it2 == mapOrphanBlocks.end()) - return it->first; - it = it2; - } while(true); -} - -// Remove a random orphan block (which does not have any dependent orphans). -void static PruneOrphanBlocks() -{ - if (mapOrphanBlocksByPrev.size() <= (size_t)std::max((int64_t)0, GetArg("-maxorphanblocks", DEFAULT_MAX_ORPHAN_BLOCKS))) - return; - - // Pick a random orphan block. - int pos = insecure_rand() % mapOrphanBlocksByPrev.size(); - std::multimap::iterator it = mapOrphanBlocksByPrev.begin(); - while (pos--) it++; - - // As long as this block has other orphans depending on it, move to one of those successors. - do { - std::multimap::iterator it2 = mapOrphanBlocksByPrev.find(it->second->hashBlock); - if (it2 == mapOrphanBlocksByPrev.end()) - break; - it = it2; - } while(1); - - uint256 hash = it->second->hashBlock; - delete it->second; - mapOrphanBlocksByPrev.erase(it); - mapOrphanBlocks.erase(hash); -} - CAmount GetBlockValue(int nHeight, const CAmount& nFees) { int64_t nSubsidy = 50 * COIN; @@ -1284,7 +1319,7 @@ void static InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state if (!state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; pblocktree->WriteBlockIndex(CDiskBlockIndex(pindex)); - setBlockIndexValid.erase(pindex); + setBlockIndexCandidates.erase(pindex); InvalidChainFound(pindex); } } @@ -1664,11 +1699,6 @@ bool ConnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, C if (fJustCheck) return true; - // Correct transaction counts. - pindex->nTx = block.vtx.size(); - if (pindex->pprev) - pindex->nChainTx = pindex->pprev->nChainTx + block.vtx.size(); - // Write undo information to disk if (pindex->GetUndoPos().IsNull() || !pindex->IsValid(BLOCK_VALID_SCRIPTS)) { @@ -1889,8 +1919,8 @@ static CBlockIndex* FindMostWorkChain() { // Find the best candidate header. { - std::set::reverse_iterator it = setBlockIndexValid.rbegin(); - if (it == setBlockIndexValid.rend()) + std::set::reverse_iterator it = setBlockIndexCandidates.rbegin(); + if (it == setBlockIndexCandidates.rend()) return NULL; pindexNew = *it; } @@ -1900,6 +1930,8 @@ static CBlockIndex* FindMostWorkChain() { CBlockIndex *pindexTest = pindexNew; bool fInvalidAncestor = false; while (pindexTest && !chainActive.Contains(pindexTest)) { + assert(pindexTest->nStatus & BLOCK_HAVE_DATA); + assert(pindexTest->nChainTx || pindexTest->nHeight == 0); if (pindexTest->nStatus & BLOCK_FAILED_MASK) { // Candidate has an invalid ancestor, remove entire chain from the set. if (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork) @@ -1907,10 +1939,10 @@ static CBlockIndex* FindMostWorkChain() { CBlockIndex *pindexFailed = pindexNew; while (pindexTest != pindexFailed) { pindexFailed->nStatus |= BLOCK_FAILED_CHILD; - setBlockIndexValid.erase(pindexFailed); + setBlockIndexCandidates.erase(pindexFailed); pindexFailed = pindexFailed->pprev; } - setBlockIndexValid.erase(pindexTest); + setBlockIndexCandidates.erase(pindexTest); fInvalidAncestor = true; break; } @@ -1937,12 +1969,20 @@ static bool ActivateBestChainStep(CValidationState &state, CBlockIndex *pindexMo // Build list of new blocks to connect. std::vector vpindexToConnect; - vpindexToConnect.reserve(pindexMostWork->nHeight - (pindexFork ? pindexFork->nHeight : -1)); - CBlockIndex *pindexIter = pindexMostWork; - while (pindexIter && pindexIter != pindexFork) { + bool fContinue = true; + int nHeight = pindexFork ? pindexFork->nHeight : -1; + while (fContinue && nHeight != pindexMostWork->nHeight) { + // Don't iterate the entire list of potential improvements toward the best tip, as we likely only need + // a few blocks along the way. + int nTargetHeight = std::min(nHeight + 32, pindexMostWork->nHeight); + vpindexToConnect.clear(); + vpindexToConnect.reserve(nTargetHeight - nHeight); + CBlockIndex *pindexIter = pindexMostWork->GetAncestor(nTargetHeight); + while (pindexIter && pindexIter->nHeight != nHeight) { vpindexToConnect.push_back(pindexIter); pindexIter = pindexIter->pprev; } + nHeight = nTargetHeight; // Connect new blocks. BOOST_REVERSE_FOREACH(CBlockIndex *pindexConnect, vpindexToConnect) { @@ -1953,27 +1993,30 @@ static bool ActivateBestChainStep(CValidationState &state, CBlockIndex *pindexMo InvalidChainFound(vpindexToConnect.back()); state = CValidationState(); fInvalidFound = true; + fContinue = false; break; } else { // A system error occurred (disk space, database error, ...). return false; } } else { - // Delete all entries in setBlockIndexValid that are worse than our new current block. + // Delete all entries in setBlockIndexCandidates that are worse than our new current block. // Note that we can't delete the current block itself, as we may need to return to it later in case a // reorganization to a better block fails. - std::set::iterator it = setBlockIndexValid.begin(); - while (setBlockIndexValid.value_comp()(*it, chainActive.Tip())) { - setBlockIndexValid.erase(it++); + std::set::iterator it = setBlockIndexCandidates.begin(); + while (setBlockIndexCandidates.value_comp()(*it, chainActive.Tip())) { + setBlockIndexCandidates.erase(it++); } - // Either the current tip or a successor of it we're working towards is left in setBlockIndexValid. - assert(!setBlockIndexValid.empty()); + // Either the current tip or a successor of it we're working towards is left in setBlockIndexCandidates. + assert(!setBlockIndexCandidates.empty()); if (!pindexOldTip || chainActive.Tip()->nChainWork > pindexOldTip->nChainWork) { // We're in a better position than we were. Return temporarily to release the lock. + fContinue = false; break; } } } + } // Callbacks/notifications for a new best chain. if (fInvalidFound) @@ -2032,7 +2075,7 @@ bool ActivateBestChain(CValidationState &state, CBlock *pblock) { return true; } -CBlockIndex* AddToBlockIndex(CBlockHeader& block) +CBlockIndex* AddToBlockIndex(const CBlockHeader& block) { // Check for duplicate uint256 hash = block.GetHash(); @@ -2043,10 +2086,10 @@ CBlockIndex* AddToBlockIndex(CBlockHeader& block) // Construct new block index object CBlockIndex* pindexNew = new CBlockIndex(block); assert(pindexNew); - { - LOCK(cs_nBlockSequenceId); - pindexNew->nSequenceId = nBlockSequenceId++; - } + // We assign the sequence id to blocks only when the full data is available, + // to avoid miners withholding blocks but broadcasting headers, to get a + // competitive advantage. + pindexNew->nSequenceId = 0; BlockMap::iterator mi = mapBlockIndex.insert(make_pair(hash, pindexNew)).first; pindexNew->phashBlock = &((*mi).first); BlockMap::iterator miPrev = mapBlockIndex.find(block.hashPrevBlock); @@ -2058,6 +2101,11 @@ CBlockIndex* AddToBlockIndex(CBlockHeader& block) } pindexNew->nChainWork = (pindexNew->pprev ? pindexNew->pprev->nChainWork : 0) + pindexNew->GetBlockWork(); pindexNew->RaiseValidity(BLOCK_VALID_TREE); + if (pindexBestHeader == NULL || pindexBestHeader->nChainWork < pindexNew->nChainWork) + pindexBestHeader = pindexNew; + + // Ok if it fails, we'll download the header again next time. + pblocktree->WriteBlockIndex(CDiskBlockIndex(pindexNew)); return pindexNew; } @@ -2066,30 +2114,45 @@ CBlockIndex* AddToBlockIndex(CBlockHeader& block) bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos) { pindexNew->nTx = block.vtx.size(); - if (pindexNew->pprev) { - // Not the genesis block. - if (pindexNew->pprev->nChainTx) { - // This parent's block's total number transactions is known, so compute outs. - pindexNew->nChainTx = pindexNew->pprev->nChainTx + pindexNew->nTx; - } else { - // The total number of transactions isn't known yet. - // We will compute it when the block is connected. - pindexNew->nChainTx = 0; - } - } else { - // Genesis block. - pindexNew->nChainTx = pindexNew->nTx; - } + pindexNew->nChainTx = 0; pindexNew->nFile = pos.nFile; pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; + pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); + { + LOCK(cs_nBlockSequenceId); + pindexNew->nSequenceId = nBlockSequenceId++; + } - if (pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS)) - setBlockIndexValid.insert(pindexNew); + if (pindexNew->pprev == NULL || pindexNew->pprev->nChainTx) { + // If pindexNew is the genesis block or all parents are BLOCK_VALID_TRANSACTIONS. + deque queue; + queue.push_back(pindexNew); - if (!pblocktree->WriteBlockIndex(CDiskBlockIndex(pindexNew))) - return state.Abort("Failed to write block index"); + // Recursively process any descendant blocks that now may be eligible to be connected. + while (!queue.empty()) { + CBlockIndex *pindex = queue.front(); + queue.pop_front(); + pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; + setBlockIndexCandidates.insert(pindex); + std::pair::iterator, std::multimap::iterator> range = mapBlocksUnlinked.equal_range(pindex); + while (range.first != range.second) { + std::multimap::iterator it = range.first; + queue.push_back(it->second); + range.first++; + mapBlocksUnlinked.erase(it); + } + if (!pblocktree->WriteBlockIndex(CDiskBlockIndex(pindex))) + return state.Abort("Failed to write block index"); + } + } else { + if (pindexNew->pprev && pindexNew->pprev->IsValid(BLOCK_VALID_TREE)) { + mapBlocksUnlinked.insert(std::make_pair(pindexNew->pprev, pindexNew)); + } + if (!pblocktree->WriteBlockIndex(CDiskBlockIndex(pindexNew))) + return state.Abort("Failed to write block index"); + } return true; } @@ -2205,12 +2268,31 @@ bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, bool f bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bool fCheckMerkleRoot) { - // These are checks that are independent of context - // that can be verified before saving an orphan block. + // These are checks that are independent of context. if (!CheckBlockHeader(block, state, fCheckPOW)) return false; + // Check the merkle root. + if (fCheckMerkleRoot) { + bool mutated; + uint256 hashMerkleRoot2 = block.BuildMerkleTree(&mutated); + if (block.hashMerkleRoot != hashMerkleRoot2) + return state.DoS(100, error("CheckBlock() : hashMerkleRoot mismatch"), + REJECT_INVALID, "bad-txnmrklroot", true); + + // Check for merkle tree malleability (CVE-2012-2459): repeating sequences + // of transactions in a block without affecting the merkle root of a block, + // while still invalidating it. + if (mutated) + return state.DoS(100, error("CheckBlock() : duplicate transaction"), + REJECT_INVALID, "bad-txns-duplicate", true); + } + + // All potential-corruption validation must be done before we do any + // transaction validation, as otherwise we may mark the header as invalid + // because we receive the wrong transactions for it. + // Size limits if (block.vtx.empty() || block.vtx.size() > MAX_BLOCK_SIZE || ::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION) > MAX_BLOCK_SIZE) return state.DoS(100, error("CheckBlock() : size limits failed"), @@ -2230,15 +2312,6 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo if (!CheckTransaction(tx, state)) return error("CheckBlock() : CheckTransaction failed"); - // Check for merkle tree malleability (CVE-2012-2459): repeating sequences - // of transactions in a block without affecting the merkle root of a block, - // while still invalidating it. - bool mutated; - uint256 hashMerkleRoot2 = block.BuildMerkleTree(&mutated); - if (mutated) - return state.DoS(100, error("CheckBlock() : duplicate transaction"), - REJECT_INVALID, "bad-txns-duplicate", true); - unsigned int nSigOps = 0; BOOST_FOREACH(const CTransaction& tx, block.vtx) { @@ -2248,15 +2321,10 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo return state.DoS(100, error("CheckBlock() : out-of-bounds SigOpCount"), REJECT_INVALID, "bad-blk-sigops", true); - // Check merkle root - if (fCheckMerkleRoot && block.hashMerkleRoot != hashMerkleRoot2) - return state.DoS(100, error("CheckBlock() : hashMerkleRoot mismatch"), - REJECT_INVALID, "bad-txnmrklroot", true); - return true; } -bool AcceptBlockHeader(CBlockHeader& block, CValidationState& state, CBlockIndex** ppindex) +bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex** ppindex) { AssertLockHeld(cs_main); // Check for duplicate @@ -2264,26 +2332,13 @@ bool AcceptBlockHeader(CBlockHeader& block, CValidationState& state, CBlockIndex BlockMap::iterator miSelf = mapBlockIndex.find(hash); CBlockIndex *pindex = NULL; if (miSelf != mapBlockIndex.end()) { + // Block header is already known. pindex = miSelf->second; + if (ppindex) + *ppindex = pindex; if (pindex->nStatus & BLOCK_FAILED_MASK) return state.Invalid(error("%s : block is marked invalid", __func__), 0, "duplicate"); - } - - CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(); - if (pcheckpoint && block.hashPrevBlock != (chainActive.Tip() ? chainActive.Tip()->GetBlockHash() : uint256(0))) - { - // Extra checks to prevent "fill up memory by spamming with bogus blocks" - int64_t deltaTime = block.GetBlockTime() - pcheckpoint->GetBlockTime(); - if (deltaTime < 0) - { - return state.DoS(100, error("%s : block with timestamp before last checkpoint", __func__), - REJECT_CHECKPOINT, "time-too-old"); - } - if (!CheckMinWork(block.nBits, pcheckpoint->nBits, deltaTime)) - { - return state.DoS(100, error("%s : block with too little proof-of-work", __func__), - REJECT_INVALID, "bad-diffbits"); - } + return true; } // Get prev block index @@ -2344,6 +2399,12 @@ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex, if (!AcceptBlockHeader(block, state, &pindex)) return false; + if (pindex->nStatus & BLOCK_HAVE_DATA) { + // TODO: deal better with duplicate blocks. + // return state.DoS(20, error("AcceptBlock() : already have block %d %s", pindex->nHeight, pindex->GetBlockHash().ToString()), REJECT_DUPLICATE, "duplicate"); + return true; + } + if (!CheckBlock(block, state)) { if (state.IsInvalid() && !state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; @@ -2456,93 +2517,26 @@ void CBlockIndex::BuildSkip() pskip = pprev->GetAncestor(GetSkipHeight(nHeight)); } -void PushGetBlocks(CNode* pnode, CBlockIndex* pindexBegin, uint256 hashEnd) -{ - AssertLockHeld(cs_main); - // Filter out duplicate requests - if (pindexBegin == pnode->pindexLastGetBlocksBegin && hashEnd == pnode->hashLastGetBlocksEnd) - return; - pnode->pindexLastGetBlocksBegin = pindexBegin; - pnode->hashLastGetBlocksEnd = hashEnd; - - pnode->PushMessage("getblocks", chainActive.GetLocator(pindexBegin), hashEnd); -} - bool ProcessBlock(CValidationState &state, CNode* pfrom, CBlock* pblock, CDiskBlockPos *dbp) { - // Check for duplicate - uint256 hash = pblock->GetHash(); - - { - LOCK(cs_main); - if (mapBlockIndex.count(hash)) - return state.Invalid(error("ProcessBlock() : already have block %d %s", mapBlockIndex[hash]->nHeight, hash.ToString()), 0, "duplicate"); - if (mapOrphanBlocks.count(hash)) - return state.Invalid(error("ProcessBlock() : already have block (orphan) %s", hash.ToString()), 0, "duplicate"); - // Preliminary checks - if (!CheckBlock(*pblock, state)) - return error("ProcessBlock() : CheckBlock FAILED"); + bool checked = CheckBlock(*pblock, state); - // If we don't already have its previous block (with full data), shunt it off to holding area until we get it - BlockMap::iterator it = mapBlockIndex.find(pblock->hashPrevBlock); - if (pblock->hashPrevBlock != 0 && (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA))) { - LogPrintf("ProcessBlock: ORPHAN BLOCK %lu, prev=%s\n", (unsigned long)mapOrphanBlocks.size(), pblock->hashPrevBlock.ToString()); - - // Accept orphans as long as there is a node to request its parents from - if (pfrom) { - PruneOrphanBlocks(); - COrphanBlock* pblock2 = new COrphanBlock(); - { - CDataStream ss(SER_DISK, CLIENT_VERSION); - ss << *pblock; - pblock2->vchBlock = std::vector(ss.begin(), ss.end()); - } - pblock2->hashBlock = hash; - pblock2->hashPrev = pblock->hashPrevBlock; - mapOrphanBlocks.insert(make_pair(hash, pblock2)); - mapOrphanBlocksByPrev.insert(make_pair(pblock2->hashPrev, pblock2)); - - // Ask this guy to fill in what we're missing - PushGetBlocks(pfrom, chainActive.Tip(), GetOrphanRoot(hash)); + LOCK(cs_main); + MarkBlockAsReceived(pblock->GetHash()); + if (!checked) { + return error("ProcessBlock() : CheckBlock FAILED"); } - return true; - } - // Store to disk - CBlockIndex *pindex = NULL; - bool ret = AcceptBlock(*pblock, state, &pindex, dbp); - if (!ret) - return error("ProcessBlock() : AcceptBlock FAILED"); - - // Recursively process any orphan blocks that depended on this one - vector vWorkQueue; - vWorkQueue.push_back(hash); - for (unsigned int i = 0; i < vWorkQueue.size(); i++) - { - uint256 hashPrev = vWorkQueue[i]; - for (multimap::iterator mi = mapOrphanBlocksByPrev.lower_bound(hashPrev); - mi != mapOrphanBlocksByPrev.upper_bound(hashPrev); - ++mi) - { - CBlock block; - { - CDataStream ss(mi->second->vchBlock, SER_DISK, CLIENT_VERSION); - ss >> block; - } - block.BuildMerkleTree(); - // Use a dummy CValidationState so someone can't setup nodes to counter-DoS based on orphan resolution (that is, feeding people an invalid block based on LegitBlockX in order to get anyone relaying LegitBlockX banned) - CValidationState stateDummy; - CBlockIndex *pindexChild = NULL; - if (AcceptBlock(block, stateDummy, &pindexChild)) - vWorkQueue.push_back(mi->second->hashBlock); - mapOrphanBlocks.erase(mi->second->hashBlock); - delete mi->second; + // Store to disk + CBlockIndex *pindex = NULL; + bool ret = AcceptBlock(*pblock, state, &pindex, dbp); + if (pindex && pfrom) { + mapBlockSource[pindex->GetBlockHash()] = pfrom->GetId(); } - mapOrphanBlocksByPrev.erase(hashPrev); - } - + if (!ret) + return error("ProcessBlock() : AcceptBlock FAILED"); } if (!ActivateBestChain(state, pblock)) @@ -2808,13 +2802,26 @@ bool static LoadBlockIndexDB() { CBlockIndex* pindex = item.second; pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + pindex->GetBlockWork(); - pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; - if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS)) - setBlockIndexValid.insert(pindex); + if (pindex->nStatus & BLOCK_HAVE_DATA) { + if (pindex->pprev) { + if (pindex->pprev->nChainTx) { + pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; + } else { + pindex->nChainTx = 0; + mapBlocksUnlinked.insert(std::make_pair(pindex->pprev, pindex)); + } + } else { + pindex->nChainTx = pindex->nTx; + } + } + if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->nChainTx || pindex->pprev == NULL)) + setBlockIndexCandidates.insert(pindex); if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nChainWork > pindexBestInvalid->nChainWork)) pindexBestInvalid = pindex; if (pindex->pprev) pindex->BuildSkip(); + if (pindex->IsValid(BLOCK_VALID_TREE) && (pindexBestHeader == NULL || CBlockIndexWorkComparator()(pindexBestHeader, pindex))) + pindexBestHeader = pindex; } // Load block file info @@ -2952,7 +2959,7 @@ bool CVerifyDB::VerifyDB(CCoinsView *coinsview, int nCheckLevel, int nCheckDepth void UnloadBlockIndex() { mapBlockIndex.clear(); - setBlockIndexValid.clear(); + setBlockIndexCandidates.clear(); chainActive.SetTip(NULL); pindexBestInvalid = NULL; } @@ -3075,21 +3082,14 @@ void PrintBlockTree() bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp) { + // Map of disk positions for blocks with unknown parent (only used for reindex) + static std::multimap mapBlocksUnknownParent; int64_t nStart = GetTimeMillis(); int nLoaded = 0; try { // This takes over fileIn and calls fclose() on it in the CBufferedFile destructor CBufferedFile blkdat(fileIn, 2*MAX_BLOCK_SIZE, MAX_BLOCK_SIZE+8, SER_DISK, CLIENT_VERSION); - uint64_t nStartByte = 0; - if (dbp) { - // (try to) skip already indexed part - CBlockFileInfo info; - if (pblocktree->ReadBlockFileInfo(dbp->nFile, info)) { - nStartByte = info.nSize; - blkdat.Seek(info.nSize); - } - } uint64_t nRewind = blkdat.GetPos(); while (!blkdat.eof()) { boost::this_thread::interruption_point(); @@ -3117,21 +3117,57 @@ bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp) try { // read block uint64_t nBlockPos = blkdat.GetPos(); + if (dbp) + dbp->nPos = nBlockPos; blkdat.SetLimit(nBlockPos + nSize); + blkdat.SetPos(nBlockPos); CBlock block; blkdat >> block; nRewind = blkdat.GetPos(); - // process block - if (nBlockPos >= nStartByte) { + // detect out of order blocks, and store them for later + uint256 hash = block.GetHash(); + if (hash != Params().HashGenesisBlock() && mapBlockIndex.find(block.hashPrevBlock) == mapBlockIndex.end()) { + LogPrint("reindex", "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), + block.hashPrevBlock.ToString()); if (dbp) - dbp->nPos = nBlockPos; + mapBlocksUnknownParent.insert(std::make_pair(block.hashPrevBlock, *dbp)); + continue; + } + + // process in case the block isn't known yet + if (mapBlockIndex.count(hash) == 0) { CValidationState state; if (ProcessBlock(state, NULL, &block, dbp)) nLoaded++; if (state.IsError()) break; } + + // Recursively process earlier encountered successors of this block + deque queue; + queue.push_back(hash); + while (!queue.empty()) { + uint256 head = queue.front(); + queue.pop_front(); + std::pair::iterator, std::multimap::iterator> range = mapBlocksUnknownParent.equal_range(head); + while (range.first != range.second) { + std::multimap::iterator it = range.first; + if (ReadBlockFromDisk(block, it->second)) + { + LogPrintf("%s: Processing out of order child %s of %s\n", __func__, block.GetHash().ToString(), + head.ToString()); + CValidationState dummy; + if (ProcessBlock(dummy, NULL, &block, &it->second)) + { + nLoaded++; + queue.push_back(block.GetHash()); + } + } + range.first++; + mapBlocksUnknownParent.erase(it); + } + } } catch (std::exception &e) { LogPrintf("%s : Deserialize or I/O error - %s", __func__, e.what()); } @@ -3226,8 +3262,7 @@ bool static AlreadyHave(const CInv& inv) pcoinsTip->HaveCoins(inv.hash); } case MSG_BLOCK: - return mapBlockIndex.count(inv.hash) || - mapOrphanBlocks.count(inv.hash); + return mapBlockIndex.count(inv.hash); } // Don't know what it is, just say we already got one return true; @@ -3375,10 +3410,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return true; } - { - LOCK(cs_main); - State(pfrom->GetId())->nLastBlockProcess = GetTimeMicros(); - } @@ -3587,6 +3618,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, LOCK(cs_main); + std::vector vToFetch; + for (unsigned int nInv = 0; nInv < vInv.size(); nInv++) { const CInv &inv = vInv[nInv]; @@ -3597,19 +3630,30 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, bool fAlreadyHave = AlreadyHave(inv); LogPrint("net", "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->id); - if (!fAlreadyHave) { - if (!fImporting && !fReindex) { - if (inv.type == MSG_BLOCK) - AddBlockToQueue(pfrom->GetId(), inv.hash); - else - pfrom->AskFor(inv); - } - } else if (inv.type == MSG_BLOCK && mapOrphanBlocks.count(inv.hash)) { - PushGetBlocks(pfrom, chainActive.Tip(), GetOrphanRoot(inv.hash)); - } + if (!fAlreadyHave && !fImporting && !fReindex && inv.type != MSG_BLOCK) + pfrom->AskFor(inv); - if (inv.type == MSG_BLOCK) + if (inv.type == MSG_BLOCK) { UpdateBlockAvailability(pfrom->GetId(), inv.hash); + if (!fAlreadyHave && !fImporting && !fReindex && !mapBlocksInFlight.count(inv.hash)) { + // First request the headers preceeding the announced block. In the normal fully-synced + // case where a new block is announced that succeeds the current tip (no reorganization), + // there are no such headers. + // Secondly, and only when we are close to being synced, we request the announced block directly, + // to avoid an extra round-trip. Note that we must *first* ask for the headers, so by the + // time the block arrives, the header chain leading up to it is already validated. Not + // doing this will result in the received block being rejected as an orphan in case it is + // not a direct successor. + pfrom->PushMessage("getheaders", chainActive.GetLocator(pindexBestHeader), inv.hash); + if (chainActive.Tip()->GetBlockTime() > GetAdjustedTime() - Params().TargetSpacing() * 20) { + vToFetch.push_back(inv); + // Mark block as in flight already, even though the actual "getdata" message only goes out + // later (within the same cs_main lock, though). + MarkBlockAsInFlight(pfrom->GetId(), inv.hash); + } + LogPrint("net", "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->id); + } + } // Track requests for our stuff g_signals.Inventory(inv.hash); @@ -3619,6 +3663,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return error("send buffer size() = %u", pfrom->nSendSize); } } + + if (!vToFetch.empty()) + pfrom->PushMessage("getdata", vToFetch); } @@ -3706,8 +3753,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // we must use CBlocks, as CBlockHeaders won't include the 0x00 nTx count at the end vector vHeaders; - int nLimit = 2000; - LogPrint("net", "getheaders %d to %s\n", (pindex ? pindex->nHeight : -1), hashStop.ToString()); + int nLimit = MAX_HEADERS_RESULTS; + LogPrint("net", "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.ToString(), pfrom->id); for (; pindex; pindex = chainActive.Next(pindex)) { vHeaders.push_back(pindex->GetBlockHeader()); @@ -3826,22 +3873,67 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } + else if (strCommand == "headers" && !fImporting && !fReindex) // Ignore headers received while importing + { + std::vector headers; + + // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. + unsigned int nCount = ReadCompactSize(vRecv); + if (nCount > MAX_HEADERS_RESULTS) { + Misbehaving(pfrom->GetId(), 20); + return error("headers message size = %u", nCount); + } + headers.resize(nCount); + for (unsigned int n = 0; n < nCount; n++) { + vRecv >> headers[n]; + ReadCompactSize(vRecv); // ignore tx count; assume it is 0. + } + + LOCK(cs_main); + + if (nCount == 0) { + // Nothing interesting. Stop asking this peers for more headers. + return true; + } + + CBlockIndex *pindexLast = NULL; + BOOST_FOREACH(const CBlockHeader& header, headers) { + CValidationState state; + if (pindexLast != NULL && header.hashPrevBlock != pindexLast->GetBlockHash()) { + Misbehaving(pfrom->GetId(), 20); + return error("non-continuous headers sequence"); + } + if (!AcceptBlockHeader(header, state, &pindexLast)) { + int nDoS; + if (state.IsInvalid(nDoS)) { + if (nDoS > 0) + Misbehaving(pfrom->GetId(), nDoS); + return error("invalid header received"); + } + } + } + + if (pindexLast) + UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); + + if (nCount == MAX_HEADERS_RESULTS && pindexLast) { + // Headers message had its maximum size; the peer may have more headers. + // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue + // from there instead. + LogPrint("net", "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); + pfrom->PushMessage("getheaders", chainActive.GetLocator(pindexLast), uint256(0)); + } + } + else if (strCommand == "block" && !fImporting && !fReindex) // Ignore blocks received while importing { CBlock block; vRecv >> block; - LogPrint("net", "received block %s peer=%d\n", block.GetHash().ToString(), pfrom->id); - CInv inv(MSG_BLOCK, block.GetHash()); - pfrom->AddInventoryKnown(inv); + LogPrint("net", "received block %s peer=%d\n", inv.hash.ToString(), pfrom->id); - { - LOCK(cs_main); - // Remember who we got this block from. - mapBlockSource[inv.hash] = pfrom->GetId(); - MarkBlockAsReceived(inv.hash, pfrom->GetId()); - } + pfrom->AddInventoryKnown(inv); CValidationState state; ProcessBlock(state, pfrom, &block); @@ -4323,9 +4415,18 @@ bool SendMessages(CNode* pto, bool fSendTrickle) state.rejects.clear(); // Start block sync - if (pto->fStartSync && !fImporting && !fReindex) { - pto->fStartSync = false; - PushGetBlocks(pto, chainActive.Tip(), uint256(0)); + if (pindexBestHeader == NULL) + pindexBestHeader = chainActive.Tip(); + bool fFetch = !pto->fInbound || (pindexBestHeader && (state.pindexLastCommonBlock ? state.pindexLastCommonBlock->nHeight : 0) + 144 > pindexBestHeader->nHeight); + if (!state.fSyncStarted && !pto->fClient && fFetch && !fImporting && !fReindex) { + // Only actively request headers from a single peer, unless we're close to today. + if (nSyncStarted == 0 || pindexBestHeader->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { + state.fSyncStarted = true; + nSyncStarted++; + CBlockIndex *pindexStart = pindexBestHeader->pprev ? pindexBestHeader->pprev : pindexBestHeader; + LogPrint("net", "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->id, pto->nStartingHeight); + pto->PushMessage("getheaders", chainActive.GetLocator(pindexStart), uint256(0)); + } } // Resend wallet transactions that haven't gotten in a block yet @@ -4384,35 +4485,35 @@ bool SendMessages(CNode* pto, bool fSendTrickle) if (!vInv.empty()) pto->PushMessage("inv", vInv); - - // Detect stalled peers. Require that blocks are in flight, we haven't - // received a (requested) block in one minute, and that all blocks are - // in flight for over two minutes, since we first had a chance to - // process an incoming block. + // Detect whether we're stalling int64_t nNow = GetTimeMicros(); - if (!pto->fDisconnect && state.nBlocksInFlight && - state.nLastBlockReceive < state.nLastBlockProcess - BLOCK_DOWNLOAD_TIMEOUT*1000000 && - state.vBlocksInFlight.front().nTime < state.nLastBlockProcess - 2*BLOCK_DOWNLOAD_TIMEOUT*1000000) { - LogPrintf("Peer %s is stalling block download, disconnecting\n", state.name); + if (!pto->fDisconnect && state.nStallingSince && state.nStallingSince < nNow - 1000000 * BLOCK_STALLING_TIMEOUT) { + // Stalling only triggers when the block download window cannot move. During normal steady state, + // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection + // should only happen during initial block download. + LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->id); pto->fDisconnect = true; } - // Update knowledge of peer's block availability. - ProcessBlockAvailability(pto->GetId()); - // // Message: getdata (blocks) // vector vGetData; - while (!pto->fDisconnect && state.nBlocksToDownload && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - uint256 hash = state.vBlocksToDownload.front(); - vGetData.push_back(CInv(MSG_BLOCK, hash)); - MarkBlockAsInFlight(pto->GetId(), hash); - LogPrint("net", "Requesting block %s peer=%d\n", hash.ToString(), pto->id); - if (vGetData.size() >= 1000) - { - pto->PushMessage("getdata", vGetData); - vGetData.clear(); + if (!pto->fDisconnect && !pto->fClient && fFetch && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + vector vToDownload; + NodeId staller = -1; + FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller); + BOOST_FOREACH(CBlockIndex *pindex, vToDownload) { + vGetData.push_back(CInv(MSG_BLOCK, pindex->GetBlockHash())); + MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), pindex); + LogPrint("net", "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), + pindex->nHeight, pto->id); + } + if (state.nBlocksInFlight == 0 && staller != -1) { + if (State(staller)->nStallingSince == 0) { + State(staller)->nStallingSince = nNow; + LogPrint("net", "Stall started peer=%d\n", staller); + } } } @@ -4519,12 +4620,6 @@ public: delete (*it1).second; mapBlockIndex.clear(); - // orphan blocks - std::map::iterator it2 = mapOrphanBlocks.begin(); - for (; it2 != mapOrphanBlocks.end(); it2++) - delete (*it2).second; - mapOrphanBlocks.clear(); - // orphan transactions mapOrphanTransactions.clear(); mapOrphanTransactionsByPrev.clear(); diff --git a/src/main.h b/src/main.h index cad7eebfb7..7939b087ec 100644 --- a/src/main.h +++ b/src/main.h @@ -72,9 +72,17 @@ static const int MAX_SCRIPTCHECK_THREADS = 16; /** -par default (number of script-checking threads, 0 = auto) */ static const int DEFAULT_SCRIPTCHECK_THREADS = 0; /** Number of blocks that can be requested at any given time from a single peer. */ -static const int MAX_BLOCKS_IN_TRANSIT_PER_PEER = 128; -/** Timeout in seconds before considering a block download peer unresponsive. */ -static const unsigned int BLOCK_DOWNLOAD_TIMEOUT = 60; +static const int MAX_BLOCKS_IN_TRANSIT_PER_PEER = 16; +/** Timeout in seconds during which a peer must stall block download progress before being disconnected. */ +static const unsigned int BLOCK_STALLING_TIMEOUT = 2; +/** Number of headers sent in one getheaders result. We rely on the assumption that if a peer sends + * less than this number, we reached their tip. Changing this value is a protocol upgrade. */ +static const unsigned int MAX_HEADERS_RESULTS = 2000; +/** Size of the "block download window": how far ahead of our current height do we fetch? + * Larger windows tolerate larger download speed differences between peer, but increase the potential + * degree of disordering of blocks on disk (which make reindexing and in the future perhaps pruning + * harder). We'll probably want to make this a per-peer adaptive value at some point. */ +static const unsigned int BLOCK_DOWNLOAD_WINDOW = 1024; /** "reject" message codes **/ static const unsigned char REJECT_MALFORMED = 0x01; @@ -110,6 +118,9 @@ extern bool fIsBareMultisigStd; extern unsigned int nCoinCacheSize; extern CFeeRate minRelayTxFee; +// Best header we've seen so far (used for getheaders queries' starting points). +extern CBlockIndex *pindexBestHeader; + // Minimum disk space required - used in CheckDiskSpace() static const uint64_t nMinDiskSpace = 52428800; @@ -137,8 +148,6 @@ void RegisterNodeSignals(CNodeSignals& nodeSignals); /** Unregister a network node */ void UnregisterNodeSignals(CNodeSignals& nodeSignals); -void PushGetBlocks(CNode* pnode, CBlockIndex* pindexBegin, uint256 hashEnd); - /** Process an incoming block */ bool ProcessBlock(CValidationState &state, CNode* pfrom, CBlock* pblock, CDiskBlockPos *dbp = NULL); /** Check whether enough disk space is available for an incoming block */ @@ -193,6 +202,8 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa struct CNodeStateStats { int nMisbehavior; int nSyncHeight; + int nCommonHeight; + std::vector vHeightInFlight; }; struct CDiskTxPos : public CDiskBlockPos @@ -439,9 +450,6 @@ bool DisconnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex // Apply the effects of this block (with given index) on the UTXO set represented by coins bool ConnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& coins, bool fJustCheck = false); -// Add this block to the block index, and if necessary, switch the active block chain to this -bool AddToBlockIndex(CBlock& block, CValidationState& state, const CDiskBlockPos& pos); - // Context-independent validity checks bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, bool fCheckPOW = true); bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW = true, bool fCheckMerkleRoot = true); @@ -449,7 +457,7 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW = t // Store block on disk // if dbp is provided, the file is known to already reside on disk bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex **pindex, CDiskBlockPos* dbp = NULL); -bool AcceptBlockHeader(CBlockHeader& block, CValidationState& state, CBlockIndex **ppindex= NULL); +bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex **ppindex= NULL); diff --git a/src/net.cpp b/src/net.cpp index dd5cb480cc..50b435cf14 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -73,7 +73,6 @@ map mapLocalHost; static bool vfReachable[NET_MAX] = {}; static bool vfLimited[NET_MAX] = {}; static CNode* pnodeLocalHost = NULL; -static CNode* pnodeSync = NULL; uint64_t nLocalHostNonce = 0; static std::vector vhListenSocket; CAddrMan addrman; @@ -519,10 +518,6 @@ void CNode::CloseSocketDisconnect() TRY_LOCK(cs_vRecvMsg, lockRecv); if (lockRecv) vRecvMsg.clear(); - - // if this was the sync node, we'll need a new one - if (this == pnodeSync) - pnodeSync = NULL; } void CNode::PushVersion() @@ -615,7 +610,6 @@ void CNode::copyStats(CNodeStats &stats) X(nSendBytes); X(nRecvBytes); X(fWhitelisted); - stats.fSyncNode = (this == pnodeSync); // It is common for nodes with good ping times to suddenly become lagged, // due to a new block arriving or other large transfer. @@ -1487,61 +1481,20 @@ bool OpenNetworkConnection(const CAddress& addrConnect, CSemaphoreGrant *grantOu } -// for now, use a very simple selection metric: the node from which we received -// most recently -static int64_t NodeSyncScore(const CNode *pnode) { - return pnode->nLastRecv; -} - -void static StartSync(const vector &vNodes) { - CNode *pnodeNewSync = NULL; - int64_t nBestScore = 0; - - int nBestHeight = g_signals.GetHeight().get_value_or(0); - - // Iterate over all nodes - BOOST_FOREACH(CNode* pnode, vNodes) { - // check preconditions for allowing a sync - if (!pnode->fClient && !pnode->fOneShot && - !pnode->fDisconnect && pnode->fSuccessfullyConnected && - (pnode->nStartingHeight > (nBestHeight - 144)) && - (pnode->nVersion < NOBLKS_VERSION_START || pnode->nVersion >= NOBLKS_VERSION_END)) { - // if ok, compare node's score with the best so far - int64_t nScore = NodeSyncScore(pnode); - if (pnodeNewSync == NULL || nScore > nBestScore) { - pnodeNewSync = pnode; - nBestScore = nScore; - } - } - } - // if a new sync candidate was found, start sync! - if (pnodeNewSync) { - pnodeNewSync->fStartSync = true; - pnodeSync = pnodeNewSync; - } -} - void ThreadMessageHandler() { SetThreadPriority(THREAD_PRIORITY_BELOW_NORMAL); while (true) { - bool fHaveSyncNode = false; - vector vNodesCopy; { LOCK(cs_vNodes); vNodesCopy = vNodes; BOOST_FOREACH(CNode* pnode, vNodesCopy) { pnode->AddRef(); - if (pnode == pnodeSync) - fHaveSyncNode = true; } } - if (!fHaveSyncNode) - StartSync(vNodesCopy); - // Poll the connected nodes for messages CNode* pnodeTrickle = NULL; if (!vNodesCopy.empty()) @@ -2078,10 +2031,7 @@ CNode::CNode(SOCKET hSocketIn, CAddress addrIn, std::string addrNameIn, bool fIn nSendSize = 0; nSendOffset = 0; hashContinue = 0; - pindexLastGetBlocksBegin = 0; - hashLastGetBlocksEnd = 0; nStartingHeight = -1; - fStartSync = false; fGetAddr = false; fRelayTxes = false; setInventoryKnown.max_size(SendBufferSize() / 1000); diff --git a/src/net.h b/src/net.h index ad0a1df7e2..18da24183f 100644 --- a/src/net.h +++ b/src/net.h @@ -158,7 +158,6 @@ public: int nStartingHeight; uint64_t nSendBytes; uint64_t nRecvBytes; - bool fSyncNode; bool fWhitelisted; double dPingTime; double dPingWait; @@ -276,10 +275,7 @@ protected: public: uint256 hashContinue; - CBlockIndex* pindexLastGetBlocksBegin; - uint256 hashLastGetBlocksEnd; int nStartingHeight; - bool fStartSync; // flood relay std::vector vAddrToSend; diff --git a/src/pow.cpp b/src/pow.cpp index d50222849c..75fbfc6a6d 100644 --- a/src/pow.cpp +++ b/src/pow.cpp @@ -98,39 +98,6 @@ bool CheckProofOfWork(uint256 hash, unsigned int nBits) return true; } -// -// true if nBits is greater than the minimum amount of work that could -// possibly be required deltaTime after minimum work required was nBase -// -bool CheckMinWork(unsigned int nBits, unsigned int nBase, int64_t deltaTime) -{ - bool fOverflow = false; - uint256 bnNewBlock; - bnNewBlock.SetCompact(nBits, NULL, &fOverflow); - if (fOverflow) - return false; - - const uint256 &bnLimit = Params().ProofOfWorkLimit(); - // Testnet has min-difficulty blocks - // after Params().TargetSpacing()*2 time between blocks: - if (Params().AllowMinDifficultyBlocks() && deltaTime > Params().TargetSpacing()*2) - return bnNewBlock <= bnLimit; - - uint256 bnResult; - bnResult.SetCompact(nBase); - while (deltaTime > 0 && bnResult < bnLimit) - { - // Maximum 400% adjustment... - bnResult *= 4; - // ... in best-case exactly 4-times-normal target time - deltaTime -= Params().TargetTimespan()*4; - } - if (bnResult > bnLimit) - bnResult = bnLimit; - - return bnNewBlock <= bnResult; -} - void UpdateTime(CBlockHeader* pblock, const CBlockIndex* pindexPrev) { pblock->nTime = std::max(pindexPrev->GetMedianTimePast()+1, GetAdjustedTime()); diff --git a/src/pow.h b/src/pow.h index 5d91108ac4..233d1f3795 100644 --- a/src/pow.h +++ b/src/pow.h @@ -16,8 +16,6 @@ unsigned int GetNextWorkRequired(const CBlockIndex* pindexLast, const CBlockHead /** Check whether a block hash satisfies the proof-of-work requirement specified by nBits */ bool CheckProofOfWork(uint256 hash, unsigned int nBits); -/** Check the work is more than the minimum a received block needs, without knowing its direct parent */ -bool CheckMinWork(unsigned int nBits, unsigned int nBase, int64_t deltaTime); void UpdateTime(CBlockHeader* block, const CBlockIndex* pindexPrev); diff --git a/src/qt/forms/rpcconsole.ui b/src/qt/forms/rpcconsole.ui index 7f28209c9a..898df2b080 100644 --- a/src/qt/forms/rpcconsole.ui +++ b/src/qt/forms/rpcconsole.ui @@ -836,29 +836,6 @@ - - - - Sync Node - - - - - - - IBeamCursor - - - N/A - - - Qt::PlainText - - - Qt::LinksAccessibleByMouse|Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse - - - diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 8129353d4b..2d2d448b49 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -611,7 +611,6 @@ void RPCConsole::updateNodeDetail(const CNodeCombinedStats *stats) ui->peerSubversion->setText(QString::fromStdString(stats->nodeStats.cleanSubVer)); ui->peerDirection->setText(stats->nodeStats.fInbound ? tr("Inbound") : tr("Outbound")); ui->peerHeight->setText(QString("%1").arg(stats->nodeStats.nStartingHeight)); - ui->peerSyncNode->setText(stats->nodeStats.fSyncNode ? tr("Yes") : tr("No")); // This check fails for example if the lock was busy and // nodeStateStats couldn't be fetched. diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index 24175215bf..5beac0512a 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -445,6 +445,7 @@ Value getblockchaininfo(const Array& params, bool fHelp) "{\n" " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" " \"blocks\": xxxxxx, (numeric) the current number of blocks processed in the server\n" + " \"headers\": xxxxxx, (numeric) the current number of headers we have validated\n" " \"bestblockhash\": \"...\", (string) the hash of the currently best block\n" " \"difficulty\": xxxxxx, (numeric) the current difficulty\n" " \"verificationprogress\": xxxx, (numeric) estimate of verification progress [0..1]\n" @@ -458,6 +459,7 @@ Value getblockchaininfo(const Array& params, bool fHelp) Object obj; obj.push_back(Pair("chain", Params().NetworkIDString())); obj.push_back(Pair("blocks", (int)chainActive.Height())); + obj.push_back(Pair("headers", pindexBestHeader ? pindexBestHeader->nHeight : -1)); obj.push_back(Pair("bestblockhash", chainActive.Tip()->GetBlockHash().GetHex())); obj.push_back(Pair("difficulty", (double)GetDifficulty())); obj.push_back(Pair("verificationprogress", Checkpoints::GuessVerificationProgress(chainActive.Tip()))); diff --git a/src/rpcnet.cpp b/src/rpcnet.cpp index bc19d1372a..12dcd5b540 100644 --- a/src/rpcnet.cpp +++ b/src/rpcnet.cpp @@ -97,7 +97,12 @@ Value getpeerinfo(const Array& params, bool fHelp) " \"inbound\": true|false, (boolean) Inbound (true) or Outbound (false)\n" " \"startingheight\": n, (numeric) The starting height (block) of the peer\n" " \"banscore\": n, (numeric) The ban score\n" - " \"syncnode\": true|false (boolean) if sync node\n" + " \"synced_headers\": n, (numeric) The last header we have in common with this peer\n" + " \"synced_blocks\": n, (numeric) The last block we have in common with this peer\n" + " \"inflight\": [\n" + " n, (numeric) The heights of blocks we're currently asking from this peer\n" + " ...\n" + " ]\n" " }\n" " ,...\n" "]\n" @@ -137,9 +142,14 @@ Value getpeerinfo(const Array& params, bool fHelp) obj.push_back(Pair("startingheight", stats.nStartingHeight)); if (fStateStats) { obj.push_back(Pair("banscore", statestats.nMisbehavior)); - obj.push_back(Pair("syncheight", statestats.nSyncHeight)); + obj.push_back(Pair("synced_headers", statestats.nSyncHeight)); + obj.push_back(Pair("synced_blocks", statestats.nCommonHeight)); + Array heights; + BOOST_FOREACH(int height, statestats.vHeightInFlight) { + heights.push_back(height); + } + obj.push_back(Pair("inflight", heights)); } - obj.push_back(Pair("syncnode", stats.fSyncNode)); obj.push_back(Pair("whitelisted", stats.fWhitelisted)); ret.push_back(obj); diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index 7bec12b665..f9746fdaa5 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -106,51 +106,6 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) BOOST_CHECK(!CNode::IsBanned(addr)); } -static bool CheckNBits(unsigned int nbits1, int64_t time1, unsigned int nbits2, int64_t time2)\ -{ - if (time1 > time2) - return CheckNBits(nbits2, time2, nbits1, time1); - int64_t deltaTime = time2-time1; - - return CheckMinWork(nbits2, nbits1, deltaTime); -} - -BOOST_AUTO_TEST_CASE(DoS_checknbits) -{ - using namespace boost::assign; // for 'map_list_of()' - - // Timestamps,nBits from the bitcoin block chain. - // These are the block-chain checkpoint blocks - typedef std::map BlockData; - BlockData chainData = - map_list_of(1239852051,486604799)(1262749024,486594666) - (1279305360,469854461)(1280200847,469830746)(1281678674,469809688) - (1296207707,453179945)(1302624061,453036989)(1309640330,437004818) - (1313172719,436789733); - - // Make sure CheckNBits considers every combination of block-chain-lock-in-points - // "sane": - BOOST_FOREACH(const BlockData::value_type& i, chainData) - { - BOOST_FOREACH(const BlockData::value_type& j, chainData) - { - BOOST_CHECK(CheckNBits(i.second, i.first, j.second, j.first)); - } - } - - // Test a couple of insane combinations: - BlockData::value_type firstcheck = *(chainData.begin()); - BlockData::value_type lastcheck = *(chainData.rbegin()); - - // First checkpoint difficulty at or a while after the last checkpoint time should fail when - // compared to last checkpoint - BOOST_CHECK(!CheckNBits(firstcheck.second, lastcheck.first+60*10, lastcheck.second, lastcheck.first)); - BOOST_CHECK(!CheckNBits(firstcheck.second, lastcheck.first+60*60*24*14, lastcheck.second, lastcheck.first)); - - // ... but OK if enough time passed for difficulty to adjust downward: - BOOST_CHECK(CheckNBits(firstcheck.second, lastcheck.first+60*60*24*365*4, lastcheck.second, lastcheck.first)); -} - CTransaction RandomOrphan() { std::map::iterator it; diff --git a/src/version.h b/src/version.h index 75cbec39b7..a1e440de24 100644 --- a/src/version.h +++ b/src/version.h @@ -33,8 +33,11 @@ static const int PROTOCOL_VERSION = 70002; // initial proto version, to be increased after version/verack negotiation static const int INIT_PROTO_VERSION = 209; +// In this version, 'getheaders' was introduced. +static const int GETHEADERS_VERSION = 31800; + // disconnect from peers older than this proto version -static const int MIN_PEER_PROTO_VERSION = 209; +static const int MIN_PEER_PROTO_VERSION = GETHEADERS_VERSION; // nTime field added to CAddress, starting with this version; // if possible, avoid requesting addresses nodes older than this