Merge #9499: Use recent-rejects, orphans, and recently-replaced txn for compact-block-reconstruction

c594580 Add braces around AddToCompactExtraTransactions (Matt Corallo)
1ccfe9b Clarify comment about mempool/extra conflicts (Matt Corallo)
fac4c78 Make PartiallyDownloadedBlock::InitData's second param const (Matt Corallo)
b55b416 Add extra_count lower bound to compact reconstruction debug print (Matt Corallo)
863edb4 Consider all (<100k memusage) txn for compact-block-extra-txn cache (Matt Corallo)
7f8c8ca Consider all orphan txn for compact-block-extra-txn cache (Matt Corallo)
93380c5 Use replaced transactions in compact block reconstruction (Matt Corallo)
1531652 Keep shared_ptrs to recently-replaced txn for compact blocks (Matt Corallo)
edded80 Make ATMP optionally return the CTransactionRefs it replaced (Matt Corallo)
c735540 Move ORPHAN constants from validation.h to net_processing.h (Matt Corallo)
This commit is contained in:
Wladimir J. van der Laan 2017-01-19 09:02:26 +01:00
commit 9c9af5ab2d
No known key found for this signature in database
GPG Key ID: 74810B012346C9A6
11 changed files with 105 additions and 31 deletions

View File

@ -47,7 +47,7 @@ uint64_t CBlockHeaderAndShortTxIDs::GetShortID(const uint256& txhash) const {
ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& cmpctblock) {
ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector<std::pair<uint256, CTransactionRef>>& extra_txn) {
if (cmpctblock.header.IsNull() || (cmpctblock.shorttxids.empty() && cmpctblock.prefilledtxn.empty()))
return READ_STATUS_INVALID;
if (cmpctblock.shorttxids.size() + cmpctblock.prefilledtxn.size() > MAX_BLOCK_BASE_SIZE / MIN_TRANSACTION_BASE_SIZE)
@ -104,6 +104,7 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c
return READ_STATUS_FAILED; // Short ID collision
std::vector<bool> have_txn(txn_available.size());
{
LOCK(pool->cs);
const std::vector<std::pair<uint256, CTxMemPool::txiter> >& vTxHashes = pool->vTxHashes;
for (size_t i = 0; i < vTxHashes.size(); i++) {
@ -130,6 +131,38 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c
if (mempool_count == shorttxids.size())
break;
}
}
for (size_t i = 0; i < extra_txn.size(); i++) {
uint64_t shortid = cmpctblock.GetShortID(extra_txn[i].first);
std::unordered_map<uint64_t, uint16_t>::iterator idit = shorttxids.find(shortid);
if (idit != shorttxids.end()) {
if (!have_txn[idit->second]) {
txn_available[idit->second] = extra_txn[i].second;
have_txn[idit->second] = true;
mempool_count++;
extra_count++;
} else {
// If we find two mempool/extra txn that match the short id, just
// request it.
// This should be rare enough that the extra bandwidth doesn't matter,
// but eating a round-trip due to FillBlock failure would be annoying
// Note that we dont want duplication between extra_txn and mempool to
// trigger this case, so we compare witness hashes first
if (txn_available[idit->second] &&
txn_available[idit->second]->GetWitnessHash() != extra_txn[i].second->GetWitnessHash()) {
txn_available[idit->second].reset();
mempool_count--;
extra_count--;
}
}
}
// Though ideally we'd continue scanning for the two-txn-match-shortid case,
// the performance win of an early exit here is too good to pass up and worth
// the extra risk.
if (mempool_count == shorttxids.size())
break;
}
LogPrint("cmpctblock", "Initialized PartiallyDownloadedBlock for block %s using a cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), GetSerializeSize(cmpctblock, SER_NETWORK, PROTOCOL_VERSION));
@ -176,7 +209,7 @@ ReadStatus PartiallyDownloadedBlock::FillBlock(CBlock& block, const std::vector<
return READ_STATUS_CHECKBLOCK_FAILED;
}
LogPrint("cmpctblock", "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool and %lu txn requested\n", hash.ToString(), prefilled_count, mempool_count, vtx_missing.size());
LogPrint("cmpctblock", "Successfully reconstructed block %s with %lu txn prefilled, %lu txn from mempool (incl at least %lu from extra pool) and %lu txn requested\n", hash.ToString(), prefilled_count, mempool_count, extra_count, vtx_missing.size());
if (vtx_missing.size() < 5) {
for (const auto& tx : vtx_missing)
LogPrint("cmpctblock", "Reconstructed block %s required tx %s\n", hash.ToString(), tx->GetHash().ToString());

View File

@ -194,13 +194,14 @@ public:
class PartiallyDownloadedBlock {
protected:
std::vector<CTransactionRef> txn_available;
size_t prefilled_count = 0, mempool_count = 0;
size_t prefilled_count = 0, mempool_count = 0, extra_count = 0;
CTxMemPool* pool;
public:
CBlockHeader header;
PartiallyDownloadedBlock(CTxMemPool* poolIn) : pool(poolIn) {}
ReadStatus InitData(const CBlockHeaderAndShortTxIDs& cmpctblock);
// extra_txn is a list of extra transactions to look at, in <witness hash, reference> form
ReadStatus InitData(const CBlockHeaderAndShortTxIDs& cmpctblock, const std::vector<std::pair<uint256, CTransactionRef>>& extra_txn);
bool IsTxAvailable(size_t index) const;
ReadStatus FillBlock(CBlock& block, const std::vector<CTransactionRef>& vtx_missing);
};

View File

@ -345,6 +345,7 @@ std::string HelpMessage(HelpMessageMode mode)
strUsage += HelpMessageOpt("-maxorphantx=<n>", strprintf(_("Keep at most <n> unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS));
strUsage += HelpMessageOpt("-maxmempool=<n>", strprintf(_("Keep the transaction memory pool below <n> megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE));
strUsage += HelpMessageOpt("-mempoolexpiry=<n>", strprintf(_("Do not keep transactions in the mempool longer than <n> hours (default: %u)"), DEFAULT_MEMPOOL_EXPIRY));
strUsage += HelpMessageOpt("-blockreconstructionextratxn=<n>", strprintf(_("Extra transactions to keep in memory for compact block reconstructions (default: %u)"), DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN));
strUsage += HelpMessageOpt("-par=<n>", strprintf(_("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)"),
-GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS));
#ifndef WIN32

View File

@ -59,6 +59,9 @@ map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(cs_main);
map<COutPoint, set<map<uint256, COrphanTx>::iterator, IteratorComparator>> mapOrphanTransactionsByPrev GUARDED_BY(cs_main);
void EraseOrphansFor(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
static size_t vExtraTxnForCompactIt = 0;
static std::vector<std::pair<uint256, CTransactionRef>> vExtraTxnForCompact GUARDED_BY(cs_main);
static const uint64_t RANDOMIZER_ID_ADDRESS_RELAY = 0x3cac0035b5866b90ULL; // SHA256("main address relay")[0:8]
// Internal stuff
@ -591,6 +594,17 @@ void UnregisterNodeSignals(CNodeSignals& nodeSignals)
// mapOrphanTransactions
//
void AddToCompactExtraTransactions(const CTransactionRef& tx)
{
size_t max_extra_txn = GetArg("-blockreconstructionextratxn", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN);
if (max_extra_txn <= 0)
return;
if (!vExtraTxnForCompact.size())
vExtraTxnForCompact.resize(max_extra_txn);
vExtraTxnForCompact[vExtraTxnForCompactIt] = std::make_pair(tx->GetWitnessHash(), tx);
vExtraTxnForCompactIt = (vExtraTxnForCompactIt + 1) % max_extra_txn;
}
bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
{
const uint256& hash = tx->GetHash();
@ -617,6 +631,8 @@ bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRE
mapOrphanTransactionsByPrev[txin.prevout].insert(ret.first);
}
AddToCompactExtraTransactions(tx);
LogPrint("mempool", "stored orphan tx %s (mapsz %u outsz %u)\n", hash.ToString(),
mapOrphanTransactions.size(), mapOrphanTransactionsByPrev.size());
return true;
@ -1722,7 +1738,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
pfrom->setAskFor.erase(inv.hash);
mapAlreadyAskedFor.erase(inv.hash);
if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, ptx, true, &fMissingInputs)) {
std::list<CTransactionRef> lRemovedTxn;
if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, ptx, true, &fMissingInputs, &lRemovedTxn)) {
mempool.check(pcoinsTip);
RelayTransaction(tx, connman);
for (unsigned int i = 0; i < tx.vout.size(); i++) {
@ -1760,7 +1778,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
if (setMisbehaving.count(fromPeer))
continue;
if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, true, &fMissingInputs2)) {
if (AcceptToMemoryPool(mempool, stateDummy, porphanTx, true, &fMissingInputs2, &lRemovedTxn)) {
LogPrint("mempool", " accepted orphan tx %s\n", orphanHash.ToString());
RelayTransaction(orphanTx, connman);
for (unsigned int i = 0; i < orphanTx.vout.size(); i++) {
@ -1833,6 +1851,11 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
// See https://github.com/bitcoin/bitcoin/issues/8279 for details.
assert(recentRejects);
recentRejects->insert(tx.GetHash());
if (RecursiveDynamicUsage(*ptx) < 100000) {
AddToCompactExtraTransactions(ptx);
}
} else if (tx.HasWitness() && RecursiveDynamicUsage(*ptx) < 100000) {
AddToCompactExtraTransactions(ptx);
}
if (pfrom->fWhitelisted && GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY)) {
@ -1853,6 +1876,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
}
}
}
for (const CTransactionRef& tx : lRemovedTxn)
AddToCompactExtraTransactions(tx);
int nDoS = 0;
if (state.IsInvalid(nDoS))
{
@ -1969,7 +1996,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
}
PartiallyDownloadedBlock& partialBlock = *(*queuedBlockIt)->partialBlock;
ReadStatus status = partialBlock.InitData(cmpctblock);
ReadStatus status = partialBlock.InitData(cmpctblock, vExtraTxnForCompact);
if (status == READ_STATUS_INVALID) {
MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case of whitelist
Misbehaving(pfrom->GetId(), 100);
@ -2005,7 +2032,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
// Optimistically try to reconstruct anyway since we might be
// able to without any round trips.
PartiallyDownloadedBlock tempBlock(&mempool);
ReadStatus status = tempBlock.InitData(cmpctblock);
ReadStatus status = tempBlock.InitData(cmpctblock, vExtraTxnForCompact);
if (status != READ_STATUS_OK) {
// TODO: don't ignore failures
return true;

View File

@ -9,6 +9,15 @@
#include "net.h"
#include "validationinterface.h"
/** Default for -maxorphantx, maximum number of orphan transactions kept in memory */
static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100;
/** Expiration time for orphan transactions in seconds */
static const int64_t ORPHAN_TX_EXPIRE_TIME = 20 * 60;
/** Minimum time between orphan transactions expire time checks in seconds */
static const int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60;
/** Default number of orphan+recently-replaced txn to keep around for block reconstruction */
static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100;
/** Register with a network node to receive its signals */
void RegisterNodeSignals(CNodeSignals& nodeSignals);
/** Unregister a network node */

View File

@ -903,7 +903,7 @@ UniValue sendrawtransaction(const JSONRPCRequest& request)
// push to local node and sync with wallets
CValidationState state;
bool fMissingInputs;
if (!AcceptToMemoryPool(mempool, state, std::move(tx), fLimitFree, &fMissingInputs, false, nMaxRawTxFee)) {
if (!AcceptToMemoryPool(mempool, state, std::move(tx), fLimitFree, &fMissingInputs, NULL, false, nMaxRawTxFee)) {
if (state.IsInvalid()) {
throw JSONRPCError(RPC_TRANSACTION_REJECTED, strprintf("%i: %s", state.GetRejectCode(), state.GetRejectReason()));
} else {

View File

@ -11,6 +11,8 @@
#include <boost/test/unit_test.hpp>
std::vector<std::pair<uint256, CTransactionRef>> extra_txn;
struct RegtestingSetup : public TestingSetup {
RegtestingSetup() : TestingSetup(CBaseChainParams::REGTEST) {}
};
@ -73,7 +75,7 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest)
stream >> shortIDs2;
PartiallyDownloadedBlock partialBlock(&pool);
BOOST_CHECK(partialBlock.InitData(shortIDs2) == READ_STATUS_OK);
BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK);
BOOST_CHECK( partialBlock.IsTxAvailable(0));
BOOST_CHECK(!partialBlock.IsTxAvailable(1));
BOOST_CHECK( partialBlock.IsTxAvailable(2));
@ -179,7 +181,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest)
stream >> shortIDs2;
PartiallyDownloadedBlock partialBlock(&pool);
BOOST_CHECK(partialBlock.InitData(shortIDs2) == READ_STATUS_OK);
BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK);
BOOST_CHECK(!partialBlock.IsTxAvailable(0));
BOOST_CHECK( partialBlock.IsTxAvailable(1));
BOOST_CHECK( partialBlock.IsTxAvailable(2));
@ -245,7 +247,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest)
stream >> shortIDs2;
PartiallyDownloadedBlock partialBlock(&pool);
BOOST_CHECK(partialBlock.InitData(shortIDs2) == READ_STATUS_OK);
BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK);
BOOST_CHECK( partialBlock.IsTxAvailable(0));
BOOST_CHECK( partialBlock.IsTxAvailable(1));
BOOST_CHECK( partialBlock.IsTxAvailable(2));
@ -300,7 +302,7 @@ BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest)
stream >> shortIDs2;
PartiallyDownloadedBlock partialBlock(&pool);
BOOST_CHECK(partialBlock.InitData(shortIDs2) == READ_STATUS_OK);
BOOST_CHECK(partialBlock.InitData(shortIDs2, extra_txn) == READ_STATUS_OK);
BOOST_CHECK(partialBlock.IsTxAvailable(0));
CBlock block2;

View File

@ -23,7 +23,7 @@ ToMemPool(CMutableTransaction& tx)
LOCK(cs_main);
CValidationState state;
return AcceptToMemoryPool(mempool, state, MakeTransactionRef(tx), false, NULL, true, 0);
return AcceptToMemoryPool(mempool, state, MakeTransactionRef(tx), false, NULL, NULL, true, 0);
}
BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup)

View File

@ -540,8 +540,8 @@ static bool IsCurrentForFeeEstimation()
}
bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransactionRef& ptx, bool fLimitFree,
bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee,
std::vector<uint256>& vHashTxnToUncache)
bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced,
bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<uint256>& vHashTxnToUncache)
{
const CTransaction& tx = *ptx;
const uint256 hash = tx.GetHash();
@ -952,6 +952,8 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C
hash.ToString(),
FormatMoney(nModifiedFees - nConflictingFees),
(int)nSize - (int)nConflictingSize);
if (plTxnReplaced)
plTxnReplaced->push_back(it->GetSharedTx());
}
pool.RemoveStaged(allConflicting, false);
@ -977,10 +979,11 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C
}
bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree,
bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit, const CAmount nAbsurdFee)
bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced,
bool fOverrideMempoolLimit, const CAmount nAbsurdFee)
{
std::vector<uint256> vHashTxToUncache;
bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache);
bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache);
if (!res) {
BOOST_FOREACH(const uint256& hashTx, vHashTxToUncache)
pcoinsTip->Uncache(hashTx);
@ -992,9 +995,10 @@ bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const
}
bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree,
bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount nAbsurdFee)
bool* pfMissingInputs, std::list<CTransactionRef>* plTxnReplaced,
bool fOverrideMempoolLimit, const CAmount nAbsurdFee)
{
return AcceptToMemoryPoolWithTime(pool, state, tx, fLimitFree, pfMissingInputs, GetTime(), fOverrideMempoolLimit, nAbsurdFee);
return AcceptToMemoryPoolWithTime(pool, state, tx, fLimitFree, pfMissingInputs, GetTime(), plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee);
}
/** Return transaction in txOut, and if it was found inside a block, its hash is placed in hashBlock */
@ -2160,7 +2164,7 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara
const CTransaction& tx = *it;
// ignore validation errors in resurrected transactions
CValidationState stateDummy;
if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, it, false, NULL, true)) {
if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, it, false, NULL, NULL, true)) {
mempool.removeRecursive(tx);
} else if (mempool.exists(tx.GetHash())) {
vHashUpdate.push_back(tx.GetHash());

View File

@ -59,12 +59,6 @@ static const CAmount DEFAULT_TRANSACTION_MAXFEE = 0.1 * COIN;
static const CAmount HIGH_TX_FEE_PER_KB = 0.01 * COIN;
//! -maxtxfee will warn if called with a higher fee than this amount (in satoshis)
static const CAmount HIGH_MAX_TX_FEE = 100 * HIGH_TX_FEE_PER_KB;
/** Default for -maxorphantx, maximum number of orphan transactions kept in memory */
static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100;
/** Expiration time for orphan transactions in seconds */
static const int64_t ORPHAN_TX_EXPIRE_TIME = 20 * 60;
/** Minimum time between orphan transactions expire time checks in seconds */
static const int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60;
/** Default for -limitancestorcount, max number of in-mempool ancestors */
static const unsigned int DEFAULT_ANCESTOR_LIMIT = 25;
/** Default for -limitancestorsize, maximum kilobytes of tx + all in-mempool ancestors */
@ -319,13 +313,16 @@ void PruneAndFlush();
/** Prune block files up to a given height */
void PruneBlockFilesManual(int nPruneUpToHeight);
/** (try to) add transaction to memory pool **/
/** (try to) add transaction to memory pool
* plTxnReplaced will be appended to with all transactions replaced from mempool **/
bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree,
bool* pfMissingInputs, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0);
bool* pfMissingInputs, std::list<CTransactionRef>* plTxnReplaced = NULL,
bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0);
/** (try to) add transaction to memory pool with a specified acceptance time **/
bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree,
bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0);
bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced = NULL,
bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0);
/** Convert CValidationState to a human-readable message for logging */
std::string FormatStateMessage(const CValidationState &state);

View File

@ -3852,5 +3852,5 @@ int CMerkleTx::GetBlocksToMaturity() const
bool CMerkleTx::AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state)
{
return ::AcceptToMemoryPool(mempool, state, tx, true, NULL, false, nAbsurdFee);
return ::AcceptToMemoryPool(mempool, state, tx, true, NULL, NULL, false, nAbsurdFee);
}