diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 6e2bbd2df..e27fbe5ed 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -57,12 +57,12 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) std::list removed; // Nothing in pool, remove should do nothing: - testPool.remove(txParent, removed, true); + testPool.removeRecursive(txParent, removed); BOOST_CHECK_EQUAL(removed.size(), 0); // Just the parent: testPool.addUnchecked(txParent.GetHash(), entry.FromTx(txParent)); - testPool.remove(txParent, removed, true); + testPool.removeRecursive(txParent, removed); BOOST_CHECK_EQUAL(removed.size(), 1); removed.clear(); @@ -74,16 +74,16 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) testPool.addUnchecked(txGrandChild[i].GetHash(), entry.FromTx(txGrandChild[i])); } // Remove Child[0], GrandChild[0] should be removed: - testPool.remove(txChild[0], removed, true); + testPool.removeRecursive(txChild[0], removed); BOOST_CHECK_EQUAL(removed.size(), 2); removed.clear(); // ... make sure grandchild and child are gone: - testPool.remove(txGrandChild[0], removed, true); + testPool.removeRecursive(txGrandChild[0], removed); BOOST_CHECK_EQUAL(removed.size(), 0); - testPool.remove(txChild[0], removed, true); + testPool.removeRecursive(txChild[0], removed); BOOST_CHECK_EQUAL(removed.size(), 0); // Remove parent, all children/grandchildren should go: - testPool.remove(txParent, removed, true); + testPool.removeRecursive(txParent, removed); BOOST_CHECK_EQUAL(removed.size(), 5); BOOST_CHECK_EQUAL(testPool.size(), 0); removed.clear(); @@ -96,7 +96,7 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) } // Now remove the parent, as might happen if a block-re-org occurs but the parent cannot be // put into the mempool (maybe because it is non-standard): - testPool.remove(txParent, removed, true); + testPool.removeRecursive(txParent, removed); BOOST_CHECK_EQUAL(removed.size(), 6); BOOST_CHECK_EQUAL(testPool.size(), 0); removed.clear(); @@ -281,11 +281,11 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) // Now try removing tx10 and verify the sort order returns to normal std::list removed; - pool.remove(pool.mapTx.find(tx10.GetHash())->GetTx(), removed, true); + pool.removeRecursive(pool.mapTx.find(tx10.GetHash())->GetTx(), removed); CheckSort(pool, snapshotOrder); - pool.remove(pool.mapTx.find(tx9.GetHash())->GetTx(), removed, true); - pool.remove(pool.mapTx.find(tx8.GetHash())->GetTx(), removed, true); + pool.removeRecursive(pool.mapTx.find(tx9.GetHash())->GetTx(), removed); + pool.removeRecursive(pool.mapTx.find(tx8.GetHash())->GetTx(), removed); /* Now check the sort on the mining score index. * Final order should be: * @@ -317,6 +317,110 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) CheckSort(pool, sortedOrder); } +BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) +{ + CTxMemPool pool(CFeeRate(0)); + TestMemPoolEntryHelper entry; + entry.hadNoDependencies = true; + + /* 3rd highest fee */ + CMutableTransaction tx1 = CMutableTransaction(); + tx1.vout.resize(1); + tx1.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx1.vout[0].nValue = 10 * COIN; + pool.addUnchecked(tx1.GetHash(), entry.Fee(10000LL).Priority(10.0).FromTx(tx1)); + + /* highest fee */ + CMutableTransaction tx2 = CMutableTransaction(); + tx2.vout.resize(1); + tx2.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx2.vout[0].nValue = 2 * COIN; + pool.addUnchecked(tx2.GetHash(), entry.Fee(20000LL).Priority(9.0).FromTx(tx2)); + uint64_t tx2Size = ::GetSerializeSize(tx2, SER_NETWORK, PROTOCOL_VERSION); + + /* lowest fee */ + CMutableTransaction tx3 = CMutableTransaction(); + tx3.vout.resize(1); + tx3.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx3.vout[0].nValue = 5 * COIN; + pool.addUnchecked(tx3.GetHash(), entry.Fee(0LL).Priority(100.0).FromTx(tx3)); + + /* 2nd highest fee */ + CMutableTransaction tx4 = CMutableTransaction(); + tx4.vout.resize(1); + tx4.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx4.vout[0].nValue = 6 * COIN; + pool.addUnchecked(tx4.GetHash(), entry.Fee(15000LL).Priority(1.0).FromTx(tx4)); + + /* equal fee rate to tx1, but newer */ + CMutableTransaction tx5 = CMutableTransaction(); + tx5.vout.resize(1); + tx5.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx5.vout[0].nValue = 11 * COIN; + pool.addUnchecked(tx5.GetHash(), entry.Fee(10000LL).FromTx(tx5)); + BOOST_CHECK_EQUAL(pool.size(), 5); + + std::vector sortedOrder; + sortedOrder.resize(5); + sortedOrder[0] = tx2.GetHash().ToString(); // 20000 + sortedOrder[1] = tx4.GetHash().ToString(); // 15000 + // tx1 and tx5 are both 10000 + // Ties are broken by hash, not timestamp, so determine which + // hash comes first. + if (tx1.GetHash() < tx5.GetHash()) { + sortedOrder[2] = tx1.GetHash().ToString(); + sortedOrder[3] = tx5.GetHash().ToString(); + } else { + sortedOrder[2] = tx5.GetHash().ToString(); + sortedOrder[3] = tx1.GetHash().ToString(); + } + sortedOrder[4] = tx3.GetHash().ToString(); // 0 + + CheckSort(pool, sortedOrder); + + /* low fee parent with high fee child */ + /* tx6 (0) -> tx7 (high) */ + CMutableTransaction tx6 = CMutableTransaction(); + tx6.vout.resize(1); + tx6.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx6.vout[0].nValue = 20 * COIN; + uint64_t tx6Size = ::GetSerializeSize(tx6, SER_NETWORK, PROTOCOL_VERSION); + + pool.addUnchecked(tx6.GetHash(), entry.Fee(0LL).FromTx(tx6)); + BOOST_CHECK_EQUAL(pool.size(), 6); + sortedOrder.push_back(tx6.GetHash().ToString()); + CheckSort(pool, sortedOrder); + + CMutableTransaction tx7 = CMutableTransaction(); + tx7.vin.resize(1); + tx7.vin[0].prevout = COutPoint(tx6.GetHash(), 0); + tx7.vin[0].scriptSig = CScript() << OP_11; + tx7.vout.resize(1); + tx7.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx7.vout[0].nValue = 10 * COIN; + uint64_t tx7Size = ::GetSerializeSize(tx7, SER_NETWORK, PROTOCOL_VERSION); + + /* set the fee to just below tx2's feerate when including ancestor */ + CAmount fee = (20000/tx2Size)*(tx7Size + tx6Size) - 1; + + //CTxMemPoolEntry entry7(tx7, fee, 2, 10.0, 1, true); + pool.addUnchecked(tx7.GetHash(), entry.Fee(fee).FromTx(tx7)); + BOOST_CHECK_EQUAL(pool.size(), 7); + sortedOrder.insert(sortedOrder.begin()+1, tx7.GetHash().ToString()); + CheckSort(pool, sortedOrder); + + /* after tx6 is mined, tx7 should move up in the sort */ + std::vector vtx; + vtx.push_back(tx6); + std::list dummy; + pool.removeForBlock(vtx, 1, dummy, false); + + sortedOrder.erase(sortedOrder.begin()+1); + sortedOrder.pop_back(); + sortedOrder.insert(sortedOrder.begin(), tx7.GetHash().ToString()); + CheckSort(pool, sortedOrder); +} + BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) { diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 043d3e0c4..0c4e6a2cf 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -39,6 +39,11 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, assert(inChainInputValue <= nValueIn); feeDelta = 0; + + nCountWithAncestors = 1; + nSizeWithAncestors = nTxSize; + nModFeesWithAncestors = nFee; + nSigOpCountWithAncestors = sigOpCount; } CTxMemPoolEntry::CTxMemPoolEntry(const CTxMemPoolEntry& other) @@ -59,6 +64,7 @@ CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const void CTxMemPoolEntry::UpdateFeeDelta(int64_t newFeeDelta) { nModFeesWithDescendants += newFeeDelta - feeDelta; + nModFeesWithAncestors += newFeeDelta - feeDelta; feeDelta = newFeeDelta; } @@ -70,21 +76,13 @@ void CTxMemPoolEntry::UpdateLockPoints(const LockPoints& lp) // Update the given tx for any in-mempool descendants. // Assumes that setMemPoolChildren is correct for the given tx and all // descendants. -bool CTxMemPool::UpdateForDescendants(txiter updateIt, int maxDescendantsToVisit, cacheMap &cachedDescendants, const std::set &setExclude) +void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap &cachedDescendants, const std::set &setExclude) { - // Track the number of entries (outside setExclude) that we'd need to visit - // (will bail out if it exceeds maxDescendantsToVisit) - int nChildrenToVisit = 0; - setEntries stageEntries, setAllDescendants; stageEntries = GetMemPoolChildren(updateIt); while (!stageEntries.empty()) { const txiter cit = *stageEntries.begin(); - if (cit->IsDirty()) { - // Don't consider any more children if any descendant is dirty - return false; - } setAllDescendants.insert(cit); stageEntries.erase(cit); const setEntries &setChildren = GetMemPoolChildren(cit); @@ -94,22 +92,11 @@ bool CTxMemPool::UpdateForDescendants(txiter updateIt, int maxDescendantsToVisit // We've already calculated this one, just add the entries for this set // but don't traverse again. BOOST_FOREACH(const txiter cacheEntry, cacheIt->second) { - // update visit count only for new child transactions - // (outside of setExclude and stageEntries) - if (setAllDescendants.insert(cacheEntry).second && - !setExclude.count(cacheEntry->GetTx().GetHash()) && - !stageEntries.count(cacheEntry)) { - nChildrenToVisit++; - } + setAllDescendants.insert(cacheEntry); } } else if (!setAllDescendants.count(childEntry)) { - // Schedule for later processing and update our visit count - if (stageEntries.insert(childEntry).second && !setExclude.count(childEntry->GetTx().GetHash())) { - nChildrenToVisit++; - } - } - if (nChildrenToVisit > maxDescendantsToVisit) { - return false; + // Schedule for later processing + stageEntries.insert(childEntry); } } } @@ -124,16 +111,18 @@ bool CTxMemPool::UpdateForDescendants(txiter updateIt, int maxDescendantsToVisit modifyFee += cit->GetModifiedFee(); modifyCount++; cachedDescendants[updateIt].insert(cit); + // Update ancestor state for each descendant + mapTx.modify(cit, update_ancestor_state(updateIt->GetTxSize(), updateIt->GetModifiedFee(), 1, updateIt->GetSigOpCount())); } } mapTx.modify(updateIt, update_descendant_state(modifySize, modifyFee, modifyCount)); - return true; } // vHashesToUpdate is the set of transaction hashes from a disconnected block // which has been re-added to the mempool. // for each entry, look for descendants that are outside hashesToUpdate, and // add fee/size information for such descendants to the parent. +// for each such descendant, also update the ancestor state to include the parent. void CTxMemPool::UpdateTransactionsFromBlock(const std::vector &vHashesToUpdate) { LOCK(cs); @@ -173,14 +162,11 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector &vHashes UpdateParent(childIter, it, true); } } - if (!UpdateForDescendants(it, 100, mapMemPoolDescendantsToUpdate, setAlreadyIncluded)) { - // Mark as dirty if we can't do the calculation. - mapTx.modify(it, set_dirty()); - } + UpdateForDescendants(it, mapMemPoolDescendantsToUpdate, setAlreadyIncluded); } } -bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntries &setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string &errString, bool fSearchForParents /* = true */) +bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntries &setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string &errString, bool fSearchForParents /* = true */) const { setEntries parentHashes; const CTransaction &tx = entry.GetTx(); @@ -257,6 +243,20 @@ void CTxMemPool::UpdateAncestorsOf(bool add, txiter it, setEntries &setAncestors } } +void CTxMemPool::UpdateEntryForAncestors(txiter it, const setEntries &setAncestors) +{ + int64_t updateCount = setAncestors.size(); + int64_t updateSize = 0; + CAmount updateFee = 0; + int updateSigOps = 0; + BOOST_FOREACH(txiter ancestorIt, setAncestors) { + updateSize += ancestorIt->GetTxSize(); + updateFee += ancestorIt->GetModifiedFee(); + updateSigOps += ancestorIt->GetSigOpCount(); + } + mapTx.modify(it, update_ancestor_state(updateSize, updateFee, updateCount, updateSigOps)); +} + void CTxMemPool::UpdateChildrenForRemoval(txiter it) { const setEntries &setMemPoolChildren = GetMemPoolChildren(it); @@ -265,11 +265,30 @@ void CTxMemPool::UpdateChildrenForRemoval(txiter it) } } -void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove) +void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove, bool updateDescendants) { // For each entry, walk back all ancestors and decrement size associated with this // transaction const uint64_t nNoLimit = std::numeric_limits::max(); + if (updateDescendants) { + // updateDescendants should be true whenever we're not recursively + // removing a tx and all its descendants, eg when a transaction is + // confirmed in a block. + // Here we only update statistics and not data in mapLinks (which + // we need to preserve until we're finished with all operations that + // need to traverse the mempool). + BOOST_FOREACH(txiter removeIt, entriesToRemove) { + setEntries setDescendants; + CalculateDescendants(removeIt, setDescendants); + setDescendants.erase(removeIt); // don't update state for self + int64_t modifySize = -((int64_t)removeIt->GetTxSize()); + CAmount modifyFee = -removeIt->GetModifiedFee(); + int modifySigOps = -removeIt->GetSigOpCount(); + BOOST_FOREACH(txiter dit, setDescendants) { + mapTx.modify(dit, update_ancestor_state(modifySize, modifyFee, -1, modifySigOps)); + } + } + } BOOST_FOREACH(txiter removeIt, entriesToRemove) { setEntries setAncestors; const CTxMemPoolEntry &entry = *removeIt; @@ -293,10 +312,7 @@ void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove) // transactions as the set of things to update for removal. CalculateMemPoolAncestors(entry, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy, false); // Note that UpdateAncestorsOf severs the child links that point to - // removeIt in the entries for the parents of removeIt. This is - // fine since we don't need to use the mempool children of any entries - // to walk back over our ancestors (but we do need the mempool - // parents!) + // removeIt in the entries for the parents of removeIt. UpdateAncestorsOf(false, removeIt, setAncestors); } // After updating all the ancestor sizes, we can now sever the link between each @@ -307,22 +323,24 @@ void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove) } } -void CTxMemPoolEntry::SetDirty() +void CTxMemPoolEntry::UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount) { - nCountWithDescendants = 0; - nSizeWithDescendants = nTxSize; - nModFeesWithDescendants = GetModifiedFee(); + nSizeWithDescendants += modifySize; + assert(int64_t(nSizeWithDescendants) > 0); + nModFeesWithDescendants += modifyFee; + nCountWithDescendants += modifyCount; + assert(int64_t(nCountWithDescendants) > 0); } -void CTxMemPoolEntry::UpdateState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount) +void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int modifySigOps) { - if (!IsDirty()) { - nSizeWithDescendants += modifySize; - assert(int64_t(nSizeWithDescendants) > 0); - nModFeesWithDescendants += modifyFee; - nCountWithDescendants += modifyCount; - assert(int64_t(nCountWithDescendants) > 0); - } + nSizeWithAncestors += modifySize; + assert(int64_t(nSizeWithAncestors) > 0); + nModFeesWithAncestors += modifyFee; + nCountWithAncestors += modifyCount; + assert(int64_t(nCountWithAncestors) > 0); + nSigOpCountWithAncestors += modifySigOps; + assert(int(nSigOpCountWithAncestors) >= 0); } CTxMemPool::CTxMemPool(const CFeeRate& _minReasonableRelayFee) : @@ -408,6 +426,7 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, } } UpdateAncestorsOf(true, newit, setAncestors); + UpdateEntryForAncestors(newit, setAncestors); nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); @@ -603,7 +622,7 @@ void CTxMemPool::CalculateDescendants(txiter entryit, setEntries &setDescendants } } -void CTxMemPool::remove(const CTransaction &origTx, std::list& removed, bool fRecursive) +void CTxMemPool::removeRecursive(const CTransaction &origTx, std::list& removed) { // Remove transaction from memory pool { @@ -612,8 +631,8 @@ void CTxMemPool::remove(const CTransaction &origTx, std::list& rem txiter origit = mapTx.find(origTx.GetHash()); if (origit != mapTx.end()) { txToRemove.insert(origit); - } else if (fRecursive) { - // If recursively removing but origTx isn't in the mempool + } else { + // When recursively removing but origTx isn't in the mempool // be sure to remove any children that are in the pool. This can // happen during chain re-orgs if origTx isn't re-accepted into // the mempool for any reason. @@ -627,17 +646,13 @@ void CTxMemPool::remove(const CTransaction &origTx, std::list& rem } } setEntries setAllRemoves; - if (fRecursive) { - BOOST_FOREACH(txiter it, txToRemove) { - CalculateDescendants(it, setAllRemoves); - } - } else { - setAllRemoves.swap(txToRemove); + BOOST_FOREACH(txiter it, txToRemove) { + CalculateDescendants(it, setAllRemoves); } BOOST_FOREACH(txiter it, setAllRemoves) { removed.push_back(it->GetTx()); } - RemoveStaged(setAllRemoves); + RemoveStaged(setAllRemoves, false); } } @@ -673,7 +688,7 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem } BOOST_FOREACH(const CTransaction& tx, transactionsToRemove) { list removed; - remove(tx, removed, true); + removeRecursive(tx, removed); } } @@ -688,7 +703,7 @@ void CTxMemPool::removeConflicts(const CTransaction &tx, std::list const CTransaction &txConflict = *it->second.ptx; if (txConflict != tx) { - remove(txConflict, removed, true); + removeRecursive(txConflict, removed); ClearPrioritisation(txConflict.GetHash()); } } @@ -713,8 +728,12 @@ void CTxMemPool::removeForBlock(const std::vector& vtx, unsigned i } BOOST_FOREACH(const CTransaction& tx, vtx) { - std::list dummy; - remove(tx, dummy, false); + txiter it = mapTx.find(tx.GetHash()); + if (it != mapTx.end()) { + setEntries stage; + stage.insert(it); + RemoveStaged(stage, true); + } removeConflicts(tx, conflicts); ClearPrioritisation(tx.GetHash()); } @@ -771,6 +790,8 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const innerUsage += memusage::DynamicUsage(links.parents) + memusage::DynamicUsage(links.children); bool fDependsWait = false; setEntries setParentCheck; + int64_t parentSizes = 0; + unsigned int parentSigOpCount = 0; BOOST_FOREACH(const CTxIn &txin, tx.vin) { // Check that every mempool transaction's inputs refer to available coins, or other mempool tx's. indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); @@ -778,7 +799,10 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const const CTransaction& tx2 = it2->GetTx(); assert(tx2.vout.size() > txin.prevout.n && !tx2.vout[txin.prevout.n].IsNull()); fDependsWait = true; - setParentCheck.insert(it2); + if (setParentCheck.insert(it2).second) { + parentSizes += it2->GetTxSize(); + parentSigOpCount += it2->GetSigOpCount(); + } } else { assert(pcoins->HaveCoin(txin.prevout)); } @@ -790,28 +814,42 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const i++; } assert(setParentCheck == GetMemPoolParents(it)); + // Verify ancestor state is correct. + setEntries setAncestors; + uint64_t nNoLimit = std::numeric_limits::max(); + std::string dummy; + CalculateMemPoolAncestors(*it, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy); + uint64_t nCountCheck = setAncestors.size() + 1; + uint64_t nSizeCheck = it->GetTxSize(); + CAmount nFeesCheck = it->GetModifiedFee(); + unsigned int nSigOpCheck = it->GetSigOpCount(); + + BOOST_FOREACH(txiter ancestorIt, setAncestors) { + nSizeCheck += ancestorIt->GetTxSize(); + nFeesCheck += ancestorIt->GetModifiedFee(); + nSigOpCheck += ancestorIt->GetSigOpCount(); + } + + assert(it->GetCountWithAncestors() == nCountCheck); + assert(it->GetSizeWithAncestors() == nSizeCheck); + assert(it->GetSigOpCountWithAncestors() == nSigOpCheck); + assert(it->GetModFeesWithAncestors() == nFeesCheck); + // Check children against mapNextTx CTxMemPool::setEntries setChildrenCheck; std::map::const_iterator iter = mapNextTx.lower_bound(COutPoint(it->GetTx().GetHash(), 0)); int64_t childSizes = 0; - CAmount childModFee = 0; for (; iter != mapNextTx.end() && iter->first.hash == it->GetTx().GetHash(); ++iter) { txiter childit = mapTx.find(iter->second.ptx->GetHash()); assert(childit != mapTx.end()); // mapNextTx points to in-mempool transactions if (setChildrenCheck.insert(childit).second) { childSizes += childit->GetTxSize(); - childModFee += childit->GetModifiedFee(); } } assert(setChildrenCheck == GetMemPoolChildren(it)); // Also check to make sure size is greater than sum with immediate children. // just a sanity check, not definitive that this calc is correct... - if (!it->IsDirty()) { - assert(it->GetSizeWithDescendants() >= childSizes + it->GetTxSize()); - } else { - assert(it->GetSizeWithDescendants() == it->GetTxSize()); - assert(it->GetModFeesWithDescendants() == it->GetModifiedFee()); - } + assert(it->GetSizeWithDescendants() >= childSizes + it->GetTxSize()); if (fDependsWait) waitingOnDependants.push_back(&(*it)); @@ -993,13 +1031,13 @@ bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { size_t CTxMemPool::DynamicMemoryUsage() const { LOCK(cs); - // Estimate the overhead of mapTx to be 12 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. - return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 12 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + cachedInnerUsage; + // Estimate the overhead of mapTx to be 15 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. + return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + cachedInnerUsage; } -void CTxMemPool::RemoveStaged(setEntries &stage) { +void CTxMemPool::RemoveStaged(setEntries &stage, bool updateDescendants) { AssertLockHeld(cs); - UpdateForRemoveFromMempool(stage); + UpdateForRemoveFromMempool(stage, updateDescendants); BOOST_FOREACH(const txiter& it, stage) { removeUnchecked(it); } @@ -1017,7 +1055,7 @@ int CTxMemPool::Expire(int64_t time) { BOOST_FOREACH(txiter removeit, toremove) { CalculateDescendants(removeit, stage); } - RemoveStaged(stage); + RemoveStaged(stage, false); return stage.size(); } @@ -1134,7 +1172,7 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector* pvNoSpends BOOST_FOREACH(txiter it, stage) txn.push_back(it->GetTx()); } - RemoveStaged(stage); + RemoveStaged(stage, false); if (pvNoSpendsRemaining) { BOOST_FOREACH(const CTransaction& tx, txn) { BOOST_FOREACH(const CTxIn& txin, tx.vin) { diff --git a/src/txmempool.h b/src/txmempool.h index 92a240e12..cae37633c 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -100,6 +100,12 @@ private: uint64_t nSizeWithDescendants; //! ... and size CAmount nModFeesWithDescendants; //! ... and total fees (all including us) + // Analogous statistics for ancestor transactions + uint64_t nCountWithAncestors; + uint64_t nSizeWithAncestors; + CAmount nModFeesWithAncestors; + unsigned int nSigOpCountWithAncestors; + public: CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, int64_t _nTime, double _entryPriority, unsigned int _entryHeight, @@ -124,25 +130,25 @@ public: const LockPoints& GetLockPoints() const { return lockPoints; } // Adjusts the descendant state, if this entry is not dirty. - void UpdateState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); + void UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); + // Adjusts the ancestor state + void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int modifySigOps); // Updates the fee delta used for mining priority score, and the // modified fees with descendants. void UpdateFeeDelta(int64_t feeDelta); // Update the LockPoints after a reorg void UpdateLockPoints(const LockPoints& lp); - /** We can set the entry to be dirty if doing the full calculation of in- - * mempool descendants will be too expensive, which can potentially happen - * when re-adding transactions from a block back to the mempool. - */ - void SetDirty(); - bool IsDirty() const { return nCountWithDescendants == 0; } - uint64_t GetCountWithDescendants() const { return nCountWithDescendants; } uint64_t GetSizeWithDescendants() const { return nSizeWithDescendants; } CAmount GetModFeesWithDescendants() const { return nModFeesWithDescendants; } bool GetSpendsCoinbase() const { return spendsCoinbase; } + + uint64_t GetCountWithAncestors() const { return nCountWithAncestors; } + uint64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } + CAmount GetModFeesWithAncestors() const { return nModFeesWithAncestors; } + unsigned int GetSigOpCountWithAncestors() const { return nSigOpCountWithAncestors; } }; // Helpers for modifying CTxMemPool::mapTx, which is a boost multi_index. @@ -153,7 +159,7 @@ struct update_descendant_state {} void operator() (CTxMemPoolEntry &e) - { e.UpdateState(modifySize, modifyFee, modifyCount); } + { e.UpdateDescendantState(modifySize, modifyFee, modifyCount); } private: int64_t modifySize; @@ -161,10 +167,20 @@ struct update_descendant_state int64_t modifyCount; }; -struct set_dirty +struct update_ancestor_state { + update_ancestor_state(int64_t _modifySize, CAmount _modifyFee, int64_t _modifyCount, int _modifySigOps) : + modifySize(_modifySize), modifyFee(_modifyFee), modifyCount(_modifyCount), modifySigOps(_modifySigOps) + {} + void operator() (CTxMemPoolEntry &e) - { e.SetDirty(); } + { e.UpdateAncestorState(modifySize, modifyFee, modifyCount, modifySigOps); } + + private: + int64_t modifySize; + CAmount modifyFee; + int64_t modifyCount; + int modifySigOps; }; struct update_fee_delta @@ -261,10 +277,34 @@ public: } }; +class CompareTxMemPoolEntryByAncestorFee +{ +public: + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) + { + double aFees = a.GetModFeesWithAncestors(); + double aSize = a.GetSizeWithAncestors(); + + double bFees = b.GetModFeesWithAncestors(); + double bSize = b.GetSizeWithAncestors(); + + // Avoid division by rewriting (a/b > c/d) as (a*d > c*b). + double f1 = aFees * bSize; + double f2 = aSize * bFees; + + if (f1 == f2) { + return a.GetTx().GetHash() < b.GetTx().GetHash(); + } + + return f1 > f2; + } +}; + // Multi_index tag names struct descendant_score {}; struct entry_time {}; struct mining_score {}; +struct ancestor_score {}; class CBlockPolicyEstimator; @@ -411,12 +451,18 @@ public: boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByEntryTime - >, + >, // sorted by score (for mining prioritization) boost::multi_index::ordered_unique< boost::multi_index::tag, boost::multi_index::identity, CompareTxMemPoolEntryByScore + >, + // sorted by fee rate with ancestors + boost::multi_index::ordered_non_unique< + boost::multi_index::tag, + boost::multi_index::identity, + CompareTxMemPoolEntryByAncestorFee > > > indexed_transaction_set; @@ -496,7 +542,7 @@ public: bool getSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value); bool removeSpentIndex(const uint256 txhash); - void remove(const CTransaction &tx, std::list& removed, bool fRecursive = false); + void removeRecursive(const CTransaction &tx, std::list& removed); void removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags); void removeConflicts(const CTransaction &tx, std::list& removed); void removeForBlock(const std::vector& vtx, unsigned int nBlockHeight, @@ -521,8 +567,12 @@ public: public: /** Remove a set of transactions from the mempool. * If a transaction is in this set, then all in-mempool descendants must - * also be in the set.*/ - void RemoveStaged(setEntries &stage); + * also be in the set, unless this transaction is being removed for being + * in a block. + * Set updateDescendants to true when removing a tx that was in a block, so + * that any in-mempool descendants have their ancestor state updated. + */ + void RemoveStaged(setEntries &stage, bool updateDescendants); /** When adding transactions from a disconnected block back to the mempool, * new mempool entries may have children in the mempool (which is generally @@ -545,7 +595,7 @@ public: * fSearchForParents = whether to search a tx's vin for in-mempool parents, or * look up parents from mapLinks. Must be true for entries not in the mempool */ - bool CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntries &setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string &errString, bool fSearchForParents = true); + bool CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntries &setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string &errString, bool fSearchForParents = true) const; /** Populate setDescendants with all in-mempool descendants of hash. * Assumes that setDescendants includes all in-mempool descendants of anything @@ -631,21 +681,21 @@ private: * updated and hence their state is already reflected in the parent * state). * - * If updating an entry requires looking at more than maxDescendantsToVisit - * transactions, outside of the ones in setExclude, then give up. - * * cachedDescendants will be updated with the descendants of the transaction * being updated, so that future invocations don't need to walk the * same transaction again, if encountered in another transaction chain. */ - bool UpdateForDescendants(txiter updateIt, - int maxDescendantsToVisit, + void UpdateForDescendants(txiter updateIt, cacheMap &cachedDescendants, const std::set &setExclude); /** Update ancestors of hash to add/remove it as a descendant transaction. */ void UpdateAncestorsOf(bool add, txiter hash, setEntries &setAncestors); - /** For each transaction being removed, update ancestors and any direct children. */ - void UpdateForRemoveFromMempool(const setEntries &entriesToRemove); + /** Set ancestor state for an entry */ + void UpdateEntryForAncestors(txiter it, const setEntries &setAncestors); + /** For each transaction being removed, update ancestors and any direct children. + * If updateDescendants is true, then also update in-mempool descendants' + * ancestor state. */ + void UpdateForRemoveFromMempool(const setEntries &entriesToRemove, bool updateDescendants); /** Sever link between specified transaction and direct children. */ void UpdateChildrenForRemoval(txiter entry); diff --git a/src/validation.cpp b/src/validation.cpp index 9965916ce..c77f21be7 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -848,20 +848,6 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C // Save these to avoid repeated lookups setIterConflicting.insert(mi); - // If this entry is "dirty", then we don't have descendant - // state for this transaction, which means we probably have - // lots of in-mempool descendants. - // Don't allow replacements of dirty transactions, to ensure - // that we don't spend too much time walking descendants. - // This should be rare. - if (mi->IsDirty()) { - return state.DoS(0, false, - REJECT_NONSTANDARD, "too many potential replacements", false, - strprintf("too many potential replacements: rejecting replacement %s; cannot replace tx %s with untracked descendants", - hash.ToString(), - mi->GetTx().GetHash().ToString())); - } - // Don't allow the replacement to reduce the feerate of the // mempool. // @@ -994,7 +980,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C FormatMoney(nModifiedFees - nConflictingFees), (int)nSize - (int)nConflictingSize); } - pool.RemoveStaged(allConflicting); + pool.RemoveStaged(allConflicting, false); // Store transaction in memory pool.addUnchecked(hash, entry, setAncestors, !IsInitialBlockDownload()); @@ -2512,7 +2498,7 @@ bool static DisconnectTip(CValidationState& state, const Consensus::Params& cons list removed; CValidationState stateDummy; if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL, true)) { - mempool.remove(tx, removed, true); + mempool.removeRecursive(tx, removed); } else if (mempool.exists(tx.GetHash())) { vHashUpdate.push_back(tx.GetHash()); }