From 0a0e60c1fbd4d023d1fb92c895b29fed860d51b3 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 2 Nov 2016 11:12:48 +0100 Subject: [PATCH] Merge #8448: Store mempool and prioritization data to disk 582068a Add mempool.dat to doc/files.md (Pieter Wuille) 3f78562 Add DumpMempool and LoadMempool (Pieter Wuille) ced7c94 Add AcceptToMemoryPoolWithTime function (Pieter Wuille) c3efb58 Add feedelta to TxMempoolInfo (Pieter Wuille) --- doc/files.md | 1 + src/init.cpp | 3 ++ src/txmempool.cpp | 8 ++- src/txmempool.h | 3 ++ src/validation.cpp | 129 +++++++++++++++++++++++++++++++++++++++++++-- src/validation.h | 10 ++++ 6 files changed, 147 insertions(+), 7 deletions(-) diff --git a/doc/files.md b/doc/files.md index 62940b639..ce1211496 100644 --- a/doc/files.md +++ b/doc/files.md @@ -10,6 +10,7 @@ * db.log: wallet database log file * debug.log: contains debug information and general logging generated by dashd or dash-qt * fee_estimates.dat: stores statistics used to estimate minimum transaction fees and priorities required for confirmation; since 0.10.0 +* mempool.dat: dump of the mempool's transactions; since 0.14.0. * governance.dat: stores data for governance obgects * masternode.conf: contains configuration settings for remote masternodes * mncache.dat: stores data for masternode list diff --git a/src/init.cpp b/src/init.cpp index d2a9ce073..1cc651445 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -250,6 +250,7 @@ void PrepareShutdown() flatdb4.Dump(netfulfilledman); UnregisterNodeSignals(GetNodeSignals()); + DumpMempool(); if (fFeeEstimatesInitialized) { @@ -760,6 +761,8 @@ void ThreadImport(std::vector vImportFiles) LogPrintf("Stopping after block import\n"); StartShutdown(); } + + LoadMempool(); } /** Sanity checks diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 729f892ce..d9ff96e1e 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -967,6 +967,10 @@ void CTxMemPool::queryHashes(vector& vtxid) } } +static TxMempoolInfo GetInfo(CTxMemPool::indexed_transaction_set::const_iterator it) { + return TxMempoolInfo{it->GetSharedTx(), it->GetTime(), CFeeRate(it->GetFee(), it->GetTxSize()), it->GetModifiedFee() - it->GetFee()}; +} + std::vector CTxMemPool::infoAll() const { LOCK(cs); @@ -975,7 +979,7 @@ std::vector CTxMemPool::infoAll() const std::vector ret; ret.reserve(mapTx.size()); for (auto it : iters) { - ret.push_back(TxMempoolInfo{it->GetSharedTx(), it->GetTime(), CFeeRate(it->GetFee(), it->GetTxSize())}); + ret.push_back(GetInfo(it)); } return ret; @@ -996,7 +1000,7 @@ TxMempoolInfo CTxMemPool::info(const uint256& hash) const indexed_transaction_set::const_iterator i = mapTx.find(hash); if (i == mapTx.end()) return TxMempoolInfo(); - return TxMempoolInfo{i->GetSharedTx(), i->GetTime(), CFeeRate(i->GetFee(), i->GetTxSize())}; + return GetInfo(i); } CFeeRate CTxMemPool::estimateFee(int nBlocks) const diff --git a/src/txmempool.h b/src/txmempool.h index 3b5b1a492..f672adff3 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -329,6 +329,9 @@ struct TxMempoolInfo /** Feerate of the transaction. */ CFeeRate feeRate; + + /** The fee delta. */ + int64_t nFeeDelta; }; class SaltedTxidHasher diff --git a/src/validation.cpp b/src/validation.cpp index dc39372ba..b688179e7 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -566,7 +566,7 @@ std::string FormatStateMessage(const CValidationState &state) } bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransaction& tx, bool fLimitFree, - bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, + bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector& coins_to_uncache, bool fDryRun) { const uint256 hash = tx.GetHash(); @@ -760,7 +760,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C } } - CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), pool.HasNoInputsOf(tx), inChainInputValue, fSpendsCoinbase, nSigOps, lp); + CTxMemPoolEntry entry(tx, nFees, nAcceptTime, dPriority, chainActive.Height(), pool.HasNoInputsOf(tx), inChainInputValue, fSpendsCoinbase, nSigOps, lp); unsigned int nSize = entry.GetTxSize(); // Check that the transaction doesn't have an excessive number of @@ -1023,11 +1023,11 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C return true; } -bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, - bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount nAbsurdFee, bool fDryRun) +bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, + bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit, const CAmount nAbsurdFee, bool fDryRun) { std::vector coins_to_uncache; - bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, fOverrideMempoolLimit, nAbsurdFee, coins_to_uncache, fDryRun); + bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, fOverrideMempoolLimit, nAbsurdFee, coins_to_uncache, fDryRun); if (!res || fDryRun) { if(!res) LogPrint("mempool", "%s: %s %s\n", __func__, tx.GetHash().ToString(), state.GetRejectReason()); BOOST_FOREACH(const COutPoint& hashTx, coins_to_uncache) @@ -1039,6 +1039,12 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa return res; } +bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, + bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount nAbsurdFee, bool fDryRun) +{ + return AcceptToMemoryPoolWithTime(pool, state, tx, fLimitFree, pfMissingInputs, GetTime(), fOverrideMempoolLimit, nAbsurdFee, fDryRun); +} + bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes) { if (!fTimestampIndex) @@ -4496,6 +4502,119 @@ int VersionBitsTipStateSinceHeight(const Consensus::Params& params, Consensus::D return VersionBitsStateSinceHeight(chainActive.Tip(), params, pos, versionbitscache); } +static const uint64_t MEMPOOL_DUMP_VERSION = 1; + +bool LoadMempool(void) +{ + int64_t nExpiryTimeout = GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; + FILE* filestr = fopen((GetDataDir() / "mempool.dat").string().c_str(), "r"); + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); + return false; + } + + int64_t count = 0; + int64_t skipped = 0; + int64_t failed = 0; + int64_t nNow = GetTime(); + + try { + uint64_t version; + file >> version; + if (version != MEMPOOL_DUMP_VERSION) { + return false; + } + uint64_t num; + file >> num; + double prioritydummy = 0; + while (num--) { + CTransaction tx; + int64_t nTime; + int64_t nFeeDelta; + file >> tx; + file >> nTime; + file >> nFeeDelta; + + CAmount amountdelta = nFeeDelta; + if (amountdelta) { + mempool.PrioritiseTransaction(tx.GetHash(), tx.GetHash().ToString(), prioritydummy, amountdelta); + } + CValidationState state; + if (nTime + nExpiryTimeout > nNow) { + LOCK(cs_main); + AcceptToMemoryPoolWithTime(mempool, state, tx, true, NULL, nTime); + if (state.IsValid()) { + ++count; + } else { + ++failed; + } + } else { + ++skipped; + } + } + std::map mapDeltas; + file >> mapDeltas; + + for (const auto& i : mapDeltas) { + mempool.PrioritiseTransaction(i.first, i.first.ToString(), prioritydummy, i.second); + } + } catch (const std::exception& e) { + LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); + return false; + } + + LogPrintf("Imported mempool transactions from disk: %i successes, %i failed, %i expired\n", count, failed, skipped); + return true; +} + +void DumpMempool(void) +{ + int64_t start = GetTimeMicros(); + + std::map mapDeltas; + std::vector vinfo; + + { + LOCK(mempool.cs); + for (const auto &i : mempool.mapDeltas) { + mapDeltas[i.first] = i.second.first; + } + vinfo = mempool.infoAll(); + } + + int64_t mid = GetTimeMicros(); + + try { + FILE* filestr = fopen((GetDataDir() / "mempool.dat.new").string().c_str(), "w"); + if (!filestr) { + return; + } + + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + + uint64_t version = MEMPOOL_DUMP_VERSION; + file << version; + + file << (uint64_t)vinfo.size(); + for (const auto& i : vinfo) { + file << *(i.tx); + file << (int64_t)i.nTime; + file << (int64_t)i.nFeeDelta; + mapDeltas.erase(i.tx->GetHash()); + } + + file << mapDeltas; + FileCommit(file.Get()); + file.fclose(); + RenameOver(GetDataDir() / "mempool.dat.new", GetDataDir() / "mempool.dat"); + int64_t last = GetTimeMicros(); + LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", (mid-start)*0.000001, (last-mid)*0.000001); + } catch (const std::exception& e) { + LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); + } +} + class CMainCleanup { public: diff --git a/src/validation.h b/src/validation.h index 1877bb815..38bfaca7f 100644 --- a/src/validation.h +++ b/src/validation.h @@ -322,6 +322,10 @@ void PruneAndFlush(); bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, bool* pfMissingInputs, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0, bool fDryRun=false); +/** (try to) add transaction to memory pool with a specified acceptance time **/ +bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, + bool* pfMissingInputs, int64_t nAcceptTime, bool fOverrideMempoolLimit=false, const CAmount nAbsurdFee=0, bool fDryRun=false); + bool GetUTXOCoin(const COutPoint& outpoint, Coin& coin); int GetUTXOHeight(const COutPoint& outpoint); int GetUTXOConfirmations(const COutPoint& outpoint); @@ -548,4 +552,10 @@ static const unsigned int REJECT_ALREADY_KNOWN = 0x101; /** Transaction conflicts with a transaction already known */ static const unsigned int REJECT_CONFLICT = 0x102; +/** Dump the mempool to disk. */ +void DumpMempool(); + +/** Load the mempool from disk. */ +bool LoadMempool(); + #endif // BITCOIN_VALIDATION_H