diff --git a/src/bls/bls.h b/src/bls/bls.h index ccebca3db7..63a6d23b8b 100644 --- a/src/bls/bls.h +++ b/src/bls/bls.h @@ -170,6 +170,11 @@ public: } public: + inline void Serialize(CSizeComputer& s) const + { + s.seek(SerSize); + } + template inline void Serialize(Stream& s) const { @@ -356,6 +361,11 @@ public: return *this; } + inline void Serialize(CSizeComputer& s) const + { + s.seek(BLSObject::SerSize); + } + template inline void Serialize(Stream& s) const { diff --git a/src/dbwrapper.h b/src/dbwrapper.h index d32c394d18..fc637bea50 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -551,6 +551,7 @@ class CDBTransaction { protected: Parent &parent; CommitTarget &commitTarget; + ssize_t memoryUsage{0}; // signed, just in case we made an error in the calculations so that we don't get an overflow struct DataStreamCmp { static bool less(const CDataStream& a, const CDataStream& b) { @@ -564,6 +565,8 @@ protected: }; struct ValueHolder { + size_t memoryUsage; + ValueHolder(size_t _memoryUsage) : memoryUsage(_memoryUsage) {} virtual ~ValueHolder() = default; virtual void Write(const CDataStream& ssKey, CommitTarget &parent) = 0; }; @@ -571,7 +574,7 @@ protected: template struct ValueHolderImpl : ValueHolder { - ValueHolderImpl(const V &_value) : value(_value) { } + ValueHolderImpl(const V &_value, size_t _memoryUsage) : ValueHolder(_memoryUsage), value(_value) {} virtual void Write(const CDataStream& ssKey, CommitTarget &commitTarget) { // we're moving the value instead of copying it. This means that Write() can only be called once per @@ -605,9 +608,18 @@ public: template void Write(const CDataStream& ssKey, const V& v) { - deletes.erase(ssKey); + auto valueMemoryUsage = ::GetSerializeSize(v, SER_DISK, CLIENT_VERSION); + + if (deletes.erase(ssKey)) { + memoryUsage -= ssKey.size(); + } auto it = writes.emplace(ssKey, nullptr).first; - it->second = std::make_unique>(v); + if (it->second) { + memoryUsage -= ssKey.size() + it->second->memoryUsage; + } + it->second = std::make_unique>(v, valueMemoryUsage); + + memoryUsage += ssKey.size() + valueMemoryUsage; } template @@ -657,13 +669,20 @@ public: } void Erase(const CDataStream& ssKey) { - writes.erase(ssKey); - deletes.emplace(ssKey); + auto it = writes.find(ssKey); + if (it != writes.end()) { + memoryUsage -= ssKey.size() + it->second->memoryUsage; + writes.erase(it); + } + if (deletes.emplace(ssKey).second) { + memoryUsage += ssKey.size(); + } } void Clear() { writes.clear(); deletes.clear(); + memoryUsage = 0; } void Commit() { @@ -680,6 +699,19 @@ public: return writes.empty() && deletes.empty(); } + size_t GetMemoryUsage() const { + if (memoryUsage < 0) { + // something went wrong when we accounted/calculated used memory... + static volatile bool didPrint = false; + if (!didPrint) { + LogPrintf("CDBTransaction::%s -- negative memoryUsage (%d)", __func__, memoryUsage); + didPrint = true; + } + return 0; + } + return (size_t)memoryUsage; + } + CDBTransactionIterator* NewIterator() { return new CDBTransactionIterator(*this); } diff --git a/src/evo/deterministicmns.h b/src/evo/deterministicmns.h index 261ac9bdd0..62cf3bed25 100644 --- a/src/evo/deterministicmns.h +++ b/src/evo/deterministicmns.h @@ -195,6 +195,21 @@ void UnserializeImmerMap(Stream& is, immer::map& m) } } +// For some reason the compiler is not able to choose the correct Serialize/Deserialize methods without a specialized +// version of SerReadWrite. It otherwise always chooses the version that calls a.Serialize() +template +inline void SerReadWrite(Stream& s, const immer::map& m, CSerActionSerialize ser_action) +{ + ::SerializeImmerMap(s, m); +} + +template +inline void SerReadWrite(Stream& s, immer::map& obj, CSerActionUnserialize ser_action) +{ + ::UnserializeImmerMap(s, obj); +} + + class CDeterministicMNList { public: @@ -226,13 +241,8 @@ public: { READWRITE(blockHash); READWRITE(nHeight); - if (ser_action.ForRead()) { - UnserializeImmerMap(s, mnMap); - UnserializeImmerMap(s, mnUniquePropertyMap); - } else { - SerializeImmerMap(s, mnMap); - SerializeImmerMap(s, mnUniquePropertyMap); - } + READWRITE(mnMap); + READWRITE(mnUniquePropertyMap); } public: diff --git a/src/evo/evodb.h b/src/evo/evodb.h index ead20ce6fe..03e9b3acfd 100644 --- a/src/evo/evodb.h +++ b/src/evo/evodb.h @@ -73,6 +73,11 @@ public: return db; } + size_t GetMemoryUsage() + { + return rootDBTransaction.GetMemoryUsage(); + } + bool CommitRootTransaction(); bool VerifyBestBlock(const uint256& hash); diff --git a/src/validation.cpp b/src/validation.cpp index f4095e2ec4..6422ac2e07 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2385,6 +2385,7 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n } int64_t nMempoolSizeMax = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t cacheSize = pcoinsTip->DynamicMemoryUsage() * DB_PEAK_USAGE_FACTOR; + cacheSize += evoDb->GetMemoryUsage() * DB_PEAK_USAGE_FACTOR; int64_t nTotalSpace = nCoinCacheUsage + std::max(nMempoolSizeMax - nMempoolUsage, 0); // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing). bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024); @@ -2522,6 +2523,7 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), GuessVerificationProgress(chainParams.TxData(), chainActive.Tip()), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); + strMessage += strprintf(" evodb_cache=%.1fMiB", evoDb->GetMemoryUsage() * (1.0 / (1<<20))); if (!warningMessages.empty()) strMessage += strprintf(" warning='%s'", boost::algorithm::join(warningMessages, ", ")); LogPrintf("%s\n", strMessage);