From 013169d63d0057478f5191d5737a585fb19f1396 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Tue, 9 Jul 2019 07:59:57 +0200 Subject: [PATCH] Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure --- src/activemasternode.cpp | 2 +- src/dbwrapper.h | 5 + src/evo/deterministicmns.cpp | 265 +++++++++++++++++++----- src/evo/deterministicmns.h | 270 +++++++++++++++++++------ src/evo/evodb.h | 4 +- src/evo/mnauth.cpp | 10 +- src/evo/providertx.cpp | 8 +- src/evo/providertx.h | 6 +- src/evo/simplifiedmns.cpp | 9 +- src/governance.cpp | 12 +- src/governance.h | 8 +- src/init.cpp | 2 + src/llmq/quorums.cpp | 14 +- src/llmq/quorums.h | 4 +- src/llmq/quorums_blockprocessor.cpp | 9 +- src/llmq/quorums_commitment.cpp | 2 +- src/llmq/quorums_debug.cpp | 13 +- src/llmq/quorums_dkgsession.cpp | 35 ++-- src/llmq/quorums_dkgsession.h | 5 +- src/llmq/quorums_dkgsessionhandler.cpp | 22 +- src/llmq/quorums_dkgsessionhandler.h | 2 +- src/llmq/quorums_dkgsessionmgr.cpp | 22 +- src/llmq/quorums_dkgsessionmgr.h | 8 +- src/llmq/quorums_utils.cpp | 14 +- src/llmq/quorums_utils.h | 6 +- src/masternode-meta.cpp | 1 - src/masternode-meta.h | 8 +- src/masternode-payments.cpp | 8 +- src/rpc/rpcevo.cpp | 4 +- src/rpc/rpcquorums.cpp | 4 +- src/script/script.h | 9 + src/spentindex.h | 2 +- 32 files changed, 572 insertions(+), 221 deletions(-) diff --git a/src/activemasternode.cpp b/src/activemasternode.cpp index 3484c16eb..fe51fba07 100644 --- a/src/activemasternode.cpp +++ b/src/activemasternode.cpp @@ -134,7 +134,7 @@ void CActiveMasternodeManager::UpdatedBlockTip(const CBlockIndex* pindexNew, con if (!deterministicMNManager->IsDIP3Enforced(pindexNew->nHeight)) return; if (state == MASTERNODE_READY) { - auto mnList = deterministicMNManager->GetListForBlock(pindexNew->GetBlockHash()); + auto mnList = deterministicMNManager->GetListForBlock(pindexNew); if (!mnList.IsMNValid(mnListEntry->proTxHash)) { // MN disappeared from MN list state = MASTERNODE_REMOVED; diff --git a/src/dbwrapper.h b/src/dbwrapper.h index fc637bea5..e434eeeaf 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -391,6 +391,11 @@ public: pdb->CompactRange(&slKey1, &slKey2); } + void CompactFull() const + { + pdb->CompactRange(nullptr, nullptr); + } + }; template diff --git a/src/evo/deterministicmns.cpp b/src/evo/deterministicmns.cpp index 96bb8086a..673a2156f 100644 --- a/src/evo/deterministicmns.cpp +++ b/src/evo/deterministicmns.cpp @@ -182,6 +182,15 @@ CDeterministicMNCPtr CDeterministicMNList::GetValidMNByService(const CService& s return dmn; } +CDeterministicMNCPtr CDeterministicMNList::GetMNByInternalId(uint64_t internalId) const +{ + auto proTxHash = mnInternalIdMap.find(internalId); + if (!proTxHash) { + return nullptr; + } + return GetMN(*proTxHash); +} + static int CompareByLastPaid_GetHeight(const CDeterministicMN& dmn) { int height = dmn.pdmnState->nLastPaidHeight; @@ -349,25 +358,31 @@ void CDeterministicMNList::PoSeDecrease(const uint256& proTxHash) CDeterministicMNListDiff CDeterministicMNList::BuildDiff(const CDeterministicMNList& to) const { CDeterministicMNListDiff diffRet; - diffRet.prevBlockHash = blockHash; - diffRet.blockHash = to.blockHash; - diffRet.nHeight = to.nHeight; to.ForEachMN(false, [&](const CDeterministicMNCPtr& toPtr) { auto fromPtr = GetMN(toPtr->proTxHash); if (fromPtr == nullptr) { - diffRet.addedMNs.emplace(toPtr->proTxHash, toPtr); - } else if (*toPtr->pdmnState != *fromPtr->pdmnState) { - diffRet.updatedMNs.emplace(toPtr->proTxHash, toPtr->pdmnState); + diffRet.addedMNs.emplace_back(toPtr); + } else if (fromPtr != toPtr || fromPtr->pdmnState != toPtr->pdmnState) { + CDeterministicMNStateDiff stateDiff(*fromPtr->pdmnState, *toPtr->pdmnState); + if (stateDiff.fields) { + diffRet.updatedMNs.emplace(toPtr->internalId, std::move(stateDiff)); + } } }); ForEachMN(false, [&](const CDeterministicMNCPtr& fromPtr) { auto toPtr = to.GetMN(fromPtr->proTxHash); if (toPtr == nullptr) { - diffRet.removedMns.insert(fromPtr->proTxHash); + diffRet.removedMns.emplace(fromPtr->internalId); } }); + // added MNs need to be sorted by internalId so that these are added in correct order when the diff is applied later + // otherwise internalIds will not match with the original list + std::sort(diffRet.addedMNs.begin(), diffRet.addedMNs.end(), [](const CDeterministicMNCPtr& a, const CDeterministicMNCPtr& b) { + return a->internalId < b->internalId; + }); + return diffRet; } @@ -399,22 +414,25 @@ CSimplifiedMNListDiff CDeterministicMNList::BuildSimplifiedDiff(const CDetermini return diffRet; } -CDeterministicMNList CDeterministicMNList::ApplyDiff(const CDeterministicMNListDiff& diff) const +CDeterministicMNList CDeterministicMNList::ApplyDiff(const CBlockIndex* pindex, const CDeterministicMNListDiff& diff) const { - assert(diff.prevBlockHash == blockHash && diff.nHeight == nHeight + 1); - CDeterministicMNList result = *this; - result.blockHash = diff.blockHash; - result.nHeight = diff.nHeight; + result.blockHash = pindex->GetBlockHash(); + result.nHeight = pindex->nHeight; - for (const auto& hash : diff.removedMns) { - result.RemoveMN(hash); + for (const auto& id : diff.removedMns) { + auto dmn = result.GetMNByInternalId(id); + assert(dmn); + result.RemoveMN(dmn->proTxHash); } - for (const auto& p : diff.addedMNs) { - result.AddMN(p.second); + for (const auto& dmn : diff.addedMNs) { + assert(dmn->internalId == result.GetTotalRegisteredCount()); + result.AddMN(dmn); + result.SetTotalRegisteredCount(result.GetTotalRegisteredCount() + 1); } for (const auto& p : diff.updatedMNs) { - result.UpdateMN(p.first, p.second); + auto dmn = result.GetMNByInternalId(p.first); + result.UpdateMN(dmn, p.second); } return result; @@ -424,6 +442,7 @@ void CDeterministicMNList::AddMN(const CDeterministicMNCPtr& dmn) { assert(!mnMap.find(dmn->proTxHash)); mnMap = mnMap.set(dmn->proTxHash, dmn); + mnInternalIdMap = mnInternalIdMap.set(dmn->internalId, dmn->proTxHash); AddUniqueProperty(dmn, dmn->collateralOutpoint); if (dmn->pdmnState->addr != CService()) { AddUniqueProperty(dmn, dmn->pdmnState->addr); @@ -434,20 +453,35 @@ void CDeterministicMNList::AddMN(const CDeterministicMNCPtr& dmn) } } -void CDeterministicMNList::UpdateMN(const uint256& proTxHash, const CDeterministicMNStateCPtr& pdmnState) +void CDeterministicMNList::UpdateMN(const CDeterministicMNCPtr& oldDmn, const CDeterministicMNStateCPtr& pdmnState) { - auto oldDmn = mnMap.find(proTxHash); assert(oldDmn != nullptr); - auto dmn = std::make_shared(**oldDmn); + auto dmn = std::make_shared(*oldDmn); auto oldState = dmn->pdmnState; dmn->pdmnState = pdmnState; - mnMap = mnMap.set(proTxHash, dmn); + mnMap = mnMap.set(oldDmn->proTxHash, dmn); UpdateUniqueProperty(dmn, oldState->addr, pdmnState->addr); UpdateUniqueProperty(dmn, oldState->keyIDOwner, pdmnState->keyIDOwner); UpdateUniqueProperty(dmn, oldState->pubKeyOperator, pdmnState->pubKeyOperator); } +void CDeterministicMNList::UpdateMN(const uint256& proTxHash, const CDeterministicMNStateCPtr& pdmnState) +{ + auto oldDmn = mnMap.find(proTxHash); + assert(oldDmn != nullptr); + UpdateMN(*oldDmn, pdmnState); +} + +void CDeterministicMNList::UpdateMN(const CDeterministicMNCPtr& oldDmn, const CDeterministicMNStateDiff& stateDiff) +{ + assert(oldDmn != nullptr); + auto oldState = oldDmn->pdmnState; + auto newState = std::make_shared(*oldState); + stateDiff.ApplyToState(*newState); + UpdateMN(oldDmn, newState); +} + void CDeterministicMNList::RemoveMN(const uint256& proTxHash) { auto dmn = GetMN(proTxHash); @@ -461,6 +495,7 @@ void CDeterministicMNList::RemoveMN(const uint256& proTxHash) DeleteUniqueProperty(dmn, dmn->pdmnState->pubKeyOperator); } mnMap = mnMap.erase(proTxHash); + mnInternalIdMap = mnInternalIdMap.erase(dmn->internalId); } CDeterministicMNManager::CDeterministicMNManager(CEvoDB& _evoDb) : @@ -500,12 +535,12 @@ bool CDeterministicMNManager::ProcessBlock(const CBlock& block, const CBlockInde newList.SetBlockHash(block.GetHash()); - oldList = GetListForBlock(pindex->pprev->GetBlockHash()); + oldList = GetListForBlock(pindex->pprev); diff = oldList.BuildDiff(newList); - evoDb.Write(std::make_pair(DB_LIST_DIFF, diff.blockHash), diff); + evoDb.Write(std::make_pair(DB_LIST_DIFF, newList.GetBlockHash()), diff); if ((nHeight % SNAPSHOT_LIST_PERIOD) == 0 || oldList.GetHeight() == -1) { - evoDb.Write(std::make_pair(DB_LIST_SNAPSHOT, diff.blockHash), newList); + evoDb.Write(std::make_pair(DB_LIST_SNAPSHOT, newList.GetBlockHash()), newList); LogPrintf("CDeterministicMNManager::%s -- Wrote snapshot. nHeight=%d, mapCurMNs.allMNsCount=%d\n", __func__, nHeight, newList.GetAllMNsCount()); } @@ -546,8 +581,8 @@ bool CDeterministicMNManager::UndoBlock(const CBlock& block, const CBlockIndex* if (diff.HasChanges()) { // need to call this before erasing - curList = GetListForBlock(blockHash); - prevList = GetListForBlock(pindex->pprev->GetBlockHash()); + curList = GetListForBlock(pindex); + prevList = GetListForBlock(pindex->pprev); } evoDb.Erase(std::make_pair(DB_LIST_DIFF, blockHash)); @@ -574,8 +609,7 @@ void CDeterministicMNManager::UpdatedBlockTip(const CBlockIndex* pindex) { LOCK(cs); - tipHeight = pindex->nHeight; - tipBlockHash = pindex->GetBlockHash(); + tipIndex = pindex; } bool CDeterministicMNManager::BuildNewListFromBlock(const CBlock& block, const CBlockIndex* pindexPrev, CValidationState& _state, CDeterministicMNList& mnListRet, bool debugLogs) @@ -584,7 +618,7 @@ bool CDeterministicMNManager::BuildNewListFromBlock(const CBlock& block, const C int nHeight = pindexPrev->nHeight + 1; - CDeterministicMNList oldList = GetListForBlock(pindexPrev->GetBlockHash()); + CDeterministicMNList oldList = GetListForBlock(pindexPrev); CDeterministicMNList newList = oldList; newList.SetBlockHash(uint256()); // we can't know the final block hash, so better not return a (invalid) block hash newList.SetHeight(nHeight); @@ -628,6 +662,8 @@ bool CDeterministicMNManager::BuildNewListFromBlock(const CBlock& block, const C auto dmn = std::make_shared(); dmn->proTxHash = tx.GetHash(); + dmn->internalId = newList.GetTotalRegisteredCount(); + newList.SetTotalRegisteredCount(newList.GetTotalRegisteredCount() + 1); // collateralOutpoint is either pointing to an external collateral or to the ProRegTx itself if (proTx.collateralOutpoint.hash.IsNull()) { @@ -771,7 +807,15 @@ bool CDeterministicMNManager::BuildNewListFromBlock(const CBlock& block, const C assert(false); // this should have been handled already } if (!qc.commitment.IsNull()) { - HandleQuorumCommitment(qc.commitment, newList, debugLogs); + const auto& params = Params().GetConsensus().llmqs.at((Consensus::LLMQType)qc.commitment.llmqType); + int quorumHeight = qc.nHeight - (qc.nHeight % params.dkgInterval); + auto quorumIndex = pindexPrev->GetAncestor(quorumHeight); + if (!quorumIndex || quorumIndex->GetBlockHash() != qc.commitment.quorumHash) { + // we should actually never get into this case as validation should have catched it...but lets be sure + return _state.DoS(100, false, REJECT_INVALID, "bad-qc-quorum-hash"); + } + + HandleQuorumCommitment(qc.commitment, quorumIndex, newList, debugLogs); } } } @@ -807,11 +851,11 @@ bool CDeterministicMNManager::BuildNewListFromBlock(const CBlock& block, const C return true; } -void CDeterministicMNManager::HandleQuorumCommitment(llmq::CFinalCommitment& qc, CDeterministicMNList& mnList, bool debugLogs) +void CDeterministicMNManager::HandleQuorumCommitment(llmq::CFinalCommitment& qc, const CBlockIndex* pindexQuorum, CDeterministicMNList& mnList, bool debugLogs) { // The commitment has already been validated at this point so it's safe to use members of it - auto members = llmq::CLLMQUtils::GetAllQuorumMembers((Consensus::LLMQType)qc.llmqType, qc.quorumHash); + auto members = llmq::CLLMQUtils::GetAllQuorumMembers((Consensus::LLMQType)qc.llmqType, pindexQuorum); for (size_t i = 0; i < members.size(); i++) { if (!mnList.HasMN(members[i]->proTxHash)) { @@ -844,52 +888,48 @@ void CDeterministicMNManager::DecreasePoSePenalties(CDeterministicMNList& mnList } } -CDeterministicMNList CDeterministicMNManager::GetListForBlock(const uint256& blockHash) +CDeterministicMNList CDeterministicMNManager::GetListForBlock(const CBlockIndex* pindex) { LOCK(cs); - auto it = mnListsCache.find(blockHash); - if (it != mnListsCache.end()) { - return it->second; - } - - uint256 blockHashTmp = blockHash; CDeterministicMNList snapshot; - std::list listDiff; + std::list> listDiff; while (true) { // try using cache before reading from disk - it = mnListsCache.find(blockHashTmp); + auto it = mnListsCache.find(pindex->GetBlockHash()); if (it != mnListsCache.end()) { snapshot = it->second; break; } - if (evoDb.Read(std::make_pair(DB_LIST_SNAPSHOT, blockHashTmp), snapshot)) { - mnListsCache.emplace(blockHashTmp, snapshot); + if (evoDb.Read(std::make_pair(DB_LIST_SNAPSHOT, pindex->GetBlockHash()), snapshot)) { + mnListsCache.emplace(pindex->GetBlockHash(), snapshot); break; } CDeterministicMNListDiff diff; - if (!evoDb.Read(std::make_pair(DB_LIST_DIFF, blockHashTmp), diff)) { - snapshot = CDeterministicMNList(blockHashTmp, -1); - mnListsCache.emplace(blockHashTmp, snapshot); + if (!evoDb.Read(std::make_pair(DB_LIST_DIFF, pindex->GetBlockHash()), diff)) { + snapshot = CDeterministicMNList(pindex->GetBlockHash(), -1, 0); + mnListsCache.emplace(pindex->GetBlockHash(), snapshot); break; } - listDiff.emplace_front(diff); - blockHashTmp = diff.prevBlockHash; + listDiff.emplace_front(pindex, std::move(diff)); + pindex = pindex->pprev; } - for (const auto& diff : listDiff) { + for (const auto& p : listDiff) { + auto diffIndex = p.first; + auto& diff = p.second; if (diff.HasChanges()) { - snapshot = snapshot.ApplyDiff(diff); + snapshot = snapshot.ApplyDiff(diffIndex, diff); } else { - snapshot.SetBlockHash(diff.blockHash); - snapshot.SetHeight(diff.nHeight); + snapshot.SetBlockHash(diffIndex->GetBlockHash()); + snapshot.SetHeight(diffIndex->nHeight); } - mnListsCache.emplace(diff.blockHash, snapshot); + mnListsCache.emplace(diffIndex->GetBlockHash(), snapshot); } return snapshot; @@ -898,7 +938,10 @@ CDeterministicMNList CDeterministicMNManager::GetListForBlock(const uint256& blo CDeterministicMNList CDeterministicMNManager::GetListAtChainTip() { LOCK(cs); - return GetListForBlock(tipBlockHash); + if (!tipIndex) { + return {}; + } + return GetListForBlock(tipIndex); } bool CDeterministicMNManager::IsProTxWithCollateral(const CTransactionRef& tx, uint32_t n) @@ -928,7 +971,7 @@ bool CDeterministicMNManager::IsDIP3Enforced(int nHeight) LOCK(cs); if (nHeight == -1) { - nHeight = tipHeight; + nHeight = tipIndex->nHeight; } return nHeight >= Params().GetConsensus().DIP0003EnforcementHeight; @@ -948,3 +991,115 @@ void CDeterministicMNManager::CleanupCache(int nHeight) mnListsCache.erase(h); } } + +bool CDeterministicMNManager::UpgradeDiff(CDBBatch& batch, const CBlockIndex* pindexNext, const CDeterministicMNList& curMNList, CDeterministicMNList& newMNList) +{ + CDataStream oldDiffData(SER_DISK, CLIENT_VERSION); + if (!evoDb.GetRawDB().ReadDataStream(std::make_pair(DB_LIST_DIFF, pindexNext->GetBlockHash()), oldDiffData)) { + LogPrintf("CDeterministicMNManager::%s -- no diff found for %s\n", __func__, pindexNext->GetBlockHash().ToString()); + newMNList = curMNList; + newMNList.SetBlockHash(pindexNext->GetBlockHash()); + newMNList.SetHeight(pindexNext->nHeight); + return false; + } + + CDeterministicMNListDiff_OldFormat oldDiff; + oldDiffData >> oldDiff; + + CDeterministicMNListDiff newDiff; + size_t addedCount = 0; + for (auto& p : oldDiff.addedMNs) { + auto dmn = std::make_shared(*p.second); + dmn->internalId = curMNList.GetTotalRegisteredCount() + addedCount; + newDiff.addedMNs.emplace_back(dmn); + + addedCount++; + } + for (auto& p : oldDiff.removedMns) { + auto dmn = curMNList.GetMN(p); + newDiff.removedMns.emplace(dmn->internalId); + } + + // applies added/removed MNs + newMNList = curMNList.ApplyDiff(pindexNext, newDiff); + + // manually apply updated MNs and calc new state diffs + for (auto& p : oldDiff.updatedMNs) { + auto oldMN = newMNList.GetMN(p.first); + newMNList.UpdateMN(p.first, p.second); + auto newMN = newMNList.GetMN(p.first); + assert(oldMN && newMN); + + newDiff.updatedMNs.emplace(std::piecewise_construct, + std::forward_as_tuple(oldMN->internalId), + std::forward_as_tuple(*oldMN->pdmnState, *newMN->pdmnState)); + } + + batch.Write(std::make_pair(DB_LIST_DIFF, pindexNext->GetBlockHash()), newDiff); + + return true; +} + +// TODO this can be completely removed in a future version +void CDeterministicMNManager::UpgradeDBIfNeeded() +{ + LOCK(cs_main); + + if (chainActive.Tip() == nullptr) { + return; + } + + if (evoDb.GetRawDB().Exists(EVODB_BEST_BLOCK)) { + return; + } + + // Removing the old EVODB_BEST_BLOCK value early results in older version to crash immediately, even if the upgrade + // process is cancelled in-between. But if the new version sees that the old EVODB_BEST_BLOCK is already removed, + // then we must assume that the upgrade process was already running before but was interrupted. + if (chainActive.Height() > 1 && !evoDb.GetRawDB().Exists(std::string("b_b"))) { + LogPrintf("CDeterministicMNManager::%s -- ERROR, upgrade process was interrupted and can't be continued. You need to reindex now.\n", __func__); + } + evoDb.GetRawDB().Erase(std::string("b_b")); + + if (chainActive.Height() < Params().GetConsensus().DIP0003Height) { + // not reached DIP3 height yet, so no upgrade needed + auto dbTx = evoDb.BeginTransaction(); + evoDb.WriteBestBlock(chainActive.Tip()->GetBlockHash()); + dbTx->Commit(); + return; + } + + LogPrintf("CDeterministicMNManager::%s -- upgrading DB to use compact diffs\n", __func__); + + CDBBatch batch(evoDb.GetRawDB()); + + CDeterministicMNList curMNList; + curMNList.SetHeight(Params().GetConsensus().DIP0003Height - 1); + curMNList.SetBlockHash(chainActive[Params().GetConsensus().DIP0003Height - 1]->GetBlockHash()); + + for (int nHeight = Params().GetConsensus().DIP0003Height; nHeight <= chainActive.Height(); nHeight++) { + auto pindex = chainActive[nHeight]; + + CDeterministicMNList newMNList; + UpgradeDiff(batch, pindex, curMNList, newMNList); + + if ((nHeight % SNAPSHOT_LIST_PERIOD) == 0) { + batch.Write(std::make_pair(DB_LIST_SNAPSHOT, pindex->GetBlockHash()), newMNList); + evoDb.GetRawDB().WriteBatch(batch); + batch.Clear(); + } + + curMNList = newMNList; + } + + evoDb.GetRawDB().WriteBatch(batch); + + LogPrintf("CDeterministicMNManager::%s -- done upgrading\n", __func__); + + // Writing EVODB_BEST_BLOCK (which is b_b2 now) marks the DB as upgraded + auto dbTx = evoDb.BeginTransaction(); + evoDb.WriteBestBlock(chainActive.Tip()->GetBlockHash()); + dbTx->Commit(); + + evoDb.GetRawDB().CompactFull(); +} diff --git a/src/evo/deterministicmns.h b/src/evo/deterministicmns.h index 62cf3bed2..7ba514661 100644 --- a/src/evo/deterministicmns.h +++ b/src/evo/deterministicmns.h @@ -83,8 +83,8 @@ public: READWRITE(pubKeyOperator); READWRITE(keyIDVoting); READWRITE(addr); - READWRITE(*(CScriptBase*)(&scriptPayout)); - READWRITE(*(CScriptBase*)(&scriptOperatorPayout)); + READWRITE(scriptPayout); + READWRITE(scriptOperatorPayout); } void ResetOperatorFields() @@ -109,29 +109,6 @@ public: h.Finalize(confirmedHashWithProRegTxHash.begin()); } - bool operator==(const CDeterministicMNState& rhs) const - { - return nRegisteredHeight == rhs.nRegisteredHeight && - nLastPaidHeight == rhs.nLastPaidHeight && - nPoSePenalty == rhs.nPoSePenalty && - nPoSeRevivedHeight == rhs.nPoSeRevivedHeight && - nPoSeBanHeight == rhs.nPoSeBanHeight && - nRevocationReason == rhs.nRevocationReason && - confirmedHash == rhs.confirmedHash && - confirmedHashWithProRegTxHash == rhs.confirmedHashWithProRegTxHash && - keyIDOwner == rhs.keyIDOwner && - pubKeyOperator == rhs.pubKeyOperator && - keyIDVoting == rhs.keyIDVoting && - addr == rhs.addr && - scriptPayout == rhs.scriptPayout && - scriptOperatorPayout == rhs.scriptOperatorPayout; - } - - bool operator!=(const CDeterministicMNState& rhs) const - { - return !(rhs == *this); - } - public: std::string ToString() const; void ToJson(UniValue& obj) const; @@ -139,6 +116,75 @@ public: typedef std::shared_ptr CDeterministicMNStatePtr; typedef std::shared_ptr CDeterministicMNStateCPtr; +class CDeterministicMNStateDiff +{ +public: + enum Field : uint32_t { + Field_nRegisteredHeight = 0x0001, + Field_nLastPaidHeight = 0x0002, + Field_nPoSePenalty = 0x0004, + Field_nPoSeRevivedHeight = 0x0008, + Field_nPoSeBanHeight = 0x0010, + Field_nRevocationReason = 0x0020, + Field_confirmedHash = 0x0040, + Field_confirmedHashWithProRegTxHash = 0x0080, + Field_keyIDOwner = 0x0100, + Field_pubKeyOperator = 0x0200, + Field_keyIDVoting = 0x0400, + Field_addr = 0x0800, + Field_scriptPayout = 0x1000, + Field_scriptOperatorPayout = 0x2000, + }; + +#define DMN_STATE_DIFF_ALL_FIELDS \ + DMN_STATE_DIFF_LINE(nRegisteredHeight) \ + DMN_STATE_DIFF_LINE(nLastPaidHeight) \ + DMN_STATE_DIFF_LINE(nPoSePenalty) \ + DMN_STATE_DIFF_LINE(nPoSeRevivedHeight) \ + DMN_STATE_DIFF_LINE(nPoSeBanHeight) \ + DMN_STATE_DIFF_LINE(nRevocationReason) \ + DMN_STATE_DIFF_LINE(confirmedHash) \ + DMN_STATE_DIFF_LINE(confirmedHashWithProRegTxHash) \ + DMN_STATE_DIFF_LINE(keyIDOwner) \ + DMN_STATE_DIFF_LINE(pubKeyOperator) \ + DMN_STATE_DIFF_LINE(keyIDVoting) \ + DMN_STATE_DIFF_LINE(addr) \ + DMN_STATE_DIFF_LINE(scriptPayout) \ + DMN_STATE_DIFF_LINE(scriptOperatorPayout) + +public: + uint32_t fields{0}; + // we reuse the state class, but only the members as noted by fields are valid + CDeterministicMNState state; + +public: + CDeterministicMNStateDiff() {} + CDeterministicMNStateDiff(const CDeterministicMNState& a, const CDeterministicMNState& b) + { +#define DMN_STATE_DIFF_LINE(f) if (a.f != b.f) { state.f = b.f; fields |= Field_##f; } + DMN_STATE_DIFF_ALL_FIELDS +#undef DMN_STATE_DIFF_LINE + } + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(VARINT(fields)); +#define DMN_STATE_DIFF_LINE(f) if (fields & Field_##f) READWRITE(state.f); + DMN_STATE_DIFF_ALL_FIELDS +#undef DMN_STATE_DIFF_LINE + } + + void ApplyToState(CDeterministicMNState& target) const + { +#define DMN_STATE_DIFF_LINE(f) if (fields & Field_##f) target.f = state.f; + DMN_STATE_DIFF_ALL_FIELDS +#undef DMN_STATE_DIFF_LINE + } +}; + class CDeterministicMN { public: @@ -150,22 +196,36 @@ public: } uint256 proTxHash; + uint64_t internalId{std::numeric_limits::max()}; COutPoint collateralOutpoint; uint16_t nOperatorReward; CDeterministicMNStateCPtr pdmnState; public: - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) + inline void SerializationOp(Stream& s, Operation ser_action, bool oldFormat) { READWRITE(proTxHash); + if (!oldFormat) { + READWRITE(VARINT(internalId)); + } READWRITE(collateralOutpoint); READWRITE(nOperatorReward); READWRITE(pdmnState); } + template + void Serialize(Stream& s) const + { + NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize(), false); + } + + template + void Unserialize(Stream& s, bool oldFormat = false) + { + SerializationOp(s, CSerActionUnserialize(), oldFormat); + } + public: std::string ToString() const; void ToJson(UniValue& obj) const; @@ -214,35 +274,60 @@ class CDeterministicMNList { public: typedef immer::map MnMap; + typedef immer::map MnInternalIdMap; typedef immer::map > MnUniquePropertyMap; private: uint256 blockHash; int nHeight{-1}; + uint32_t nTotalRegisteredCount{0}; MnMap mnMap; + MnInternalIdMap mnInternalIdMap; // map of unique properties like address and keys // we keep track of this as checking for duplicates would otherwise be painfully slow - // the entries in the map are ref counted as some properties might appear multiple times per MN (e.g. operator/owner keys) MnUniquePropertyMap mnUniquePropertyMap; public: CDeterministicMNList() {} - explicit CDeterministicMNList(const uint256& _blockHash, int _height) : + explicit CDeterministicMNList(const uint256& _blockHash, int _height, uint32_t _totalRegisteredCount) : blockHash(_blockHash), - nHeight(_height) + nHeight(_height), + nTotalRegisteredCount(_totalRegisteredCount) { } - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) + inline void SerializationOpBase(Stream& s, Operation ser_action) { READWRITE(blockHash); READWRITE(nHeight); - READWRITE(mnMap); - READWRITE(mnUniquePropertyMap); + READWRITE(nTotalRegisteredCount); + } + + template + void Serialize(Stream& s) const + { + NCONST_PTR(this)->SerializationOpBase(s, CSerActionSerialize()); + // Serialize the map as a vector + WriteCompactSize(s, mnMap.size()); + for (const auto& p : mnMap) { + s << *p.second; + } + } + + template + void Unserialize(Stream& s) { + mnMap = MnMap(); + mnUniquePropertyMap = MnUniquePropertyMap(); + mnInternalIdMap = MnInternalIdMap(); + + SerializationOpBase(s, CSerActionUnserialize()); + + size_t cnt = ReadCompactSize(s); + for (size_t i = 0; i < cnt; i++) { + AddMN(std::make_shared(deserialize, s)); + } } public: @@ -289,6 +374,14 @@ public: { nHeight = _height; } + uint32_t GetTotalRegisteredCount() const + { + return nTotalRegisteredCount; + } + void SetTotalRegisteredCount(uint32_t _count) + { + nTotalRegisteredCount = _count; + } bool IsMNValid(const uint256& proTxHash) const; bool IsMNPoSeBanned(const uint256& proTxHash) const; @@ -318,6 +411,7 @@ public: CDeterministicMNCPtr GetValidMNByCollateral(const COutPoint& collateralOutpoint) const; CDeterministicMNCPtr GetMNByService(const CService& service) const; CDeterministicMNCPtr GetValidMNByService(const CService& service) const; + CDeterministicMNCPtr GetMNByInternalId(uint64_t internalId) const; CDeterministicMNCPtr GetMNPayee() const; /** @@ -372,10 +466,12 @@ public: CDeterministicMNListDiff BuildDiff(const CDeterministicMNList& to) const; CSimplifiedMNListDiff BuildSimplifiedDiff(const CDeterministicMNList& to) const; - CDeterministicMNList ApplyDiff(const CDeterministicMNListDiff& diff) const; + CDeterministicMNList ApplyDiff(const CBlockIndex* pindex, const CDeterministicMNListDiff& diff) const; void AddMN(const CDeterministicMNCPtr& dmn); + void UpdateMN(const CDeterministicMNCPtr& oldDmn, const CDeterministicMNStateCPtr& pdmnState); void UpdateMN(const uint256& proTxHash, const CDeterministicMNStateCPtr& pdmnState); + void UpdateMN(const CDeterministicMNCPtr& oldDmn, const CDeterministicMNStateDiff& stateDiff); void RemoveMN(const uint256& proTxHash); template @@ -444,6 +540,61 @@ private: class CDeterministicMNListDiff { +public: + std::vector addedMNs; + // keys are all relating to the internalId of MNs + std::map updatedMNs; + std::set removedMns; + +public: + template + void Serialize(Stream& s) const + { + s << addedMNs; + WriteCompactSize(s, updatedMNs.size()); + for (const auto& p : updatedMNs) { + WriteVarInt(s, p.first); + s << p.second; + } + WriteCompactSize(s, removedMns.size()); + for (const auto& p : removedMns) { + WriteVarInt(s, p); + } + } + + template + void Unserialize(Stream& s) + { + updatedMNs.clear(); + removedMns.clear(); + + size_t tmp; + uint64_t tmp2; + s >> addedMNs; + tmp = ReadCompactSize(s); + for (size_t i = 0; i < tmp; i++) { + CDeterministicMNStateDiff diff; + tmp2 = ReadVarInt(s); + s >> diff; + updatedMNs.emplace(tmp2, std::move(diff)); + } + tmp = ReadCompactSize(s); + for (size_t i = 0; i < tmp; i++) { + tmp2 = ReadVarInt(s); + removedMns.emplace(tmp2); + } + } + +public: + bool HasChanges() const + { + return !addedMNs.empty() || !updatedMNs.empty() || !removedMns.empty(); + } +}; + +// TODO can be removed in a future version +class CDeterministicMNListDiff_OldFormat +{ public: uint256 prevBlockHash; uint256 blockHash; @@ -453,23 +604,22 @@ public: std::set removedMns; public: - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(prevBlockHash); - READWRITE(blockHash); - READWRITE(nHeight); - READWRITE(addedMNs); - READWRITE(updatedMNs); - READWRITE(removedMns); - } - -public: - bool HasChanges() const - { - return !addedMNs.empty() || !updatedMNs.empty() || !removedMns.empty(); + template + void Unserialize(Stream& s) { + addedMNs.clear(); + s >> prevBlockHash; + s >> blockHash; + s >> nHeight; + size_t cnt = ReadCompactSize(s); + for (size_t i = 0; i < cnt; i++) { + uint256 proTxHash; + auto dmn = std::make_shared(); + s >> proTxHash; + dmn->Unserialize(s, true); + addedMNs.emplace(proTxHash, dmn); + } + s >> updatedMNs; + s >> removedMns; } }; @@ -485,8 +635,7 @@ private: CEvoDB& evoDb; std::map mnListsCache; - int tipHeight{-1}; - uint256 tipBlockHash; + const CBlockIndex* tipIndex{nullptr}; public: CDeterministicMNManager(CEvoDB& _evoDb); @@ -498,10 +647,10 @@ public: // the returned list will not contain the correct block hash (we can't know it yet as the coinbase TX is not updated yet) bool BuildNewListFromBlock(const CBlock& block, const CBlockIndex* pindexPrev, CValidationState& state, CDeterministicMNList& mnListRet, bool debugLogs); - void HandleQuorumCommitment(llmq::CFinalCommitment& qc, CDeterministicMNList& mnList, bool debugLogs); + void HandleQuorumCommitment(llmq::CFinalCommitment& qc, const CBlockIndex* pindexQuorum, CDeterministicMNList& mnList, bool debugLogs); void DecreasePoSePenalties(CDeterministicMNList& mnList); - CDeterministicMNList GetListForBlock(const uint256& blockHash); + CDeterministicMNList GetListForBlock(const CBlockIndex* pindex); CDeterministicMNList GetListAtChainTip(); // Test if given TX is a ProRegTx which also contains the collateral at index n @@ -509,6 +658,11 @@ public: bool IsDIP3Enforced(int nHeight = -1); +public: + // TODO these can all be removed in a future version + bool UpgradeDiff(CDBBatch& batch, const CBlockIndex* pindexNext, const CDeterministicMNList& curMNList, CDeterministicMNList& newMNList); + void UpgradeDBIfNeeded(); + private: void CleanupCache(int nHeight); }; diff --git a/src/evo/evodb.h b/src/evo/evodb.h index 03e9b3acf..b5f282d35 100644 --- a/src/evo/evodb.h +++ b/src/evo/evodb.h @@ -9,7 +9,9 @@ #include "sync.h" #include "uint256.h" -static const std::string EVODB_BEST_BLOCK = "b_b"; +// "b_b" was used in the initial version of deterministic MN storage +// "b_b2" was used after compact diffs were introduced +static const std::string EVODB_BEST_BLOCK = "b_b2"; class CEvoDB { diff --git a/src/evo/mnauth.cpp b/src/evo/mnauth.cpp index a8aed3fd4..e5f0c44e6 100644 --- a/src/evo/mnauth.cpp +++ b/src/evo/mnauth.cpp @@ -127,13 +127,17 @@ void CMNAuth::NotifyMasternodeListChanged(bool undo, const CDeterministicMNList& if (pnode->verifiedProRegTxHash.IsNull()) { return; } + auto verifiedDmn = oldMNList.GetMN(pnode->verifiedProRegTxHash); + if (!verifiedDmn) { + return; + } bool doRemove = false; - if (diff.removedMns.count(pnode->verifiedProRegTxHash)) { + if (diff.removedMns.count(verifiedDmn->internalId)) { doRemove = true; } else { - auto it = diff.updatedMNs.find(pnode->verifiedProRegTxHash); + auto it = diff.updatedMNs.find(verifiedDmn->internalId); if (it != diff.updatedMNs.end()) { - if (it->second->pubKeyOperator.GetHash() != pnode->verifiedPubKeyHash) { + if ((it->second.fields & CDeterministicMNStateDiff::Field_pubKeyOperator) && it->second.state.pubKeyOperator.GetHash() != pnode->verifiedPubKeyHash) { doRemove = true; } } diff --git a/src/evo/providertx.cpp b/src/evo/providertx.cpp index 019f427f6..02dc664e7 100644 --- a/src/evo/providertx.cpp +++ b/src/evo/providertx.cpp @@ -174,7 +174,7 @@ bool CheckProRegTx(const CTransaction& tx, const CBlockIndex* pindexPrev, CValid } if (pindexPrev) { - auto mnList = deterministicMNManager->GetListForBlock(pindexPrev->GetBlockHash()); + auto mnList = deterministicMNManager->GetListForBlock(pindexPrev); // only allow reusing of addresses when it's for the same collateral (which replaces the old MN) if (mnList.HasUniqueProperty(ptx.addr) && mnList.GetUniquePropertyMN(ptx.addr)->collateralOutpoint != collateralOutpoint) { @@ -232,7 +232,7 @@ bool CheckProUpServTx(const CTransaction& tx, const CBlockIndex* pindexPrev, CVa } if (pindexPrev) { - auto mnList = deterministicMNManager->GetListForBlock(pindexPrev->GetBlockHash()); + auto mnList = deterministicMNManager->GetListForBlock(pindexPrev); auto mn = mnList.GetMN(ptx.proTxHash); if (!mn) { return state.DoS(100, false, REJECT_INVALID, "bad-protx-hash"); @@ -297,7 +297,7 @@ bool CheckProUpRegTx(const CTransaction& tx, const CBlockIndex* pindexPrev, CVal } if (pindexPrev) { - auto mnList = deterministicMNManager->GetListForBlock(pindexPrev->GetBlockHash()); + auto mnList = deterministicMNManager->GetListForBlock(pindexPrev); auto dmn = mnList.GetMN(ptx.proTxHash); if (!dmn) { return state.DoS(100, false, REJECT_INVALID, "bad-protx-hash"); @@ -369,7 +369,7 @@ bool CheckProUpRevTx(const CTransaction& tx, const CBlockIndex* pindexPrev, CVal } if (pindexPrev) { - auto mnList = deterministicMNManager->GetListForBlock(pindexPrev->GetBlockHash()); + auto mnList = deterministicMNManager->GetListForBlock(pindexPrev); auto dmn = mnList.GetMN(ptx.proTxHash); if (!dmn) return state.DoS(100, false, REJECT_INVALID, "bad-protx-hash"); diff --git a/src/evo/providertx.h b/src/evo/providertx.h index bb5c0f18e..b7eb6fe87 100644 --- a/src/evo/providertx.h +++ b/src/evo/providertx.h @@ -49,7 +49,7 @@ public: READWRITE(pubKeyOperator); READWRITE(keyIDVoting); READWRITE(nOperatorReward); - READWRITE(*(CScriptBase*)(&scriptPayout)); + READWRITE(scriptPayout); READWRITE(inputsHash); if (!(s.GetType() & SER_GETHASH)) { READWRITE(vchSig); @@ -86,7 +86,7 @@ public: READWRITE(nVersion); READWRITE(proTxHash); READWRITE(addr); - READWRITE(*(CScriptBase*)(&scriptOperatorPayout)); + READWRITE(scriptOperatorPayout); READWRITE(inputsHash); if (!(s.GetType() & SER_GETHASH)) { READWRITE(sig); @@ -124,7 +124,7 @@ public: READWRITE(nMode); READWRITE(pubKeyOperator); READWRITE(keyIDVoting); - READWRITE(*(CScriptBase*)(&scriptPayout)); + READWRITE(scriptPayout); READWRITE(inputsHash); if (!(s.GetType() & SER_GETHASH)) { READWRITE(vchSig); diff --git a/src/evo/simplifiedmns.cpp b/src/evo/simplifiedmns.cpp index 10c0d59b4..db272d02e 100644 --- a/src/evo/simplifiedmns.cpp +++ b/src/evo/simplifiedmns.cpp @@ -217,10 +217,15 @@ bool BuildSimplifiedMNListDiff(const uint256& baseBlockHash, const uint256& bloc LOCK(deterministicMNManager->cs); - auto baseDmnList = deterministicMNManager->GetListForBlock(baseBlockHash); - auto dmnList = deterministicMNManager->GetListForBlock(blockHash); + auto baseDmnList = deterministicMNManager->GetListForBlock(baseBlockIndex); + auto dmnList = deterministicMNManager->GetListForBlock(blockIndex); mnListDiffRet = baseDmnList.BuildSimplifiedDiff(dmnList); + // We need to return the value that was provided by the other peer as it otherwise won't be able to recognize the + // response. This will usually be identical to the block found in baseBlockIndex. The only difference is when a + // null block hash was provided to get the diff from the genesis block. + mnListDiffRet.baseBlockHash = baseBlockHash; + if (!mnListDiffRet.BuildQuorumsDiff(baseBlockIndex, blockIndex)) { errorRet = strprintf("failed to build quorums diff"); return false; diff --git a/src/governance.cpp b/src/governance.cpp index 18a2b1181..46a09b44a 100644 --- a/src/governance.cpp +++ b/src/governance.cpp @@ -24,7 +24,7 @@ CGovernanceManager governance; int nSubmittedFinalBudget; -const std::string CGovernanceManager::SERIALIZATION_VERSION_STRING = "CGovernanceManager-Version-14"; +const std::string CGovernanceManager::SERIALIZATION_VERSION_STRING = "CGovernanceManager-Version-15"; const int CGovernanceManager::MAX_TIME_FUTURE_DEVIATION = 60 * 60; const int CGovernanceManager::RELIABLE_PROPAGATION_TIME = 60; @@ -1337,15 +1337,15 @@ void CGovernanceManager::RemoveInvalidVotes() std::vector changedKeyMNs; for (const auto& p : diff.updatedMNs) { - auto oldDmn = lastMNListForVotingKeys.GetMN(p.first); - if (p.second->keyIDVoting != oldDmn->pdmnState->keyIDVoting) { + auto oldDmn = lastMNListForVotingKeys.GetMNByInternalId(p.first); + if ((p.second.fields & CDeterministicMNStateDiff::Field_keyIDVoting) && p.second.state.keyIDVoting != oldDmn->pdmnState->keyIDVoting) { changedKeyMNs.emplace_back(oldDmn->collateralOutpoint); - } else if (p.second->pubKeyOperator != oldDmn->pdmnState->pubKeyOperator) { + } else if ((p.second.fields & CDeterministicMNStateDiff::Field_pubKeyOperator) && p.second.state.pubKeyOperator != oldDmn->pdmnState->pubKeyOperator) { changedKeyMNs.emplace_back(oldDmn->collateralOutpoint); } } - for (const auto& proTxHash : diff.removedMns) { - auto oldDmn = lastMNListForVotingKeys.GetMN(proTxHash); + for (const auto& id : diff.removedMns) { + auto oldDmn = lastMNListForVotingKeys.GetMNByInternalId(id); changedKeyMNs.emplace_back(oldDmn->collateralOutpoint); } diff --git a/src/governance.h b/src/governance.h index 466b0f971..968960f60 100644 --- a/src/governance.h +++ b/src/governance.h @@ -342,7 +342,11 @@ public: LOCK(cs); std::string strVersion; if (ser_action.ForRead()) { + Clear(); READWRITE(strVersion); + if (strVersion != SERIALIZATION_VERSION_STRING) { + return; + } } else { strVersion = SERIALIZATION_VERSION_STRING; READWRITE(strVersion); @@ -354,10 +358,6 @@ public: READWRITE(mapObjects); READWRITE(mapLastMasternodeObject); READWRITE(lastMNListForVotingKeys); - if (ser_action.ForRead() && (strVersion != SERIALIZATION_VERSION_STRING)) { - Clear(); - return; - } } void UpdatedBlockTip(const CBlockIndex* pindex, CConnman& connman); diff --git a/src/init.cpp b/src/init.cpp index 651012156..246b1f963 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1831,6 +1831,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) break; } + deterministicMNManager->UpgradeDBIfNeeded(); + uiInterface.InitMessage(_("Verifying blocks...")); if (fHavePruned && GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks", diff --git a/src/llmq/quorums.cpp b/src/llmq/quorums.cpp index f739053b7..ec938f0a1 100644 --- a/src/llmq/quorums.cpp +++ b/src/llmq/quorums.cpp @@ -49,10 +49,10 @@ CQuorum::~CQuorum() } } -void CQuorum::Init(const CFinalCommitment& _qc, int _height, const uint256& _minedBlockHash, const std::vector& _members) +void CQuorum::Init(const CFinalCommitment& _qc, const CBlockIndex* _pindexQuorum, const uint256& _minedBlockHash, const std::vector& _members) { qc = _qc; - height = _height; + pindexQuorum = _pindexQuorum; members = _members; minedBlockHash = _minedBlockHash; } @@ -193,9 +193,9 @@ void CQuorumManager::EnsureQuorumConnections(Consensus::LLMQType llmqType, const if (!g_connman->HasMasternodeQuorumNodes(llmqType, quorum->qc.quorumHash)) { std::set connections; if (quorum->IsMember(myProTxHash)) { - connections = CLLMQUtils::GetQuorumConnections(llmqType, quorum->qc.quorumHash, myProTxHash); + connections = CLLMQUtils::GetQuorumConnections(llmqType, quorum->pindexQuorum, myProTxHash); } else { - auto cindexes = CLLMQUtils::CalcDeterministicWatchConnections(llmqType, quorum->qc.quorumHash, quorum->members.size(), 1); + auto cindexes = CLLMQUtils::CalcDeterministicWatchConnections(llmqType, quorum->pindexQuorum, quorum->members.size(), 1); for (auto idx : cindexes) { connections.emplace(quorum->members[idx]->proTxHash); } @@ -231,9 +231,9 @@ bool CQuorumManager::BuildQuorumFromCommitment(const CFinalCommitment& qc, const assert(pindexQuorum); assert(qc.quorumHash == pindexQuorum->GetBlockHash()); - auto members = CLLMQUtils::GetAllQuorumMembers((Consensus::LLMQType)qc.llmqType, qc.quorumHash); + auto members = CLLMQUtils::GetAllQuorumMembers((Consensus::LLMQType)qc.llmqType, pindexQuorum); - quorum->Init(qc, pindexQuorum->nHeight, minedBlockHash, members); + quorum->Init(qc, pindexQuorum, minedBlockHash, members); bool hasValidVvec = false; if (quorum->ReadContributions(evoDb)) { @@ -262,7 +262,7 @@ bool CQuorumManager::BuildQuorumContributions(const CFinalCommitment& fqc, std:: std::vector memberIndexes; std::vector vvecs; BLSSecretKeyVector skContributions; - if (!dkgManager.GetVerifiedContributions((Consensus::LLMQType)fqc.llmqType, fqc.quorumHash, fqc.validMembers, memberIndexes, vvecs, skContributions)) { + if (!dkgManager.GetVerifiedContributions((Consensus::LLMQType)fqc.llmqType, quorum->pindexQuorum, fqc.validMembers, memberIndexes, vvecs, skContributions)) { return false; } diff --git a/src/llmq/quorums.h b/src/llmq/quorums.h index 8adaf5f44..37da0f0d5 100644 --- a/src/llmq/quorums.h +++ b/src/llmq/quorums.h @@ -37,7 +37,7 @@ class CQuorum public: const Consensus::LLMQParams& params; CFinalCommitment qc; - int height; + const CBlockIndex* pindexQuorum; uint256 minedBlockHash; std::vector members; @@ -55,7 +55,7 @@ private: public: CQuorum(const Consensus::LLMQParams& _params, CBLSWorker& _blsWorker) : params(_params), blsCache(_blsWorker), stopCachePopulatorThread(false) {} ~CQuorum(); - void Init(const CFinalCommitment& _qc, int _height, const uint256& _minedBlockHash, const std::vector& _members); + void Init(const CFinalCommitment& _qc, const CBlockIndex* _pindexQuorum, const uint256& _minedBlockHash, const std::vector& _members); bool IsMember(const uint256& proTxHash) const; bool IsValidMember(const uint256& proTxHash) const; diff --git a/src/llmq/quorums_blockprocessor.cpp b/src/llmq/quorums_blockprocessor.cpp index eb0c3ba27..c24170c28 100644 --- a/src/llmq/quorums_blockprocessor.cpp +++ b/src/llmq/quorums_blockprocessor.cpp @@ -57,6 +57,7 @@ void CQuorumBlockProcessor::ProcessMessage(CNode* pfrom, const std::string& strC const auto& params = Params().GetConsensus().llmqs.at(type); // Verify that quorumHash is part of the active chain and that it's the first block in the DKG interval + const CBlockIndex* pquorumIndex; { LOCK(cs_main); if (!mapBlockIndex.count(qc.quorumHash)) { @@ -66,7 +67,7 @@ void CQuorumBlockProcessor::ProcessMessage(CNode* pfrom, const std::string& strC // fully synced return; } - auto pquorumIndex = mapBlockIndex[qc.quorumHash]; + pquorumIndex = mapBlockIndex[qc.quorumHash]; if (chainActive.Tip()->GetAncestor(pquorumIndex->nHeight) != pquorumIndex) { LogPrintf("CQuorumBlockProcessor::%s -- block %s not in active chain, peer=%d\n", __func__, qc.quorumHash.ToString(), pfrom->id); @@ -98,7 +99,7 @@ void CQuorumBlockProcessor::ProcessMessage(CNode* pfrom, const std::string& strC } } - auto members = CLLMQUtils::GetAllQuorumMembers(type, qc.quorumHash); + auto members = CLLMQUtils::GetAllQuorumMembers(type, pquorumIndex); if (!qc.Verify(members, true)) { LOCK(cs_main); @@ -203,14 +204,14 @@ bool CQuorumBlockProcessor::ProcessCommitment(int nHeight, const uint256& blockH return state.DoS(100, false, REJECT_INVALID, "bad-qc-height"); } - auto members = CLLMQUtils::GetAllQuorumMembers(params.type, quorumHash); + auto quorumIndex = mapBlockIndex.at(qc.quorumHash); + auto members = CLLMQUtils::GetAllQuorumMembers(params.type, quorumIndex); if (!qc.Verify(members, true)) { return state.DoS(100, false, REJECT_INVALID, "bad-qc-invalid"); } // Store commitment in DB - auto quorumIndex = mapBlockIndex.at(qc.quorumHash); evoDb.Write(std::make_pair(DB_MINED_COMMITMENT, std::make_pair((uint8_t)params.type, quorumHash)), std::make_pair(qc, blockHash)); evoDb.Write(BuildInversedHeightKey(params.type, nHeight), quorumIndex->nHeight); diff --git a/src/llmq/quorums_commitment.cpp b/src/llmq/quorums_commitment.cpp index 5f84ab942..85ef4bfa3 100644 --- a/src/llmq/quorums_commitment.cpp +++ b/src/llmq/quorums_commitment.cpp @@ -193,7 +193,7 @@ bool CheckLLMQCommitment(const CTransaction& tx, const CBlockIndex* pindexPrev, return true; } - auto members = CLLMQUtils::GetAllQuorumMembers(params.type, qcTx.commitment.quorumHash); + auto members = CLLMQUtils::GetAllQuorumMembers(params.type, pindexQuorum); if (!qcTx.commitment.Verify(members, false)) { return state.DoS(100, false, REJECT_INVALID, "bad-qc-invalid"); } diff --git a/src/llmq/quorums_debug.cpp b/src/llmq/quorums_debug.cpp index dba8963e9..13be503d0 100644 --- a/src/llmq/quorums_debug.cpp +++ b/src/llmq/quorums_debug.cpp @@ -5,6 +5,7 @@ #include "quorums_debug.h" #include "chainparams.h" +#include "validation.h" #include "evo/deterministicmns.h" #include "quorums_utils.h" @@ -23,7 +24,17 @@ UniValue CDKGDebugSessionStatus::ToJson(int detailLevel) const std::vector dmnMembers; if (detailLevel == 2) { - dmnMembers = CLLMQUtils::GetAllQuorumMembers((Consensus::LLMQType) llmqType, quorumHash); + const CBlockIndex* pindex = nullptr; + { + LOCK(cs_main); + auto it = mapBlockIndex.find(quorumHash); + if (it != mapBlockIndex.end()) { + pindex = it->second; + } + } + if (pindex != nullptr) { + dmnMembers = CLLMQUtils::GetAllQuorumMembers((Consensus::LLMQType) llmqType, pindex); + } } ret.push_back(Pair("llmqType", llmqType)); diff --git a/src/llmq/quorums_dkgsession.cpp b/src/llmq/quorums_dkgsession.cpp index 1cfd343d3..d41633ecf 100644 --- a/src/llmq/quorums_dkgsession.cpp +++ b/src/llmq/quorums_dkgsession.cpp @@ -60,7 +60,7 @@ static bool ShouldSimulateError(const std::string& type) } CDKGLogger::CDKGLogger(const CDKGSession& _quorumDkg, const std::string& _func) : - CDKGLogger(_quorumDkg.params.type, _quorumDkg.quorumHash, _quorumDkg.height, _quorumDkg.AreWeMember(), _func) + CDKGLogger(_quorumDkg.params.type, _quorumDkg.pindexQuorum->GetBlockHash(), _quorumDkg.pindexQuorum->nHeight, _quorumDkg.AreWeMember(), _func) { } @@ -88,14 +88,13 @@ CDKGMember::CDKGMember(CDeterministicMNCPtr _dmn, size_t _idx) : } -bool CDKGSession::Init(int _height, const uint256& _quorumHash, const std::vector& mns, const uint256& _myProTxHash) +bool CDKGSession::Init(const CBlockIndex* _pindexQuorum, const std::vector& mns, const uint256& _myProTxHash) { if (mns.size() < params.minSize) { return false; } - height = _height; - quorumHash = _quorumHash; + pindexQuorum = _pindexQuorum; members.resize(mns.size()); memberIds.resize(members.size()); @@ -121,7 +120,7 @@ bool CDKGSession::Init(int _height, const uint256& _quorumHash, const std::vecto } if (!myProTxHash.IsNull()) { - quorumDKGDebugManager->InitLocalSessionStatus(params.type, quorumHash, height); + quorumDKGDebugManager->InitLocalSessionStatus(params.type, pindexQuorum->GetBlockHash(), pindexQuorum->nHeight); } CDKGLogger logger(*this, __func__); @@ -170,7 +169,7 @@ void CDKGSession::SendContributions(CDKGPendingMessages& pendingMessages) CDKGContribution qc; qc.llmqType = (uint8_t)params.type; - qc.quorumHash = quorumHash; + qc.quorumHash = pindexQuorum->GetBlockHash(); qc.proTxHash = myProTxHash; qc.vvec = vvecContribution; @@ -216,7 +215,7 @@ bool CDKGSession::PreVerifyMessage(const uint256& hash, const CDKGContribution& retBan = false; - if (qc.quorumHash != quorumHash) { + if (qc.quorumHash != pindexQuorum->GetBlockHash()) { logger.Batch("contribution for wrong quorum, rejecting"); return false; } @@ -316,7 +315,7 @@ void CDKGSession::ReceiveMessage(const uint256& hash, const CDKGContribution& qc return; } - dkgManager.WriteVerifiedVvecContribution(params.type, qc.quorumHash, qc.proTxHash, qc.vvec); + dkgManager.WriteVerifiedVvecContribution(params.type, pindexQuorum, qc.proTxHash, qc.vvec); bool complain = false; CBLSSecretKey skContribution; @@ -398,7 +397,7 @@ void CDKGSession::VerifyPendingContributions() }); } else { size_t memberIdx = memberIndexes[i]; - dkgManager.WriteVerifiedSkContribution(params.type, quorumHash, members[memberIdx]->dmn->proTxHash, skContributions[i]); + dkgManager.WriteVerifiedSkContribution(params.type, pindexQuorum, members[memberIdx]->dmn->proTxHash, skContributions[i]); } } @@ -449,7 +448,7 @@ void CDKGSession::SendComplaint(CDKGPendingMessages& pendingMessages) CDKGComplaint qc(params); qc.llmqType = (uint8_t)params.type; - qc.quorumHash = quorumHash; + qc.quorumHash = pindexQuorum->GetBlockHash(); qc.proTxHash = myProTxHash; int badCount = 0; @@ -490,7 +489,7 @@ bool CDKGSession::PreVerifyMessage(const uint256& hash, const CDKGComplaint& qc, retBan = false; - if (qc.quorumHash != quorumHash) { + if (qc.quorumHash != pindexQuorum->GetBlockHash()) { logger.Batch("complaint for wrong quorum, rejecting"); return false; } @@ -643,7 +642,7 @@ void CDKGSession::SendJustification(CDKGPendingMessages& pendingMessages, const CDKGJustification qj; qj.llmqType = (uint8_t)params.type; - qj.quorumHash = quorumHash; + qj.quorumHash = pindexQuorum->GetBlockHash(); qj.proTxHash = myProTxHash; qj.contributions.reserve(forMembers.size()); @@ -688,7 +687,7 @@ bool CDKGSession::PreVerifyMessage(const uint256& hash, const CDKGJustification& retBan = false; - if (qj.quorumHash != quorumHash) { + if (qj.quorumHash != pindexQuorum->GetBlockHash()) { logger.Batch("justification for wrong quorum, rejecting"); return false; } @@ -824,7 +823,7 @@ void CDKGSession::ReceiveMessage(const uint256& hash, const CDKGJustification& q receivedSkContributions[member->idx] = skContribution; member->weComplain = false; - dkgManager.WriteVerifiedSkContribution(params.type, quorumHash, member->dmn->proTxHash, skContribution); + dkgManager.WriteVerifiedSkContribution(params.type, pindexQuorum, member->dmn->proTxHash, skContribution); } member->complaintsFromOthers.erase(member2->dmn->proTxHash); } @@ -899,7 +898,7 @@ void CDKGSession::SendCommitment(CDKGPendingMessages& pendingMessages) CDKGPrematureCommitment qc(params); qc.llmqType = (uint8_t)params.type; - qc.quorumHash = quorumHash; + qc.quorumHash = pindexQuorum->GetBlockHash(); qc.proTxHash = myProTxHash; for (size_t i = 0; i < members.size(); i++) { @@ -925,7 +924,7 @@ void CDKGSession::SendCommitment(CDKGPendingMessages& pendingMessages) std::vector memberIndexes; std::vector vvecs; BLSSecretKeyVector skContributions; - if (!dkgManager.GetVerifiedContributions(params.type, quorumHash, qc.validMembers, memberIndexes, vvecs, skContributions)) { + if (!dkgManager.GetVerifiedContributions(params.type, pindexQuorum, qc.validMembers, memberIndexes, vvecs, skContributions)) { logger.Batch("failed to get valid contributions"); return; } @@ -1012,7 +1011,7 @@ bool CDKGSession::PreVerifyMessage(const uint256& hash, const CDKGPrematureCommi retBan = false; - if (qc.quorumHash != quorumHash) { + if (qc.quorumHash != pindexQuorum->GetBlockHash()) { logger.Batch("commitment for wrong quorum, rejecting"); return false; } @@ -1091,7 +1090,7 @@ void CDKGSession::ReceiveMessage(const uint256& hash, const CDKGPrematureCommitm std::vector vvecs; BLSSecretKeyVector skContributions; BLSVerificationVectorPtr quorumVvec; - if (dkgManager.GetVerifiedContributions(params.type, qc.quorumHash, qc.validMembers, memberIndexes, vvecs, skContributions)) { + if (dkgManager.GetVerifiedContributions(params.type, pindexQuorum, qc.validMembers, memberIndexes, vvecs, skContributions)) { quorumVvec = cache.BuildQuorumVerificationVector(::SerializeHash(memberIndexes), vvecs); } diff --git a/src/llmq/quorums_dkgsession.h b/src/llmq/quorums_dkgsession.h index e6a23a5f6..38e96dc65 100644 --- a/src/llmq/quorums_dkgsession.h +++ b/src/llmq/quorums_dkgsession.h @@ -249,8 +249,7 @@ private: CBLSWorkerCache cache; CDKGSessionManager& dkgManager; - uint256 quorumHash; - int height{-1}; + const CBlockIndex* pindexQuorum; private: std::vector> members; @@ -287,7 +286,7 @@ public: CDKGSession(const Consensus::LLMQParams& _params, CBLSWorker& _blsWorker, CDKGSessionManager& _dkgManager) : params(_params), blsWorker(_blsWorker), cache(_blsWorker), dkgManager(_dkgManager) {} - bool Init(int _height, const uint256& _quorumHash, const std::vector& mns, const uint256& _myProTxHash); + bool Init(const CBlockIndex* pindexQuorum, const std::vector& mns, const uint256& _myProTxHash); size_t GetMyMemberIndex() const { return myIdx; } diff --git a/src/llmq/quorums_dkgsessionhandler.cpp b/src/llmq/quorums_dkgsessionhandler.cpp index 938b2fe7d..e20bf952a 100644 --- a/src/llmq/quorums_dkgsessionhandler.cpp +++ b/src/llmq/quorums_dkgsessionhandler.cpp @@ -139,7 +139,7 @@ void CDKGSessionHandler::ProcessMessage(CNode* pfrom, const std::string& strComm } } -bool CDKGSessionHandler::InitNewQuorum(int newQuorumHeight, const uint256& newQuorumHash) +bool CDKGSessionHandler::InitNewQuorum(const CBlockIndex* pindexQuorum) { //AssertLockHeld(cs_main); @@ -147,13 +147,13 @@ bool CDKGSessionHandler::InitNewQuorum(int newQuorumHeight, const uint256& newQu curSession = std::make_shared(params, blsWorker, dkgManager); - if (!deterministicMNManager->IsDIP3Enforced(newQuorumHeight)) { + if (!deterministicMNManager->IsDIP3Enforced(pindexQuorum->nHeight)) { return false; } - auto mns = CLLMQUtils::GetAllQuorumMembers(params.type, newQuorumHash); + auto mns = CLLMQUtils::GetAllQuorumMembers(params.type, pindexQuorum); - if (!curSession->Init(newQuorumHeight, newQuorumHash, mns, activeMasternodeInfo.proTxHash)) { + if (!curSession->Init(pindexQuorum, mns, activeMasternodeInfo.proTxHash)) { LogPrintf("CDKGSessionManager::%s -- quorum initialiation failed\n", __func__); return false; } @@ -455,7 +455,13 @@ void CDKGSessionHandler::HandleDKGRound() curQuorumHeight = quorumHeight; } - if (!InitNewQuorum(curQuorumHeight, curQuorumHash)) { + const CBlockIndex* pindexQuorum; + { + LOCK(cs_main); + pindexQuorum = mapBlockIndex.at(curQuorumHash); + } + + if (!InitNewQuorum(pindexQuorum)) { // should actually never happen WaitForNewQuorum(curQuorumHash); throw AbortPhaseException(); @@ -470,16 +476,16 @@ void CDKGSessionHandler::HandleDKGRound() if (curSession->AreWeMember() || GetBoolArg("-watchquorums", DEFAULT_WATCH_QUORUMS)) { std::set connections; if (curSession->AreWeMember()) { - connections = CLLMQUtils::GetQuorumConnections(params.type, curQuorumHash, curSession->myProTxHash); + connections = CLLMQUtils::GetQuorumConnections(params.type, pindexQuorum, curSession->myProTxHash); } else { - auto cindexes = CLLMQUtils::CalcDeterministicWatchConnections(params.type, curQuorumHash, curSession->members.size(), 1); + auto cindexes = CLLMQUtils::CalcDeterministicWatchConnections(params.type, pindexQuorum, curSession->members.size(), 1); for (auto idx : cindexes) { connections.emplace(curSession->members[idx]->dmn->proTxHash); } } if (!connections.empty()) { if (LogAcceptCategory("llmq-dkg")) { - std::string debugMsg = strprintf("CDKGSessionManager::%s -- adding masternodes quorum connections for quorum %s:\n", __func__, curSession->quorumHash.ToString()); + std::string debugMsg = strprintf("CDKGSessionManager::%s -- adding masternodes quorum connections for quorum %s:\n", __func__, curSession->pindexQuorum->GetBlockHash().ToString()); auto mnList = deterministicMNManager->GetListAtChainTip(); for (const auto& c : connections) { auto dmn = mnList.GetValidMN(c); diff --git a/src/llmq/quorums_dkgsessionhandler.h b/src/llmq/quorums_dkgsessionhandler.h index c297057e9..6613d06b3 100644 --- a/src/llmq/quorums_dkgsessionhandler.h +++ b/src/llmq/quorums_dkgsessionhandler.h @@ -125,7 +125,7 @@ public: void ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, CConnman& connman); private: - bool InitNewQuorum(int newQuorumHeight, const uint256& newQuorumHash); + bool InitNewQuorum(const CBlockIndex* pindexQuorum); std::pair GetPhaseAndQuorumHash() const; diff --git a/src/llmq/quorums_dkgsessionmgr.cpp b/src/llmq/quorums_dkgsessionmgr.cpp index c68d95d00..b42de05f2 100644 --- a/src/llmq/quorums_dkgsessionmgr.cpp +++ b/src/llmq/quorums_dkgsessionmgr.cpp @@ -198,19 +198,19 @@ bool CDKGSessionManager::GetPrematureCommitment(const uint256& hash, CDKGPrematu return false; } -void CDKGSessionManager::WriteVerifiedVvecContribution(Consensus::LLMQType llmqType, const uint256& quorumHash, const uint256& proTxHash, const BLSVerificationVectorPtr& vvec) +void CDKGSessionManager::WriteVerifiedVvecContribution(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& proTxHash, const BLSVerificationVectorPtr& vvec) { - llmqDb.Write(std::make_tuple(DB_VVEC, (uint8_t)llmqType, quorumHash, proTxHash), *vvec); + llmqDb.Write(std::make_tuple(DB_VVEC, (uint8_t) llmqType, pindexQuorum->GetBlockHash(), proTxHash), *vvec); } -void CDKGSessionManager::WriteVerifiedSkContribution(Consensus::LLMQType llmqType, const uint256& quorumHash, const uint256& proTxHash, const CBLSSecretKey& skContribution) +void CDKGSessionManager::WriteVerifiedSkContribution(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& proTxHash, const CBLSSecretKey& skContribution) { - llmqDb.Write(std::make_tuple(DB_SKCONTRIB, (uint8_t)llmqType, quorumHash, proTxHash), skContribution); + llmqDb.Write(std::make_tuple(DB_SKCONTRIB, (uint8_t) llmqType, pindexQuorum->GetBlockHash(), proTxHash), skContribution); } -bool CDKGSessionManager::GetVerifiedContributions(Consensus::LLMQType llmqType, const uint256& quorumHash, const std::vector& validMembers, std::vector& memberIndexesRet, std::vector& vvecsRet, BLSSecretKeyVector& skContributionsRet) +bool CDKGSessionManager::GetVerifiedContributions(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const std::vector& validMembers, std::vector& memberIndexesRet, std::vector& vvecsRet, BLSSecretKeyVector& skContributionsRet) { - auto members = CLLMQUtils::GetAllQuorumMembers(llmqType, quorumHash); + auto members = CLLMQUtils::GetAllQuorumMembers(llmqType, pindexQuorum); memberIndexesRet.clear(); vvecsRet.clear(); @@ -222,7 +222,7 @@ bool CDKGSessionManager::GetVerifiedContributions(Consensus::LLMQType llmqType, if (validMembers[i]) { BLSVerificationVectorPtr vvec; CBLSSecretKey skContribution; - if (!GetVerifiedContribution(llmqType, quorumHash, members[i]->proTxHash, vvec, skContribution)) { + if (!GetVerifiedContribution(llmqType, pindexQuorum, members[i]->proTxHash, vvec, skContribution)) { return false; } @@ -234,10 +234,10 @@ bool CDKGSessionManager::GetVerifiedContributions(Consensus::LLMQType llmqType, return true; } -bool CDKGSessionManager::GetVerifiedContribution(Consensus::LLMQType llmqType, const uint256& quorumHash, const uint256& proTxHash, BLSVerificationVectorPtr& vvecRet, CBLSSecretKey& skContributionRet) +bool CDKGSessionManager::GetVerifiedContribution(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& proTxHash, BLSVerificationVectorPtr& vvecRet, CBLSSecretKey& skContributionRet) { LOCK(contributionsCacheCs); - ContributionsCacheKey cacheKey = {llmqType, quorumHash, proTxHash}; + ContributionsCacheKey cacheKey = {llmqType, pindexQuorum->GetBlockHash(), proTxHash}; auto it = contributionsCache.find(cacheKey); if (it != contributionsCache.end()) { vvecRet = it->second.vvec; @@ -248,10 +248,10 @@ bool CDKGSessionManager::GetVerifiedContribution(Consensus::LLMQType llmqType, c BLSVerificationVector vvec; BLSVerificationVectorPtr vvecPtr; CBLSSecretKey skContribution; - if (llmqDb.Read(std::make_tuple(DB_VVEC, (uint8_t)llmqType, quorumHash, proTxHash), vvec)) { + if (llmqDb.Read(std::make_tuple(DB_VVEC, (uint8_t) llmqType, pindexQuorum->GetBlockHash(), proTxHash), vvec)) { vvecPtr = std::make_shared(std::move(vvec)); } - llmqDb.Read(std::make_tuple(DB_SKCONTRIB, (uint8_t)llmqType, quorumHash, proTxHash), skContribution); + llmqDb.Read(std::make_tuple(DB_SKCONTRIB, (uint8_t) llmqType, pindexQuorum->GetBlockHash(), proTxHash), skContribution); it = contributionsCache.emplace(cacheKey, ContributionsCacheEntry{GetTimeMillis(), vvecPtr, skContribution}).first; diff --git a/src/llmq/quorums_dkgsessionmgr.h b/src/llmq/quorums_dkgsessionmgr.h index cfb9850ec..721f18978 100644 --- a/src/llmq/quorums_dkgsessionmgr.h +++ b/src/llmq/quorums_dkgsessionmgr.h @@ -63,10 +63,10 @@ public: bool GetPrematureCommitment(const uint256& hash, CDKGPrematureCommitment& ret) const; // Verified contributions are written while in the DKG - void WriteVerifiedVvecContribution(Consensus::LLMQType llmqType, const uint256& quorumHash, const uint256& proTxHash, const BLSVerificationVectorPtr& vvec); - void WriteVerifiedSkContribution(Consensus::LLMQType llmqType, const uint256& quorumHash, const uint256& proTxHash, const CBLSSecretKey& skContribution); - bool GetVerifiedContributions(Consensus::LLMQType llmqType, const uint256& quorumHash, const std::vector& validMembers, std::vector& memberIndexesRet, std::vector& vvecsRet, BLSSecretKeyVector& skContributionsRet); - bool GetVerifiedContribution(Consensus::LLMQType llmqType, const uint256& quorumHash, const uint256& proTxHash, BLSVerificationVectorPtr& vvecRet, CBLSSecretKey& skContributionRet); + void WriteVerifiedVvecContribution(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& proTxHash, const BLSVerificationVectorPtr& vvec); + void WriteVerifiedSkContribution(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& proTxHash, const CBLSSecretKey& skContribution); + bool GetVerifiedContributions(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const std::vector& validMembers, std::vector& memberIndexesRet, std::vector& vvecsRet, BLSSecretKeyVector& skContributionsRet); + bool GetVerifiedContribution(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& proTxHash, BLSVerificationVectorPtr& vvecRet, CBLSSecretKey& skContributionRet); private: void CleanupCache(); diff --git a/src/llmq/quorums_utils.cpp b/src/llmq/quorums_utils.cpp index 338b01bf5..3af8aa853 100644 --- a/src/llmq/quorums_utils.cpp +++ b/src/llmq/quorums_utils.cpp @@ -12,11 +12,11 @@ namespace llmq { -std::vector CLLMQUtils::GetAllQuorumMembers(Consensus::LLMQType llmqType, const uint256& blockHash) +std::vector CLLMQUtils::GetAllQuorumMembers(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum) { auto& params = Params().GetConsensus().llmqs.at(llmqType); - auto allMns = deterministicMNManager->GetListForBlock(blockHash); - auto modifier = ::SerializeHash(std::make_pair((uint8_t)llmqType, blockHash)); + auto allMns = deterministicMNManager->GetListForBlock(pindexQuorum); + auto modifier = ::SerializeHash(std::make_pair((uint8_t) llmqType, pindexQuorum->GetBlockHash())); return allMns.CalculateQuorum(params.size, modifier); } @@ -41,11 +41,11 @@ uint256 CLLMQUtils::BuildSignHash(Consensus::LLMQType llmqType, const uint256& q return h.GetHash(); } -std::set CLLMQUtils::GetQuorumConnections(Consensus::LLMQType llmqType, const uint256& blockHash, const uint256& forMember) +std::set CLLMQUtils::GetQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& forMember) { auto& params = Params().GetConsensus().llmqs.at(llmqType); - auto mns = GetAllQuorumMembers(llmqType, blockHash); + auto mns = GetAllQuorumMembers(llmqType, pindexQuorum); std::set result; for (size_t i = 0; i < mns.size(); i++) { auto& dmn = mns[i]; @@ -73,7 +73,7 @@ std::set CLLMQUtils::GetQuorumConnections(Consensus::LLMQType llmqType, return result; } -std::set CLLMQUtils::CalcDeterministicWatchConnections(Consensus::LLMQType llmqType, const uint256& blockHash, size_t memberCount, size_t connectionCount) +std::set CLLMQUtils::CalcDeterministicWatchConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, size_t memberCount, size_t connectionCount) { static uint256 qwatchConnectionSeed; static std::atomic qwatchConnectionSeedGenerated{false}; @@ -89,7 +89,7 @@ std::set CLLMQUtils::CalcDeterministicWatchConnections(Consensus::LLMQTy std::set result; uint256 rnd = qwatchConnectionSeed; for (size_t i = 0; i < connectionCount; i++) { - rnd = ::SerializeHash(std::make_pair(rnd, std::make_pair((uint8_t)llmqType, blockHash))); + rnd = ::SerializeHash(std::make_pair(rnd, std::make_pair((uint8_t) llmqType, pindexQuorum->GetBlockHash()))); result.emplace(rnd.GetUint64(0) % memberCount); } return result; diff --git a/src/llmq/quorums_utils.h b/src/llmq/quorums_utils.h index a8c5ba792..6b1c3db07 100644 --- a/src/llmq/quorums_utils.h +++ b/src/llmq/quorums_utils.h @@ -19,7 +19,7 @@ class CLLMQUtils { public: // includes members which failed DKG - static std::vector GetAllQuorumMembers(Consensus::LLMQType llmqType, const uint256& blockHash); + static std::vector GetAllQuorumMembers(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum); static uint256 BuildCommitmentHash(uint8_t llmqType, const uint256& blockHash, const std::vector& validMembers, const CBLSPublicKey& pubKey, const uint256& vvecHash); static uint256 BuildSignHash(Consensus::LLMQType llmqType, const uint256& quorumHash, const uint256& id, const uint256& msgHash); @@ -31,8 +31,8 @@ public: return BuildSignHash((Consensus::LLMQType)s.llmqType, s.quorumHash, s.id, s.msgHash); } - static std::set GetQuorumConnections(Consensus::LLMQType llmqType, const uint256& blockHash, const uint256& forMember); - static std::set CalcDeterministicWatchConnections(Consensus::LLMQType llmqType, const uint256& blockHash, size_t memberCount, size_t connectionCount); + static std::set GetQuorumConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, const uint256& forMember); + static std::set CalcDeterministicWatchConnections(Consensus::LLMQType llmqType, const CBlockIndex* pindexQuorum, size_t memberCount, size_t connectionCount); static bool IsQuorumActive(Consensus::LLMQType llmqType, const uint256& quorumHash); diff --git a/src/masternode-meta.cpp b/src/masternode-meta.cpp index 18f07f1be..9ec37b9e2 100644 --- a/src/masternode-meta.cpp +++ b/src/masternode-meta.cpp @@ -118,7 +118,6 @@ std::string CMasternodeMetaMan::ToString() const std::ostringstream info; info << "Masternodes: meta infos object count: " << (int)metaInfos.size() << - ", deterministic masternode count: " << deterministicMNManager->GetListAtChainTip().GetAllMNsCount() << ", nDsqCount: " << (int)nDsqCount; return info.str(); } diff --git a/src/masternode-meta.h b/src/masternode-meta.h index 8ef5c753f..fe4104611 100644 --- a/src/masternode-meta.h +++ b/src/masternode-meta.h @@ -100,7 +100,11 @@ public: std::string strVersion; if(ser_action.ForRead()) { + Clear(); READWRITE(strVersion); + if (strVersion != SERIALIZATION_VERSION_STRING) { + return; + } } else { strVersion = SERIALIZATION_VERSION_STRING; @@ -122,10 +126,6 @@ public: } READWRITE(nDsqCount); - - if(ser_action.ForRead() && (strVersion != SERIALIZATION_VERSION_STRING)) { - Clear(); - } } public: diff --git a/src/masternode-payments.cpp b/src/masternode-payments.cpp index d71b15df7..6e4ede3e4 100644 --- a/src/masternode-payments.cpp +++ b/src/masternode-payments.cpp @@ -271,7 +271,7 @@ std::map GetRequiredPaymentsStrings(int nStartHeight, int nEnd bool doProjection = false; for(int h = nStartHeight; h < nEndHeight; h++) { if (h <= nChainTipHeight) { - auto payee = deterministicMNManager->GetListForBlock(chainActive[h - 1]->GetBlockHash()).GetMNPayee(); + auto payee = deterministicMNManager->GetListForBlock(chainActive[h - 1]).GetMNPayee(); mapPayments.emplace(h, GetRequiredPaymentsString(h, payee)); } else { doProjection = true; @@ -323,13 +323,13 @@ bool CMasternodePayments::GetBlockTxOuts(int nBlockHeight, CAmount blockReward, CAmount masternodeReward = GetMasternodePayment(nBlockHeight, blockReward); - uint256 blockHash; + const CBlockIndex* pindex; { LOCK(cs_main); - blockHash = chainActive[nBlockHeight - 1]->GetBlockHash(); + pindex = chainActive[nBlockHeight - 1]; } uint256 proTxHash; - auto dmnPayee = deterministicMNManager->GetListForBlock(blockHash).GetMNPayee(); + auto dmnPayee = deterministicMNManager->GetListForBlock(pindex).GetMNPayee(); if (!dmnPayee) { return false; } diff --git a/src/rpc/rpcevo.cpp b/src/rpc/rpcevo.cpp index 4c5ed8468..35913e245 100644 --- a/src/rpc/rpcevo.cpp +++ b/src/rpc/rpcevo.cpp @@ -972,7 +972,7 @@ UniValue protx_list(const JSONRPCRequest& request) setOutpts.emplace(outpt); } - CDeterministicMNList mnList = deterministicMNManager->GetListForBlock(chainActive[height]->GetBlockHash()); + CDeterministicMNList mnList = deterministicMNManager->GetListForBlock(chainActive[height]); mnList.ForEachMN(false, [&](const CDeterministicMNCPtr& dmn) { if (setOutpts.count(dmn->collateralOutpoint) || CheckWalletOwnsKey(pwallet, dmn->pdmnState->keyIDOwner) || @@ -997,7 +997,7 @@ UniValue protx_list(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_PARAMETER, "invalid height specified"); } - CDeterministicMNList mnList = deterministicMNManager->GetListForBlock(chainActive[height]->GetBlockHash()); + CDeterministicMNList mnList = deterministicMNManager->GetListForBlock(chainActive[height]); bool onlyValid = type == "valid"; mnList.ForEachMN(onlyValid, [&](const CDeterministicMNCPtr& dmn) { ret.push_back(BuildDMNListEntry(pwallet, dmn, detailed)); diff --git a/src/rpc/rpcquorums.cpp b/src/rpc/rpcquorums.cpp index b9709ef8c..78ae2bb83 100644 --- a/src/rpc/rpcquorums.cpp +++ b/src/rpc/rpcquorums.cpp @@ -80,7 +80,7 @@ UniValue BuildQuorumInfo(const llmq::CQuorumCPtr& quorum, bool includeMembers, b { UniValue ret(UniValue::VOBJ); - ret.push_back(Pair("height", quorum->height)); + ret.push_back(Pair("height", quorum->pindexQuorum->nHeight)); ret.push_back(Pair("type", quorum->params.name)); ret.push_back(Pair("quorumHash", quorum->qc.quorumHash.ToString())); ret.push_back(Pair("minedBlock", quorum->minedBlockHash.ToString())); @@ -222,7 +222,7 @@ UniValue quorum_memberof(const JSONRPCRequest& request) pindexTip = chainActive.Tip(); } - auto mnList = deterministicMNManager->GetListForBlock(pindexTip->GetBlockHash()); + auto mnList = deterministicMNManager->GetListForBlock(pindexTip); auto dmn = mnList.GetMN(protxHash); if (!dmn) { throw JSONRPCError(RPC_INVALID_PARAMETER, "masternode not found"); diff --git a/src/script/script.h b/src/script/script.h index 74cbb2bf1..d0a784504 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -645,6 +645,15 @@ public: // The default std::vector::clear() does not release memory. CScriptBase().swap(*this); } + + template + void Serialize(Stream& s) const { + s << *(CScriptBase*)this; + } + template + void Unserialize(Stream& s) { + s >> *(CScriptBase*)this; + } }; class CReserveScript diff --git a/src/spentindex.h b/src/spentindex.h index a3d84e86d..c79a007a0 100644 --- a/src/spentindex.h +++ b/src/spentindex.h @@ -210,7 +210,7 @@ struct CAddressUnspentValue { template inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(satoshis); - READWRITE(*(CScriptBase*)(&script)); + READWRITE(script); READWRITE(blockHeight); }