Merge #5481: feat: Split quorum contrib data out of evodb

ad35c1ab2f apply suggestions (UdjinM6)
600dcf3dda refactor: make clang-format happy (UdjinM6)
da9b8e038b feat: Split quorum contribution db out of evodb (UdjinM6)

Pull request description:

  ## Issue being fixed or feature implemented
  Quorum data is not stored on-chain, it's a temporary data produced during dkg and should not be a part of evodb.

  ## What was done?
  Use new db in `llmq/` to store quorum data, migrate old data to it.

  ## How Has This Been Tested?
  Run tests, run a node on testnet

  ## Breaking Changes
  n/a

  ## Checklist:
  - [x] I have performed a self-review of my own code
  - [ ] I have commented my code, particularly in hard-to-understand areas
  - [ ] I have added or updated relevant unit/integration/functional/e2e tests
  - [ ] I have made corresponding changes to the documentation
  - [x] I have assigned this pull request to a milestone _(for repository code-owners and collaborators only)_

ACKs for top commit:
  knst:
    utACK ad35c1ab2f
  PastaPastaPasta:
    utACK ad35c1ab2f

Tree-SHA512: b394a53815897f02ec28776c948a32c258ad7f0d9b78fbfd509146a92c8af53e7577d503bc62ae53c40f88aab2c30160fec09f45f0d3447ba64c05c0887676eb

src/llmq/context.cpp

@@ -18,15 +18,20 @@
 #include <llmq/signing_shares.h>
 
 LLMQContext::LLMQContext(CChainState& chainstate, CConnman& connman, CDeterministicMNManager& dmnman, CEvoDB& evo_db,
-                         CMasternodeMetaMan& mn_metaman, CMNHFManager& mnhfman, CSporkManager& sporkman, CTxMemPool& mempool,
+                         CMasternodeMetaMan& mn_metaman, CMNHFManager& mnhfman, CSporkManager& sporkman,
-                         const CActiveMasternodeManager* const mn_activeman, const CMasternodeSync& mn_sync,
+                         CTxMemPool& mempool, const CActiveMasternodeManager* const mn_activeman,
-                         const std::unique_ptr<PeerManager>& peerman, bool unit_tests, bool wipe) :
+                         const CMasternodeSync& mn_sync, const std::unique_ptr<PeerManager>& peerman, bool unit_tests,
+                         bool wipe) :
     is_masternode{mn_activeman != nullptr},
     bls_worker{std::make_shared<CBLSWorker>()},
     dkg_debugman{std::make_unique<llmq::CDKGDebugManager>()},
     quorum_block_processor{std::make_unique<llmq::CQuorumBlockProcessor>(chainstate, dmnman, evo_db, peerman)},
-    qdkgsman{std::make_unique<llmq::CDKGSessionManager>(*bls_worker, chainstate, connman, dmnman, *dkg_debugman, mn_metaman, *quorum_block_processor, mn_activeman, sporkman, peerman, unit_tests, wipe)},
+    qdkgsman{std::make_unique<llmq::CDKGSessionManager>(*bls_worker, chainstate, connman, dmnman, *dkg_debugman,
-    qman{std::make_unique<llmq::CQuorumManager>(*bls_worker, chainstate, connman, dmnman, *qdkgsman, evo_db, *quorum_block_processor, mn_activeman, mn_sync, sporkman)},
+                                                        mn_metaman, *quorum_block_processor, mn_activeman, sporkman,
+                                                        peerman, unit_tests, wipe)},
+    qman{std::make_unique<llmq::CQuorumManager>(*bls_worker, chainstate, connman, dmnman, *qdkgsman, evo_db,
+                                                *quorum_block_processor, mn_activeman, mn_sync, sporkman, unit_tests,
+                                                wipe)},
     sigman{std::make_unique<llmq::CSigningManager>(connman, mn_activeman, chainstate, *qman, peerman, unit_tests, wipe)},
     shareman{std::make_unique<llmq::CSigSharesManager>(connman, *sigman, mn_activeman, *qman, sporkman, peerman)},
     clhandler{[&]() -> llmq::CChainLocksHandler* const {
@@ -39,7 +44,8 @@ LLMQContext::LLMQContext(CChainState& chainstate, CConnman& connman, CDeterminis
         llmq::quorumInstantSendManager = std::make_unique<llmq::CInstantSendManager>(*llmq::chainLocksHandler, chainstate, connman, *qman, *sigman, *shareman, sporkman, mempool, mn_sync, peerman, is_masternode, unit_tests, wipe);
         return llmq::quorumInstantSendManager.get();
     }()},
-    ehfSignalsHandler{std::make_unique<llmq::CEHFSignalsHandler>(chainstate, mnhfman, *sigman, *shareman, mempool, *qman, sporkman, peerman)}
+    ehfSignalsHandler{std::make_unique<llmq::CEHFSignalsHandler>(chainstate, mnhfman, *sigman, *shareman, mempool,
+                                                                 *qman, sporkman, peerman)}
 {
     // NOTE: we use this only to wipe the old db, do NOT use it for anything else
     // TODO: remove it in some future version

src/llmq/quorums.cpp

@@ -164,7 +164,7 @@ int CQuorum::GetMemberIndex(const uint256& proTxHash) const
     return -1;
 }
 
-void CQuorum::WriteContributions(CEvoDB& evoDb) const
+void CQuorum::WriteContributions(CDBWrapper& db) const
 {
     uint256 dbKey = MakeQuorumKey(*this);
 
@@ -175,19 +175,19 @@ void CQuorum::WriteContributions(CEvoDB& evoDb) const
         for (auto& pubkey : *quorumVvec) {
             s << CBLSPublicKeyVersionWrapper(pubkey, false);
         }
-        evoDb.GetRawDB().Write(std::make_pair(DB_QUORUM_QUORUM_VVEC, dbKey), s);
+        db.Write(std::make_pair(DB_QUORUM_QUORUM_VVEC, dbKey), s);
     }
     if (skShare.IsValid()) {
-        evoDb.GetRawDB().Write(std::make_pair(DB_QUORUM_SK_SHARE, dbKey), skShare);
+        db.Write(std::make_pair(DB_QUORUM_SK_SHARE, dbKey), skShare);
     }
 }
 
-bool CQuorum::ReadContributions(CEvoDB& evoDb)
+bool CQuorum::ReadContributions(const CDBWrapper& db)
 {
     uint256 dbKey = MakeQuorumKey(*this);
     CDataStream s(SER_DISK, CLIENT_VERSION);
 
-    if (!evoDb.GetRawDB().ReadDataStream(std::make_pair(DB_QUORUM_QUORUM_VVEC, dbKey), s)) {
+    if (!db.ReadDataStream(std::make_pair(DB_QUORUM_QUORUM_VVEC, dbKey), s)) {
         return false;
     }
 
@@ -203,20 +203,23 @@ bool CQuorum::ReadContributions(CEvoDB& evoDb)
     quorumVvec = std::make_shared<std::vector<CBLSPublicKey>>(std::move(qv));
     // We ignore the return value here as it is ok if this fails. If it fails, it usually means that we are not a
     // member of the quorum but observed the whole DKG process to have the quorum verification vector.
-    evoDb.GetRawDB().Read(std::make_pair(DB_QUORUM_SK_SHARE, dbKey), skShare);
+    db.Read(std::make_pair(DB_QUORUM_SK_SHARE, dbKey), skShare);
 
     return true;
 }
 
-CQuorumManager::CQuorumManager(CBLSWorker& _blsWorker, CChainState& chainstate, CConnman& _connman, CDeterministicMNManager& dmnman,
+CQuorumManager::CQuorumManager(CBLSWorker& _blsWorker, CChainState& chainstate, CConnman& _connman,
-                               CDKGSessionManager& _dkgManager, CEvoDB& _evoDb, CQuorumBlockProcessor& _quorumBlockProcessor,
+                               CDeterministicMNManager& dmnman, CDKGSessionManager& _dkgManager, CEvoDB& _evoDb,
-                               const CActiveMasternodeManager* const mn_activeman, const CMasternodeSync& mn_sync, const CSporkManager& sporkman) :
+                               CQuorumBlockProcessor& _quorumBlockProcessor,
+                               const CActiveMasternodeManager* const mn_activeman, const CMasternodeSync& mn_sync,
+                               const CSporkManager& sporkman, bool unit_tests, bool wipe) :
+    db(std::make_unique<CDBWrapper>(unit_tests ? "" : (gArgs.GetDataDirNet() / "llmq" / "quorumdb"), 1 << 20,
+                                    unit_tests, wipe)),
     blsWorker(_blsWorker),
     m_chainstate(chainstate),
     connman(_connman),
     m_dmnman(dmnman),
     dkgManager(_dkgManager),
-    m_evoDb(_evoDb),
     quorumBlockProcessor(_quorumBlockProcessor),
     m_mn_activeman(mn_activeman),
     m_mn_sync(mn_sync),
@@ -224,6 +227,7 @@ CQuorumManager::CQuorumManager(CBLSWorker& _blsWorker, CChainState& chainstate,
 {
     utils::InitQuorumsCache(mapQuorumsCache, false);
     quorumThreadInterrupt.reset();
+    MigrateOldQuorumDB(_evoDb);
 }
 
 void CQuorumManager::Start()
@@ -404,11 +408,11 @@ CQuorumPtr CQuorumManager::BuildQuorumFromCommitment(const Consensus::LLMQType l
     quorum->Init(std::move(qc), pQuorumBaseBlockIndex, minedBlockHash, members);
 
     bool hasValidVvec = false;
-    if (quorum->ReadContributions(m_evoDb)) {
+    if (WITH_LOCK(cs_db, return quorum->ReadContributions(*db))) {
         hasValidVvec = true;
     } else {
         if (BuildQuorumContributions(quorum->qc, quorum)) {
-            quorum->WriteContributions(m_evoDb);
+            WITH_LOCK(cs_db, quorum->WriteContributions(*db));
             hasValidVvec = true;
         } else {
             LogPrint(BCLog::LLMQ, "CQuorumManager::%s -- llmqType[%d] quorumIndex[%d] quorum.ReadContributions and BuildQuorumContributions for quorumHash[%s] failed\n", __func__, ToUnderlying(llmqType), quorum->qc->quorumIndex, quorum->qc->quorumHash.ToString());
@@ -846,7 +850,7 @@ PeerMsgRet CQuorumManager::ProcessMessage(CNode& pfrom, const std::string& msg_t
                 return errorHandler("Invalid secret key share received");
             }
         }
-        pQuorum->WriteContributions(m_evoDb);
+        WITH_LOCK(cs_db, pQuorum->WriteContributions(*db));
         return {};
     }
     return {};
@@ -1100,13 +1104,76 @@ void CQuorumManager::StartCleanupOldQuorumDataThread(const CBlockIndex* pIndex)
         }
 
         if (!quorumThreadInterrupt) {
-            DataCleanupHelper(m_evoDb.GetRawDB(), dbKeysToSkip);
+            WITH_LOCK(cs_db, DataCleanupHelper(*db, dbKeysToSkip));
         }
 
         LogPrint(BCLog::LLMQ, "CQuorumManager::StartCleanupOldQuorumDataThread -- done. time=%d\n", t.count());
     });
 }
 
+// TODO: remove in v23
+void CQuorumManager::MigrateOldQuorumDB(CEvoDB& evoDb) const
+{
+    LOCK(cs_db);
+    if (!db->IsEmpty()) return;
+
+    const auto prefixes = {DB_QUORUM_QUORUM_VVEC, DB_QUORUM_SK_SHARE};
+
+    LogPrint(BCLog::LLMQ, "CQuorumManager::%s -- start\n", __func__);
+
+    CDBBatch batch(*db);
+    std::unique_ptr<CDBIterator> pcursor(evoDb.GetRawDB().NewIterator());
+
+    for (const auto& prefix : prefixes) {
+        auto start = std::make_tuple(prefix, uint256());
+        pcursor->Seek(start);
+
+        int count{0};
+        while (pcursor->Valid()) {
+            decltype(start) k;
+            CDataStream s(SER_DISK, CLIENT_VERSION);
+            CBLSSecretKey sk;
+
+            if (!pcursor->GetKey(k) || std::get<0>(k) != prefix) {
+                break;
+            }
+
+            if (prefix == DB_QUORUM_QUORUM_VVEC) {
+                if (!evoDb.GetRawDB().ReadDataStream(k, s)) {
+                    break;
+                }
+                batch.Write(k, s);
+            }
+            if (prefix == DB_QUORUM_SK_SHARE) {
+                if (!pcursor->GetValue(sk)) {
+                    break;
+                }
+                batch.Write(k, sk);
+            }
+
+            if (batch.SizeEstimate() >= (1 << 24)) {
+                db->WriteBatch(batch);
+                batch.Clear();
+            }
+
+            ++count;
+            pcursor->Next();
+        }
+
+        db->WriteBatch(batch);
+
+        LogPrint(BCLog::LLMQ, "CQuorumManager::%s -- %s moved %d\n", __func__, prefix, count);
+    }
+
+    pcursor.reset();
+    db->CompactFull();
+
+    DataCleanupHelper(evoDb.GetRawDB(), {});
+    evoDb.CommitRootTransaction();
+
+    LogPrint(BCLog::LLMQ, "CQuorumManager::%s -- done\n", __func__);
+}
+
 CQuorumCPtr SelectQuorumForSigning(const Consensus::LLMQParams& llmq_params, const CChain& active_chain, const CQuorumManager& qman,
                                    const uint256& selectionHash, int signHeight, int signOffset)
 {

src/llmq/quorums.h

@@ -217,8 +217,8 @@ public:
 
 private:
     bool HasVerificationVectorInternal() const EXCLUSIVE_LOCKS_REQUIRED(cs_vvec_shShare);
-    void WriteContributions(CEvoDB& evoDb) const;
+    void WriteContributions(CDBWrapper& db) const;
-    bool ReadContributions(CEvoDB& evoDb);
+    bool ReadContributions(const CDBWrapper& db);
 };
 
 /**
@@ -230,12 +230,14 @@ private:
 class CQuorumManager
 {
 private:
+    mutable Mutex cs_db;
+    std::unique_ptr<CDBWrapper> db GUARDED_BY(cs_db){nullptr};
+
     CBLSWorker& blsWorker;
     CChainState& m_chainstate;
     CConnman& connman;
     CDeterministicMNManager& m_dmnman;
     CDKGSessionManager& dkgManager;
-    CEvoDB& m_evoDb;
     CQuorumBlockProcessor& quorumBlockProcessor;
     const CActiveMasternodeManager* const m_mn_activeman;
     const CMasternodeSync& m_mn_sync;
@@ -254,7 +256,8 @@ private:
 public:
     CQuorumManager(CBLSWorker& _blsWorker, CChainState& chainstate, CConnman& _connman, CDeterministicMNManager& dmnman,
                    CDKGSessionManager& _dkgManager, CEvoDB& _evoDb, CQuorumBlockProcessor& _quorumBlockProcessor,
-                   const CActiveMasternodeManager* const mn_activeman, const CMasternodeSync& mn_sync, const CSporkManager& sporkman);
+                   const CActiveMasternodeManager* const mn_activeman, const CMasternodeSync& mn_sync,
+                   const CSporkManager& sporkman, bool unit_tests, bool wipe);
     ~CQuorumManager() { Stop(); };
 
     void Start();
@@ -294,6 +297,7 @@ private:
     void StartQuorumDataRecoveryThread(const CQuorumCPtr pQuorum, const CBlockIndex* pIndex, uint16_t nDataMask) const;
 
     void StartCleanupOldQuorumDataThread(const CBlockIndex* pIndex) const;
+    void MigrateOldQuorumDB(CEvoDB& evoDb) const;
 };
 
 // when selecting a quorum for signing and verification, we use CQuorumManager::SelectQuorum with this offset as