Merge pull request #2746 from codablock/pr_v13_backport

[0.13.x] Backport multiple PRs from develop

share/qt/Info.plist.in

@@ -17,7 +17,7 @@
   <string>APPL</string>
 
   <key>CFBundleGetInfoString</key>
-  <string>@CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_REVISION@, Copyright © 2009-@COPYRIGHT_YEAR@ The Bitcoin Core developers, 2014-@COPYRIGHT_YEAR@ @COPYRIGHT_HOLDERS_FINAL@</string>
+  <string>@CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_REVISION@.@CLIENT_VERSION_BUILD@, Copyright © 2009-@COPYRIGHT_YEAR@ The Bitcoin Core developers, 2014-@COPYRIGHT_YEAR@ @COPYRIGHT_HOLDERS_FINAL@</string>
 
   <key>CFBundleShortVersionString</key>
   <string>@CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_REVISION@</string>

src/dbwrapper.h

@@ -344,14 +344,16 @@ public:
 
 };
 
+template<typename Parent, typename CommitTarget>
 class CDBTransaction {
-private:
+protected:
-    CDBWrapper &db;
+    Parent &parent;
+    CommitTarget &commitTarget;
 
     struct KeyHolder {
         virtual ~KeyHolder() = default;
         virtual bool Less(const KeyHolder &b) const = 0;
-        virtual void Erase(CDBBatch &batch) = 0;
+        virtual void Erase(CommitTarget &commitTarget) = 0;
     };
     typedef std::unique_ptr<KeyHolder> KeyHolderPtr;
 
@@ -364,15 +366,15 @@ private:
             auto *b2 = dynamic_cast<const KeyHolderImpl<K>*>(&b);
             return key < b2->key;
         }
-        virtual void Erase(CDBBatch &batch) {
+        virtual void Erase(CommitTarget &commitTarget) {
-            batch.Erase(key);
+            commitTarget.Erase(key);
         }
         K key;
     };
 
     struct KeyValueHolder {
         virtual ~KeyValueHolder() = default;
-        virtual void Write(CDBBatch &batch) = 0;
+        virtual void Write(CommitTarget &parent) = 0;
     };
     typedef std::unique_ptr<KeyValueHolder> KeyValueHolderPtr;
 
@@ -381,8 +383,13 @@ private:
         KeyValueHolderImpl(const KeyHolderImpl<K> &_key, const V &_value)
                 : key(_key),
                   value(_value) { }
-        virtual void Write(CDBBatch &batch) {
+        KeyValueHolderImpl(const KeyHolderImpl<K> &_key, V &&_value)
-            batch.Write(key.key, value);
+                : key(_key),
+                  value(std::forward<V>(_value)) { }
+        virtual void Write(CommitTarget &commitTarget) {
+            // we're moving the value instead of copying it. This means that Write() can only be called once per
+            // KeyValueHolderImpl instance. Commit() clears the write maps, so this ok.
+            commitTarget.Write(key.key, std::move(value));
         }
         const KeyHolderImpl<K> &key;
         V value;
@@ -422,22 +429,34 @@ private:
         return getMapForType<K>(deletes, create);
     }
 
-public:
+    template <typename K, typename KV>
-    CDBTransaction(CDBWrapper &_db) : db(_db) {}
+    void writeImpl(KeyHolderImpl<K>* k, KV&& kv) {
-
+        auto k2 = KeyHolderPtr(k);
-    template <typename K, typename V>
-    void Write(const K& key, const V& value) {
-        KeyHolderPtr k(new KeyHolderImpl<K>(key));
-        KeyHolderImpl<K>* k2 = dynamic_cast<KeyHolderImpl<K>*>(k.get());
-        KeyValueHolderPtr kv(new KeyValueHolderImpl<K,V>(*k2, value));
 
         KeyValueMap *ds = getDeletesMap<K>(false);
         if (ds)
-            ds->erase(k);
+            ds->erase(k2);
 
         KeyValueMap *ws = getWritesMap<K>(true);
-        ws->erase(k);
+        ws->erase(k2);
-        ws->emplace(std::make_pair(std::move(k), std::move(kv)));
+        ws->emplace(std::make_pair(std::move(k2), std::forward<KV>(kv)));
+    }
+
+public:
+    CDBTransaction(Parent &_parent, CommitTarget &_commitTarget) : parent(_parent), commitTarget(_commitTarget) {}
+
+    template <typename K, typename V>
+    void Write(const K& key, const V& v) {
+        auto k = new KeyHolderImpl<K>(key);
+        auto kv = std::make_unique<KeyValueHolderImpl<K, V>>(*k, v);
+        writeImpl(k, std::move(kv));
+    }
+
+    template <typename K, typename V>
+    void Write(const K& key, V&& v) {
+        auto k = new KeyHolderImpl<K>(key);
+        auto kv = std::make_unique<KeyValueHolderImpl<K, typename std::remove_reference<V>::type>>(*k, std::forward<V>(v));
+        writeImpl(k, std::move(kv));
     }
 
     template <typename K, typename V>
@@ -450,7 +469,7 @@ public:
 
         KeyValueMap *ws = getWritesMap<K>(false);
         if (ws) {
-            KeyValueMap::iterator it = ws->find(k);
+            auto it = ws->find(k);
             if (it != ws->end()) {
                 auto *impl = dynamic_cast<KeyValueHolderImpl<K, V> *>(it->second.get());
                 if (!impl)
@@ -460,7 +479,7 @@ public:
             }
         }
 
-        return db.Read(key, value);
+        return parent.Read(key, value);
     }
 
     template <typename K>
@@ -475,7 +494,7 @@ public:
         if (ws && ws->count(k))
             return true;
 
-        return db.Exists(key);
+        return parent.Exists(key);
     }
 
     template <typename K>
@@ -494,21 +513,18 @@ public:
         deletes.clear();
     }
 
-    bool Commit() {
+    void Commit() {
-        CDBBatch batch(db);
         for (auto &p : deletes) {
             for (auto &p2 : p.second) {
-                p2.first->Erase(batch);
+                p2.first->Erase(commitTarget);
             }
         }
         for (auto &p : writes) {
             for (auto &p2 : p.second) {
-                p2.second->Write(batch);
+                p2.second->Write(commitTarget);
             }
         }
-        bool ret = db.WriteBatch(batch, true);
         Clear();
-        return ret;
     }
 
     bool IsClean() {
@@ -516,26 +532,29 @@ public:
     }
 };
 
+template<typename Parent, typename CommitTarget>
 class CScopedDBTransaction {
+public:
+    typedef CDBTransaction<Parent, CommitTarget> Transaction;
+
 private:
-    CDBTransaction &dbTransaction;
+    Transaction &dbTransaction;
     std::function<void ()> commitHandler;
     std::function<void ()> rollbackHandler;
     bool didCommitOrRollback{};
 
 public:
-    CScopedDBTransaction(CDBTransaction &dbTx) : dbTransaction(dbTx) {}
+    CScopedDBTransaction(Transaction &dbTx) : dbTransaction(dbTx) {}
     ~CScopedDBTransaction() {
         if (!didCommitOrRollback)
             Rollback();
     }
-    bool Commit() {
+    void Commit() {
         assert(!didCommitOrRollback);
         didCommitOrRollback = true;
-        bool result = dbTransaction.Commit();
+        dbTransaction.Commit();
         if (commitHandler)
             commitHandler();
-        return result;
     }
     void Rollback() {
         assert(!didCommitOrRollback);
@@ -545,9 +564,9 @@ public:
             rollbackHandler();
     }
 
-    static std::unique_ptr<CScopedDBTransaction> Begin(CDBTransaction &dbTx) {
+    static std::unique_ptr<CScopedDBTransaction<Parent, CommitTarget>> Begin(Transaction &dbTx) {
         assert(dbTx.IsClean());
-        return std::unique_ptr<CScopedDBTransaction>(new CScopedDBTransaction(dbTx));
+        return std::make_unique<CScopedDBTransaction<Parent, CommitTarget>>(dbTx);
     }
 
     void SetCommitHandler(const std::function<void ()> &h) {

src/evo/deterministicmns.cpp

@@ -83,6 +83,15 @@ void CDeterministicMN::ToJson(UniValue& obj) const
     obj.push_back(Pair("proTxHash", proTxHash.ToString()));
     obj.push_back(Pair("collateralHash", collateralOutpoint.hash.ToString()));
     obj.push_back(Pair("collateralIndex", (int)collateralOutpoint.n));
+
+    Coin coin;
+    if (GetUTXOCoin(collateralOutpoint, coin)) {
+        CTxDestination dest;
+        if (ExtractDestination(coin.out.scriptPubKey, dest)) {
+            obj.push_back(Pair("collateralAddress", CBitcoinAddress(dest).ToString()));
+        }
+    }
+
     obj.push_back(Pair("operatorReward", (double)nOperatorReward / 100));
     obj.push_back(Pair("state", stateObj));
 }
@@ -438,6 +447,13 @@ CDeterministicMNManager::CDeterministicMNManager(CEvoDB& _evoDb) :
 
 bool CDeterministicMNManager::ProcessBlock(const CBlock& block, const CBlockIndex* pindex, CValidationState& _state)
 {
+    AssertLockHeld(cs_main);
+
+    bool fDIP0003Active = VersionBitsState(pindex->pprev, Params().GetConsensus(), Consensus::DEPLOYMENT_DIP0003, versionbitscache) == THRESHOLD_ACTIVE;
+    if (!fDIP0003Active) {
+        return true;
+    }
+
     LOCK(cs);
 
     int nHeight = pindex->nHeight;
@@ -457,7 +473,7 @@ bool CDeterministicMNManager::ProcessBlock(const CBlock& block, const CBlockInde
     CDeterministicMNListDiff diff = oldList.BuildDiff(newList);
 
     evoDb.Write(std::make_pair(DB_LIST_DIFF, diff.blockHash), diff);
-    if ((nHeight % SNAPSHOT_LIST_PERIOD) == 0) {
+    if ((nHeight % SNAPSHOT_LIST_PERIOD) == 0 || oldList.GetHeight() == -1) {
         evoDb.Write(std::make_pair(DB_LIST_SNAPSHOT, diff.blockHash), newList);
         LogPrintf("CDeterministicMNManager::%s -- Wrote snapshot. nHeight=%d, mapCurMNs.allMNsCount=%d\n",
             __func__, nHeight, newList.GetAllMNsCount());

src/evo/evodb.cpp

@@ -8,10 +8,21 @@ CEvoDB* evoDb;
 
 CEvoDB::CEvoDB(size_t nCacheSize, bool fMemory, bool fWipe) :
     db(fMemory ? "" : (GetDataDir() / "evodb"), nCacheSize, fMemory, fWipe),
-    dbTransaction(db)
+    rootBatch(db),
+    rootDBTransaction(db, rootBatch),
+    curDBTransaction(rootDBTransaction, rootDBTransaction)
 {
 }
 
+bool CEvoDB::CommitRootTransaction()
+{
+    assert(curDBTransaction.IsClean());
+    rootDBTransaction.Commit();
+    bool ret = db.WriteBatch(rootBatch);
+    rootBatch.Clear();
+    return ret;
+}
+
 bool CEvoDB::VerifyBestBlock(const uint256& hash)
 {
     // Make sure evodb is consistent.

src/evo/evodb.h

@@ -16,15 +16,22 @@ class CEvoDB
 private:
     CCriticalSection cs;
     CDBWrapper db;
-    CDBTransaction dbTransaction;
+
+    typedef CDBTransaction<CDBWrapper, CDBBatch> RootTransaction;
+    typedef CDBTransaction<RootTransaction, RootTransaction> CurTransaction;
+    typedef CScopedDBTransaction<RootTransaction, RootTransaction> ScopedTransaction;
+
+    CDBBatch rootBatch;
+    RootTransaction rootDBTransaction;
+    CurTransaction curDBTransaction;
 
 public:
     CEvoDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false);
 
-    std::unique_ptr<CScopedDBTransaction> BeginTransaction()
+    std::unique_ptr<ScopedTransaction> BeginTransaction()
     {
         LOCK(cs);
-        auto t = CScopedDBTransaction::Begin(dbTransaction);
+        auto t = ScopedTransaction::Begin(curDBTransaction);
         return t;
     }
 
@@ -32,28 +39,28 @@ public:
     bool Read(const K& key, V& value)
     {
         LOCK(cs);
-        return dbTransaction.Read(key, value);
+        return curDBTransaction.Read(key, value);
     }
 
     template <typename K, typename V>
     void Write(const K& key, const V& value)
     {
         LOCK(cs);
-        dbTransaction.Write(key, value);
+        curDBTransaction.Write(key, value);
     }
 
     template <typename K>
     bool Exists(const K& key)
     {
         LOCK(cs);
-        return dbTransaction.Exists(key);
+        return curDBTransaction.Exists(key);
     }
 
     template <typename K>
     void Erase(const K& key)
     {
         LOCK(cs);
-        dbTransaction.Erase(key);
+        curDBTransaction.Erase(key);
     }
 
     CDBWrapper& GetRawDB()
@@ -61,6 +68,8 @@ public:
         return db;
     }
 
+    bool CommitRootTransaction();
+
     bool VerifyBestBlock(const uint256& hash);
     void WriteBestBlock(const uint256& hash);
 };

src/evo/simplifiedmns.cpp

@@ -119,25 +119,27 @@ bool BuildSimplifiedMNListDiff(const uint256& baseBlockHash, const uint256& bloc
     AssertLockHeld(cs_main);
     mnListDiffRet = CSimplifiedMNListDiff();
 
-    BlockMap::iterator baseBlockIt = mapBlockIndex.begin();
+    const CBlockIndex* baseBlockIndex = chainActive.Genesis();
     if (!baseBlockHash.IsNull()) {
-        baseBlockIt = mapBlockIndex.find(baseBlockHash);
+        auto it = mapBlockIndex.find(baseBlockHash);
+        if (it == mapBlockIndex.end()) {
+            errorRet = strprintf("block %s not found", baseBlockHash.ToString());
+            return false;
+        }
+        baseBlockIndex = it->second;
     }
     auto blockIt = mapBlockIndex.find(blockHash);
-    if (baseBlockIt == mapBlockIndex.end()) {
-        errorRet = strprintf("block %s not found", baseBlockHash.ToString());
-        return false;
-    }
     if (blockIt == mapBlockIndex.end()) {
         errorRet = strprintf("block %s not found", blockHash.ToString());
         return false;
     }
+    const CBlockIndex* blockIndex = blockIt->second;
 
-    if (!chainActive.Contains(baseBlockIt->second) || !chainActive.Contains(blockIt->second)) {
+    if (!chainActive.Contains(baseBlockIndex) || !chainActive.Contains(blockIndex)) {
         errorRet = strprintf("block %s and %s are not in the same chain", baseBlockHash.ToString(), blockHash.ToString());
         return false;
     }
-    if (baseBlockIt->second->nHeight > blockIt->second->nHeight) {
+    if (baseBlockIndex->nHeight > blockIndex->nHeight) {
         errorRet = strprintf("base block %s is higher then block %s", baseBlockHash.ToString(), blockHash.ToString());
         return false;
     }
@@ -150,7 +152,7 @@ bool BuildSimplifiedMNListDiff(const uint256& baseBlockHash, const uint256& bloc
 
     // TODO store coinbase TX in CBlockIndex
     CBlock block;
-    if (!ReadBlockFromDisk(block, blockIt->second, Params().GetConsensus())) {
+    if (!ReadBlockFromDisk(block, blockIndex, Params().GetConsensus())) {
         errorRet = strprintf("failed to read block %s from disk", blockHash.ToString());
         return false;
     }

src/init.cpp

@@ -256,6 +256,10 @@ void PrepareShutdown()
     MapPort(false);
     UnregisterValidationInterface(peerLogic.get());
     peerLogic.reset();
+    if (g_connman) {
+        // make sure to stop all threads before g_connman is reset to nullptr as these threads might still be accessing it
+        g_connman->Stop();
+    }
     g_connman.reset();
 
     if (!fLiteMode && !fRPCInWarmup) {
@@ -1963,7 +1967,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler)
                 return InitError(_("Invalid masternodeblsprivkey. Please see documenation."));
             }
         } else {
-            InitWarning(_("You should specify a masternodeblsprivkey in the configuration. Please see documentation for help."));
+            return InitError(_("You must specify a masternodeblsprivkey in the configuration. Please see documentation for help."));
         }
 
         // init and register activeMasternodeManager

src/merkleblock.cpp

@@ -19,10 +19,24 @@ CMerkleBlock::CMerkleBlock(const CBlock& block, CBloomFilter& filter)
     vMatch.reserve(block.vtx.size());
     vHashes.reserve(block.vtx.size());
 
+    const static std::set<int> allowedTxTypes = {
+            TRANSACTION_NORMAL,
+            TRANSACTION_PROVIDER_REGISTER,
+            TRANSACTION_PROVIDER_UPDATE_SERVICE,
+            TRANSACTION_PROVIDER_UPDATE_REGISTRAR,
+            TRANSACTION_PROVIDER_UPDATE_REVOKE,
+            TRANSACTION_COINBASE,
+    };
+
     for (unsigned int i = 0; i < block.vtx.size(); i++)
     {
-        const uint256& hash = block.vtx[i]->GetHash();
+        const auto& tx = *block.vtx[i];
-        if (filter.IsRelevantAndUpdate(*block.vtx[i]))
+        if (tx.nVersion == 3 && !allowedTxTypes.count(tx.nType)) {
+            continue;
+        }
+
+        const uint256& hash = tx.GetHash();
+        if (filter.IsRelevantAndUpdate(tx))
         {
             vMatch.push_back(true);
             vMatchedTxn.push_back(std::make_pair(i, hash));

src/rpc/masternode.cpp

@@ -890,17 +890,26 @@ UniValue masternodelist(const JSONRPCRequest& request)
             std::string strOutpoint = mnpair.first.ToStringShort();
 
             CScript payeeScript;
+            std::string collateralAddressStr = "UNKNOWN";
             if (deterministicMNManager->IsDeterministicMNsSporkActive()) {
                 auto dmn = deterministicMNManager->GetListAtChainTip().GetMNByCollateral(mn.outpoint);
                 if (dmn) {
                     payeeScript = dmn->pdmnState->scriptPayout;
+                    Coin coin;
+                    if (GetUTXOCoin(dmn->collateralOutpoint, coin)) {
+                        CTxDestination collateralDest;
+                        if (ExtractDestination(coin.out.scriptPubKey, collateralDest)) {
+                            collateralAddressStr = CBitcoinAddress(collateralDest).ToString();
+                        }
+                    }
                 }
             } else {
                 payeeScript = GetScriptForDestination(mn.keyIDCollateralAddress);
+                collateralAddressStr = CBitcoinAddress(mn.keyIDCollateralAddress).ToString();
             }
 
             CTxDestination payeeDest;
-            std::string payeeStr = "UNKOWN";
+            std::string payeeStr = "UNKNOWN";
             if (ExtractDestination(payeeScript, payeeDest)) {
                 payeeStr = CBitcoinAddress(payeeDest).ToString();
             }
@@ -965,7 +974,8 @@ UniValue masternodelist(const JSONRPCRequest& request)
                                (int64_t)mn.lastPing.sigTime << " " <<
                                (int64_t)(mn.lastPing.sigTime - mn.sigTime) << " " <<
                                mn.GetLastPaidTime() << " " <<
-                               mn.GetLastPaidBlock();
+                               mn.GetLastPaidBlock() << " " <<
+                               collateralAddressStr;
                 std::string strInfo = streamInfo.str();
                 if (strFilter !="" && strInfo.find(strFilter) == std::string::npos &&
                     strOutpoint.find(strFilter) == std::string::npos) continue;
@@ -983,6 +993,7 @@ UniValue masternodelist(const JSONRPCRequest& request)
                 objMN.push_back(Pair("lastpaidblock", mn.GetLastPaidBlock()));
                 objMN.push_back(Pair("owneraddress", CBitcoinAddress(mn.keyIDOwner).ToString()));
                 objMN.push_back(Pair("votingaddress", CBitcoinAddress(mn.keyIDVoting).ToString()));
+                objMN.push_back(Pair("collateraladdress", collateralAddressStr));
                 obj.push_back(Pair(strOutpoint, objMN));
             } else if (strMode == "keyid") {
                 if (strFilter !="" && strOutpoint.find(strFilter) == std::string::npos) continue;

src/rpc/rpcevo.cpp

@@ -693,7 +693,7 @@ UniValue protx_update_registrar(const JSONRPCRequest& request)
         ptx.pubKeyOperator = ParseBLSPubKey(request.params[2].get_str(), "operator BLS address");
     }
     if (request.params[3].get_str() != "") {
-        ptx.keyIDVoting = ParsePubKeyIDFromAddress(request.params[3].get_str(), "operator address");
+        ptx.keyIDVoting = ParsePubKeyIDFromAddress(request.params[3].get_str(), "voting address");
     }
 
     CBitcoinAddress payoutAddress(request.params[4].get_str());

src/validation.cpp

@@ -2414,6 +2414,9 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n
         // Flush the chainstate (which may refer to block index entries).
         if (!pcoinsTip->Flush())
             return AbortNode(state, "Failed to write to coin database");
+        if (!evoDb->CommitRootTransaction()) {
+            return AbortNode(state, "Failed to commit EvoDB");
+        }
         nLastFlush = nNow;
     }
     if (fDoFullFlush || ((mode == FLUSH_STATE_ALWAYS || mode == FLUSH_STATE_PERIODIC) && nNow > nLastSetChain + (int64_t)DATABASE_WRITE_INTERVAL * 1000000)) {
@@ -2519,12 +2522,11 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara
             return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString());
         bool flushed = view.Flush();
         assert(flushed);
-        bool committed = dbTx->Commit();
+        dbTx->Commit();
-        assert(committed);
     }
     LogPrint("bench", "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001);
     // Write the chain state to disk, if necessary.
-    if (!FlushStateToDisk(state, IsInitialBlockDownload() ? FLUSH_STATE_IF_NEEDED : FLUSH_STATE_ALWAYS))
+    if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED))
         return false;
     // Resurrect mempool transactions from the disconnected block.
     std::vector<uint256> vHashUpdate;
@@ -2609,13 +2611,12 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams,
         LogPrint("bench", "  - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001);
         bool flushed = view.Flush();
         assert(flushed);
-        bool committed = dbTx->Commit();
+        dbTx->Commit();
-        assert(committed);
     }
     int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3;
     LogPrint("bench", "  - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001);
     // Write the chain state to disk, if necessary.
-    if (!FlushStateToDisk(state, IsInitialBlockDownload() ? FLUSH_STATE_IF_NEEDED : FLUSH_STATE_ALWAYS))
+    if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED))
         return false;
     int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4;
     LogPrint("bench", "  - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001);