Fix sync issues (#1225)

* Store time we saw mnb last time, bump sync timeout
if we received seen mnb but we are too close to MASTERNODE_NEW_START_REQUIRED_SECONDS

* Reset blockchain sync status if new blocks were accepted during sync

* Add some debug log output

* wait for at least one new block to be accepted

* bump CGovernanceManager-Version

src/governance.cpp

@@ -20,7 +20,7 @@ std::map<uint256, int64_t> mapAskedForGovernanceObject;
 
 int nSubmittedFinalBudget;
 
-const std::string CGovernanceManager::SERIALIZATION_VERSION_STRING = "CGovernanceManager-Version-6";
+const std::string CGovernanceManager::SERIALIZATION_VERSION_STRING = "CGovernanceManager-Version-7";
 
 CGovernanceManager::CGovernanceManager()
     : pCurrentBlockIndex(NULL),

src/main.cpp

@@ -4013,6 +4013,8 @@ bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, c
     if (!ActivateBestChain(state, chainparams, pblock))
         return error("%s: ActivateBestChain failed", __func__);
 
+    masternodeSync.IsBlockchainSynced(true);
+
     LogPrintf("%s : ACCEPTED\n", __func__);
     return true;
 }
@@ -5145,7 +5147,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam
                     if(mnodeman.mapSeenMasternodeBroadcast.count(inv.hash)){
                         CDataStream ss(SER_NETWORK, PROTOCOL_VERSION);
                         ss.reserve(1000);
-                        ss << mnodeman.mapSeenMasternodeBroadcast[inv.hash];
+                        ss << mnodeman.mapSeenMasternodeBroadcast[inv.hash].second;
                         // backward compatibility patch
                         if(pfrom->nVersion < 70204) {
                             ss << (int64_t)0;

src/masternode-sync.cpp

@@ -17,23 +17,52 @@
 class CMasternodeSync;
 CMasternodeSync masternodeSync;
 
-bool CMasternodeSync::IsBlockchainSynced()
+bool CMasternodeSync::IsBlockchainSynced(bool fBlockAccepted)
 {
     static bool fBlockchainSynced = false;
     static int64_t nTimeLastProcess = GetTime();
+    static int nSkipped = 0;
+    static bool fFirstBlockAccepted = false;
 
     // if the last call to this function was more than 60 minutes ago (client was in sleep mode) reset the sync process
     if(GetTime() - nTimeLastProcess > 60*60) {
         Reset();
         fBlockchainSynced = false;
     }
+
+    if(!pCurrentBlockIndex || !pindexBestHeader || fImporting || fReindex) return false;
+
+    if(fBlockAccepted) {
+        // this should be only triggered while we are still syncing
+        if(!IsSynced()) {
+            // we are trying to download smth, reset blockchain sync status
+            if(fDebug) LogPrintf("CMasternodeSync::IsBlockchainSynced -- reset\n");
+            fFirstBlockAccepted = true;
+            fBlockchainSynced = false;
             nTimeLastProcess = GetTime();
+            return false;
+        }
+    } else {
+        // skip if we already checked less than 1 tick ago
+        if(GetTime() - nTimeLastProcess < MASTERNODE_SYNC_TICK_SECONDS) {
+            nSkipped++;
+            return fBlockchainSynced;
+        }
+    }
+
+    if(fDebug) LogPrintf("CMasternodeSync::IsBlockchainSynced -- state before check: %ssynced, skipped %d times\n", fBlockchainSynced ? "" : "not ", nSkipped);
+
+    nTimeLastProcess = GetTime();
+    nSkipped = 0;
 
     if(fBlockchainSynced) return true;
-    if(!pCurrentBlockIndex || !pindexBestHeader || fImporting || fReindex) return false;
+
     if(fCheckpointsEnabled && pCurrentBlockIndex->nHeight < Checkpoints::GetTotalBlocksEstimate(Params().Checkpoints()))
         return false;
 
+    // wait for at least one new block to be accepted
+    if(!fFirstBlockAccepted) return false;
+
     // same as !IsInitialBlockDownload() but no cs_main needed here
     int64_t nMaxBlockTime = std::max(pCurrentBlockIndex->GetBlockTime(), pindexBestHeader->GetBlockTime());
     fBlockchainSynced = pindexBestHeader->nHeight - pCurrentBlockIndex->nHeight < 24 * 6 &&
@@ -172,7 +201,7 @@ void ReleaseNodes(const std::vector<CNode*> &vNodesCopy)
 void CMasternodeSync::ProcessTick()
 {
     static int nTick = 0;
-    if(nTick++ % 6 != 0) return;
+    if(nTick++ % MASTERNODE_SYNC_TICK_SECONDS != 0) return;
     if(!pCurrentBlockIndex) return;
 
     //the actual count of masternodes we have currently
@@ -214,6 +243,9 @@ void CMasternodeSync::ProcessTick()
             !IsBlockchainSynced() && nRequestedMasternodeAssets > MASTERNODE_SYNC_SPORKS)
     {
         LogPrintf("CMasternodeSync::ProcessTick -- nTick %d nRequestedMasternodeAssets %d nRequestedMasternodeAttempt %d -- blockchain is not synced yet\n", nTick, nRequestedMasternodeAssets, nRequestedMasternodeAttempt);
+        nTimeLastMasternodeList = GetTime();
+        nTimeLastPaymentVote = GetTime();
+        nTimeLastGovernanceItem = GetTime();
         return;
     }
 

src/masternode-sync.h

@@ -21,6 +21,7 @@ static const int MASTERNODE_SYNC_GOVOBJ          = 10;
 static const int MASTERNODE_SYNC_GOVOBJ_VOTE     = 11;
 static const int MASTERNODE_SYNC_FINISHED        = 999;
 
+static const int MASTERNODE_SYNC_TICK_SECONDS    = 6;
 static const int MASTERNODE_SYNC_TIMEOUT_SECONDS = 30; // our blocks are 2.5 minutes so 30 seconds should be fine
 
 extern CMasternodeSync masternodeSync;
@@ -64,7 +65,7 @@ public:
     void AddedGovernanceItem() { nTimeLastGovernanceItem = GetTime(); };
 
     bool IsFailed() { return nRequestedMasternodeAssets == MASTERNODE_SYNC_FAILED; }
-    bool IsBlockchainSynced();
+    bool IsBlockchainSynced(bool fBlockAccepted = false);
     bool IsMasternodeListSynced() { return nRequestedMasternodeAssets > MASTERNODE_SYNC_LIST; }
     bool IsWinnersListSynced() { return nRequestedMasternodeAssets > MASTERNODE_SYNC_MNW; }
     bool IsSynced() { return nRequestedMasternodeAssets == MASTERNODE_SYNC_FINISHED; }

src/masternode.cpp

@@ -911,7 +911,7 @@ bool CMasternodePing::CheckAndUpdate(CMasternode* pmn, int& nDos)
     CMasternodeBroadcast mnb(*pmn);
     uint256 hash = mnb.GetHash();
     if (mnodeman.mapSeenMasternodeBroadcast.count(hash)) {
-        mnodeman.mapSeenMasternodeBroadcast[hash].lastPing = *this;
+        mnodeman.mapSeenMasternodeBroadcast[hash].second.lastPing = *this;
     }
 
     pmn->Check(true); // force update, ignoring cache

src/masternodeman.cpp

@@ -15,7 +15,7 @@
 /** Masternode manager */
 CMasternodeMan mnodeman;
 
-const std::string CMasternodeMan::SERIALIZATION_VERSION_STRING = "CMasternodeMan-Version-2";
+const std::string CMasternodeMan::SERIALIZATION_VERSION_STRING = "CMasternodeMan-Version-3";
 
 struct CompareLastPaidBlock
 {
@@ -804,7 +804,7 @@ void CMasternodeMan::ProcessMessage(CNode* pfrom, std::string& strCommand, CData
             nInvCount++;
 
             if (!mapSeenMasternodeBroadcast.count(hash)) {
-                mapSeenMasternodeBroadcast.insert(std::make_pair(hash, mnb));
+                mapSeenMasternodeBroadcast.insert(std::make_pair(hash, std::make_pair(GetTime(), mnb)));
             }
 
             if (vin == mn.vin) {
@@ -1281,7 +1281,7 @@ void CMasternodeMan::UpdateMasternodeList(CMasternodeBroadcast mnb)
 {
     LOCK(cs);
     mapSeenMasternodePing.insert(std::make_pair(mnb.lastPing.GetHash(), mnb.lastPing));
-    mapSeenMasternodeBroadcast.insert(std::make_pair(mnb.GetHash(), mnb));
+    mapSeenMasternodeBroadcast.insert(std::make_pair(mnb.GetHash(), std::make_pair(GetTime(), mnb)));
 
     LogPrintf("CMasternodeMan::UpdateMasternodeList -- masternode=%s  addr=%s\n", mnb.vin.prevout.ToStringShort(), mnb.addr.ToString());
 
@@ -1292,7 +1292,7 @@ void CMasternodeMan::UpdateMasternodeList(CMasternodeBroadcast mnb)
             masternodeSync.AddedMasternodeList();
         }
     } else {
-        CMasternodeBroadcast mnbOld = mapSeenMasternodeBroadcast[CMasternodeBroadcast(*pmn).GetHash()];
+        CMasternodeBroadcast mnbOld = mapSeenMasternodeBroadcast[CMasternodeBroadcast(*pmn).GetHash()].second;
         if(pmn->UpdateFromNewBroadcast(mnb)) {
             masternodeSync.AddedMasternodeList();
             mapSeenMasternodeBroadcast.erase(mnbOld.GetHash());
@@ -1308,11 +1308,18 @@ bool CMasternodeMan::CheckMnbAndUpdateMasternodeList(CMasternodeBroadcast mnb, i
     nDos = 0;
     LogPrint("masternode", "CMasternodeMan::CheckMnbAndUpdateMasternodeList -- masternode=%s\n", mnb.vin.prevout.ToStringShort());
 
-    if(mapSeenMasternodeBroadcast.count(mnb.GetHash())) { //seen
+    uint256 hash = mnb.GetHash();
+    if(mapSeenMasternodeBroadcast.count(hash)) { //seen
         LogPrint("masternode", "CMasternodeMan::CheckMnbAndUpdateMasternodeList -- masternode=%s seen\n", mnb.vin.prevout.ToStringShort());
+        // less then 2 pings left before this MN goes into non-recoverable state, bump sync timeout
+        if(GetTime() - mapSeenMasternodeBroadcast[hash].first > MASTERNODE_NEW_START_REQUIRED_SECONDS - MASTERNODE_MIN_MNP_SECONDS * 2) {
+            LogPrint("masternode", "CMasternodeMan::CheckMnbAndUpdateMasternodeList -- masternode=%s seen update\n", mnb.vin.prevout.ToStringShort());
+            mapSeenMasternodeBroadcast[hash].first = GetTime();
+            masternodeSync.AddedMasternodeList();
+        }
         return true;
     }
-    mapSeenMasternodeBroadcast.insert(std::make_pair(mnb.GetHash(), mnb));
+    mapSeenMasternodeBroadcast.insert(std::make_pair(hash, std::make_pair(GetTime(), mnb)));
 
     LogPrint("masternode", "CMasternodeMan::CheckMnbAndUpdateMasternodeList -- masternode=%s new\n", mnb.vin.prevout.ToStringShort());
 
@@ -1324,12 +1331,12 @@ bool CMasternodeMan::CheckMnbAndUpdateMasternodeList(CMasternodeBroadcast mnb, i
     // search Masternode list
     CMasternode* pmn = Find(mnb.vin);
     if(pmn) {
-        CMasternodeBroadcast mnbOld = mapSeenMasternodeBroadcast[CMasternodeBroadcast(*pmn).GetHash()];
+        CMasternodeBroadcast mnbOld = mapSeenMasternodeBroadcast[CMasternodeBroadcast(*pmn).GetHash()].second;
         if(!mnb.Update(pmn, nDos)) {
             LogPrint("masternode", "CMasternodeMan::CheckMnbAndUpdateMasternodeList -- Update() failed, masternode=%s\n", mnb.vin.prevout.ToStringShort());
             return false;
         }
-        if(mnb.GetHash() != mnbOld.GetHash()) {
+        if(hash != mnbOld.GetHash()) {
             mapSeenMasternodeBroadcast.erase(mnbOld.GetHash());
         }
     } else {
@@ -1509,7 +1516,7 @@ void CMasternodeMan::SetMasternodeLastPing(const CTxIn& vin, const CMasternodePi
     CMasternodeBroadcast mnb(*pMN);
     uint256 hash = mnb.GetHash();
     if(mapSeenMasternodeBroadcast.count(hash)) {
-        mapSeenMasternodeBroadcast[hash].lastPing = mnp;
+        mapSeenMasternodeBroadcast[hash].second.lastPing = mnp;
     }
 }
 

src/masternodeman.h

@@ -144,7 +144,7 @@ private:
 
 public:
     // Keep track of all broadcasts I've seen
-    std::map<uint256, CMasternodeBroadcast> mapSeenMasternodeBroadcast;
+    std::map<uint256, std::pair<int64_t, CMasternodeBroadcast> > mapSeenMasternodeBroadcast;
     // Keep track of all pings I've seen
     std::map<uint256, CMasternodePing> mapSeenMasternodePing;
     // Keep track of all verifications I've seen