Fix recovery from coin db crashes (and dbcrash.py test) (#3467)

* 🪲 improve evodb consistency recovering from dbcrash

* Adjust the fix

Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>

* Fix it

Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>

* Disable recovery from a crash during a fork and a corresponding part of dbcrash.py

Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>

* Skip some checks in CQuorumBlockProcessor when replaying blocks after the crash

Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>

* Process special txes in RollforwardBlock

Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>

* Update src/init.cpp

Co-authored-by: PastaPastaPasta <6443210+PastaPastaPasta@users.noreply.github.com>

Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>
Co-authored-by: PastaPastaPasta <6443210+PastaPastaPasta@users.noreply.github.com>

src/init.cpp

@@ -1905,6 +1905,12 @@ bool AppInitMain()
                 // The on-disk coinsdb is now in a good state, create the cache
                 pcoinsTip.reset(new CCoinsViewCache(pcoinscatcher.get()));
 
+                // flush evodb
+                if (!evoDb->CommitRootTransaction()) {
+                    strLoadError = _("Failed to commit EvoDB");
+                    break;
+                }
+
                 bool is_coinsview_empty = fReset || fReindexChainState || pcoinsTip->GetBestBlock().IsNull();
                 if (!is_coinsview_empty) {
                     // LoadChainTip sets chainActive based on pcoinsTip's best block

src/llmq/quorums_blockprocessor.cpp

@@ -135,6 +135,11 @@ bool CQuorumBlockProcessor::ProcessBlock(const CBlock& block, const CBlockIndex*
     // until the first non-null commitment has been mined. After the non-null commitment, no other commitments are
     // allowed, including null commitments.
     for (const auto& p : Params().GetConsensus().llmqs) {
+        // skip these checks when replaying blocks after the crash
+        if (!chainActive.Tip()) {
+            break;
+        }
+
         auto type = p.first;
 
         // does the currently processed block contain a (possibly null) commitment for the current session?
@@ -180,6 +185,12 @@ bool CQuorumBlockProcessor::ProcessCommitment(int nHeight, const uint256& blockH
     auto& params = Params().GetConsensus().llmqs.at((Consensus::LLMQType)qc.llmqType);
 
     uint256 quorumHash = GetQuorumBlockHash((Consensus::LLMQType)qc.llmqType, nHeight);
+
+    // skip `bad-qc-block` checks below when replaying blocks after the crash
+    if (!chainActive.Tip()) {
+        quorumHash = qc.quorumHash;
+    }
+
     if (quorumHash.IsNull()) {
         return state.DoS(100, false, REJECT_INVALID, "bad-qc-block");
     }

src/validation.cpp

@@ -1728,8 +1728,6 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI
         }
     }
 
-    // move best block pointer to prevout block
-    view.SetBestBlock(pindex->pprev->GetBlockHash());
 
     if (fSpentIndex) {
         if (!pblocktree->UpdateSpentIndex(spentIndex)) {
@@ -1749,6 +1747,8 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI
         }
     }
 
+    // move best block pointer to prevout block
+    view.SetBestBlock(pindex->pprev->GetBlockHash());
     evoDb->WriteBestBlock(pindex->pprev->GetBlockHash());
 
     return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN;
@@ -4322,6 +4322,13 @@ bool CChainState::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& i
         // Pass check = true as every addition may be an overwrite.
         AddCoins(inputs, *tx, pindex->nHeight, true);
     }
+
+    CValidationState state;
+    if (!ProcessSpecialTxsInBlock(block, pindex, state, false /*fJustCheck*/, false /*fScriptChecks*/)) {
+        return error("RollforwardBlock(DASH): ProcessSpecialTxsInBlock for block %s failed with %s",
+            pindex->GetBlockHash().ToString(), FormatStateMessage(state));
+    }
+
     return true;
 }
 
@@ -4356,8 +4363,13 @@ bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view)
         assert(pindexFork != nullptr);
     }
 
+    auto dbTx = evoDb->BeginTransaction();
+
     // Rollback along the old branch.
     while (pindexOld != pindexFork) {
+        // TODO: RollforwardBlock should update not only coins but also evodb and additional indexes.
+        // Disable recovery from a crash during a fork until this is implemented.
+        return error("ReplayBlocks(): recovery from a db crash during a fork is not supported yet");
         if (pindexOld->nHeight > 0) { // Never disconnect the genesis block.
             CBlock block;
             if (!ReadBlockFromDisk(block, pindexOld, params.GetConsensus())) {
@@ -4385,7 +4397,10 @@ bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view)
     }
 
     cache.SetBestBlock(pindexNew->GetBlockHash());
-    cache.Flush();
+    evoDb->WriteBestBlock(pindexNew->GetBlockHash());
+    bool flushed = cache.Flush();
+    assert(flushed);
+    dbTx->Commit();
     uiInterface.ShowProgress("", 100, false);
     return true;
 }

test/functional/dbcrash.py

@@ -229,14 +229,15 @@ class ChainstateWriteCrashTest(BitcoinTestFramework):
             self.generate_small_transactions(self.nodes[3], 2500, utxo_list)
             # Pick a random block between current tip, and starting tip
             current_height = self.nodes[3].getblockcount()
-            random_height = random.randint(starting_tip_height, current_height)
+            # TODO: re-enable this when ReplayBlocks is fixed to support evodb and additional indexes
-            self.log.debug("At height %d, considering height %d", current_height, random_height)
+            # random_height = random.randint(starting_tip_height, current_height)
-            if random_height > starting_tip_height:
+            # self.log.debug("At height %d, considering height %d", current_height, random_height)
-                # Randomly reorg from this point with some probability (1/4 for
+            # if random_height > starting_tip_height:
-                # tip, 1/5 for tip-1, ...)
+            #     # Randomly reorg from this point with some probability (1/4 for
-                if random.random() < 1.0/(current_height + 4 - random_height):
+            #     # tip, 1/5 for tip-1, ...)
-                    self.log.debug("Invalidating block at height %d", random_height)
+            #     if random.random() < 1.0/(current_height + 4 - random_height):
-                    self.nodes[3].invalidateblock(self.nodes[3].getblockhash(random_height))
+            #         self.log.debug("Invalidating block at height %d", random_height)
+            #         self.nodes[3].invalidateblock(self.nodes[3].getblockhash(random_height))
 
             # Now generate new blocks until we pass the old tip height
             self.log.debug("Mining longer tip")