From 13c12406ae076b58cba580aa47c3419ef5bec361 Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Sat, 9 Sep 2023 20:28:12 +0300 Subject: [PATCH] fix: actually erase old evodb data on db migration (#5570) ## Issue being fixed or feature implemented The old evodb data wasn't dropped on db migration really. `Erase()` does nothing. ## What was done? Loop through the old data and drop it in batches per db key. Do this both for nodes that are doing migration for the first time and for nodes that did migration in the past already. ## How Has This Been Tested? Running different versions on testnet ``` # reindex with 18.2.2 till block 850000 (pre-v19 block) $ du -hd1 ~/.dashcore/testnet3/evodb/ 276M .dashcore/testnet3/evodb # continue with develop, migration just finished, keep syncing till current tip, block 901000+ $ du -hd1 ~/.dashcore/testnet3/evodb/ 469M .dashcore/testnet3/evodb # continue with this PR, start at current tip, "migration already done. cleaned old data." $ du -hd1 ~/.dashcore/testnet3/evodb/ 302M .dashcore/testnet3/evodb ``` ## Breaking Changes Should be none. ## Checklist: - [x] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have added or updated relevant unit/integration/functional/e2e tests - [ ] I have made corresponding changes to the documentation - [x] I have assigned this pull request to a milestone _(for repository code-owners and collaborators only)_ --- src/evo/deterministicmns.cpp | 68 ++++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/src/evo/deterministicmns.cpp b/src/evo/deterministicmns.cpp index 0708ff9d26..ad0e01b7ba 100644 --- a/src/evo/deterministicmns.cpp +++ b/src/evo/deterministicmns.cpp @@ -1181,6 +1181,32 @@ void CDeterministicMNManager::CleanupCache(int nHeight) } +[[nodiscard]] static bool EraseOldDBData(CDBWrapper& db, const std::vector& db_key_prefixes) +{ + bool erased{false}; + for(const auto& db_key_prefix : db_key_prefixes) { + CDBBatch batch{db}; + std::unique_ptr it{db.NewIterator()}; + std::pair firstKey{db_key_prefix, uint256()}; + it->Seek(firstKey); + while (it->Valid()) { + decltype(firstKey) curKey; + if (!it->GetKey(curKey) || std::get<0>(curKey) != db_key_prefix) { + break; + } + batch.Erase(curKey); + erased = true; + it->Next(); + } + if (erased) { + LogPrintf("CDeterministicMNManager::%s -- updating db...\n", __func__); + db.WriteBatch(batch); + LogPrintf("CDeterministicMNManager::%s -- done cleaning old data for %s\n", __func__, db_key_prefix); + } + } + return erased; +} + bool CDeterministicMNManager::MigrateDBIfNeeded() { static const std::string DB_OLD_LIST_SNAPSHOT = "dmn_S"; @@ -1199,7 +1225,19 @@ bool CDeterministicMNManager::MigrateDBIfNeeded() } if (m_evoDb.GetRawDB().Exists(EVODB_BEST_BLOCK) || m_evoDb.GetRawDB().Exists(DB_OLD_BEST_BLOCK2)) { - LogPrintf("CDeterministicMNManager::%s -- migration already done. skipping.\n", __func__); + if (EraseOldDBData(m_evoDb.GetRawDB(), {DB_OLD_LIST_DIFF, DB_OLD_LIST_SNAPSHOT})) { + // we messed up, make sure this time we actually drop old data + LogPrintf("CDeterministicMNManager::%s -- migration already done. cleaned old data.\n", __func__); + m_evoDb.GetRawDB().CompactFull(); + LogPrintf("CDeterministicMNManager::%s -- done compacting database\n", __func__); + // flush it to disk + if (!m_evoDb.CommitRootTransaction()) { + LogPrintf("CDeterministicMNManager::%s -- failed to commit to evoDB\n", __func__); + return false; + } + } else { + LogPrintf("CDeterministicMNManager::%s -- migration already done. skipping.\n", __func__); + } return true; } @@ -1263,10 +1301,9 @@ bool CDeterministicMNManager::MigrateDBIfNeeded() LogPrintf("CDeterministicMNManager::%s -- done migrating\n", __func__); - m_evoDb.GetRawDB().Erase(DB_OLD_LIST_DIFF); - m_evoDb.GetRawDB().Erase(DB_OLD_LIST_SNAPSHOT); - - LogPrintf("CDeterministicMNManager::%s -- done cleaning old data\n", __func__); + if (EraseOldDBData(m_evoDb.GetRawDB(), {DB_OLD_LIST_DIFF, DB_OLD_LIST_SNAPSHOT})) { + LogPrintf("CDeterministicMNManager::%s -- done cleaning old data\n", __func__); + } m_evoDb.GetRawDB().CompactFull(); @@ -1298,7 +1335,19 @@ bool CDeterministicMNManager::MigrateDBIfNeeded2() } if (m_evoDb.GetRawDB().Exists(EVODB_BEST_BLOCK)) { - LogPrintf("CDeterministicMNManager::%s -- migration already done. skipping.\n", __func__); + if (EraseOldDBData(m_evoDb.GetRawDB(), {DB_OLD_LIST_DIFF, DB_OLD_LIST_SNAPSHOT})) { + // we messed up, make sure this time we actually drop old data + LogPrintf("CDeterministicMNManager::%s -- migration already done. cleaned old data.\n", __func__); + m_evoDb.GetRawDB().CompactFull(); + LogPrintf("CDeterministicMNManager::%s -- done compacting database\n", __func__); + // flush it to disk + if (!m_evoDb.CommitRootTransaction()) { + LogPrintf("CDeterministicMNManager::%s -- failed to commit to evoDB\n", __func__); + return false; + } + } else { + LogPrintf("CDeterministicMNManager::%s -- migration already done. skipping.\n", __func__); + } return true; } @@ -1362,10 +1411,9 @@ bool CDeterministicMNManager::MigrateDBIfNeeded2() LogPrintf("CDeterministicMNManager::%s -- done migrating\n", __func__); - m_evoDb.GetRawDB().Erase(DB_OLD_LIST_DIFF); - m_evoDb.GetRawDB().Erase(DB_OLD_LIST_SNAPSHOT); - - LogPrintf("CDeterministicMNManager::%s -- done cleaning old data\n", __func__); + if (EraseOldDBData(m_evoDb.GetRawDB(), {DB_OLD_LIST_DIFF, DB_OLD_LIST_SNAPSHOT})) { + LogPrintf("CDeterministicMNManager::%s -- done cleaning old data\n", __func__); + } m_evoDb.GetRawDB().CompactFull();