From c92cbce6a547ca361f83a3560d7d4a2a605bea68 Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Mon, 29 Aug 2022 12:42:29 +0300 Subject: [PATCH 1/7] trivial: Fix trailing whitespaces in release notes (#4989) Local CI linter complains --- doc/release-notes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index eed0aab8cd..a0654a231d 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -31,8 +31,8 @@ from version 0.13 should not require any additional actions. When upgrading from a version prior to 18.0.1, the first startup of Dash Core will run a migration process which can take anywhere -from a few minutes to thirty minutes to finish. After the migration, a -downgrade to an older version is only possible with a reindex +from a few minutes to thirty minutes to finish. After the migration, a +downgrade to an older version is only possible with a reindex (or reindex-chainstate). Downgrade warning From 8efd7f04c67ac1e7e0317b135139ba669eaedde0 Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Fri, 23 Sep 2022 17:43:39 +0300 Subject: [PATCH 2/7] Merge bitcoin/bitcoin#25739: Update leveldb subtree (#5005) f608f25313d2867a6abdfc38abdb86da40924cfc Squashed 'src/leveldb/' changes from 330dd6235f..22f1e4a02f (fanquake) Pull request description: Replaces #25463 (for master). Includes: - https://github.com/bitcoin-core/leveldb-subtree/pull/32 Guix Build (arm64): ```bash ed41ae2555ae3b638b65d870cef385805e621481831ae992e84645f5c234af63 guix-build-bec911e37ac8/output/arm-linux-gnueabihf/SHA256SUMS.part 7d8237026bfccedee0e56e14d7b89cf2dcb52195b94070dc4b6c3c6970fdc774 guix-build-bec911e37ac8/output/arm-linux-gnueabihf/bitcoin-bec911e37ac8-arm-linux-gnueabihf-debug.tar.gz 1afeff9d70864be66f7ac48d31d1977c7844e2cc117d3f0438fb2f9c6f6a56a4 guix-build-bec911e37ac8/output/arm-linux-gnueabihf/bitcoin-bec911e37ac8-arm-linux-gnueabihf.tar.gz ab2df265897a0142093b582d3c61e2b17a713d93e478b47c7aa2b0a677295007 guix-build-bec911e37ac8/output/arm64-apple-darwin/SHA256SUMS.part 025a52babdee479800801e951f6fe1fe490e1f5fe8361104eb85e7b0319cdb37 guix-build-bec911e37ac8/output/arm64-apple-darwin/bitcoin-bec911e37ac8-arm64-apple-darwin-unsigned.dmg 7b2992bf5543891b1e6ef4f48c52fc5febc58cc31ccf4307edd27d4d630aa54a guix-build-bec911e37ac8/output/arm64-apple-darwin/bitcoin-bec911e37ac8-arm64-apple-darwin-unsigned.tar.gz 3d93eb009ab6459cdca5fe767795f94ba5e00e5969e44bac19c7a5f344d42030 guix-build-bec911e37ac8/output/arm64-apple-darwin/bitcoin-bec911e37ac8-arm64-apple-darwin.tar.gz b8c30c5c561c96bc4e280ececc0dd1cd673bc6194591b848903aa6c54c9d37cf guix-build-bec911e37ac8/output/dist-archive/bitcoin-bec911e37ac8.tar.gz cb2632b9b5ff473e504d3d6244191eb5a852718f8ac8bb1032ba1c65e07d6b3f guix-build-bec911e37ac8/output/powerpc64-linux-gnu/SHA256SUMS.part 26ecc2f42ce37f8bd7ba24bf2f80f493cd1fd45b58409de71c44e2445c291c8c guix-build-bec911e37ac8/output/powerpc64-linux-gnu/bitcoin-bec911e37ac8-powerpc64-linux-gnu-debug.tar.gz 4a83381ea472cc71b4a1c6569483a7e85a5f53065c1633828bf7a75d357b0297 guix-build-bec911e37ac8/output/powerpc64-linux-gnu/bitcoin-bec911e37ac8-powerpc64-linux-gnu.tar.gz 455a9af5e7ee1c2857d87abec29284ae7bb447cf7cb2b3befa2f8e0a0a8cbec6 guix-build-bec911e37ac8/output/powerpc64le-linux-gnu/SHA256SUMS.part 3445f53fd150032ec3c3f324e001b4ecf72728900961a7a7789c32ceb7616617 guix-build-bec911e37ac8/output/powerpc64le-linux-gnu/bitcoin-bec911e37ac8-powerpc64le-linux-gnu-debug.tar.gz baefeaac88bb4fbf8662c8e1150b043aa2534535a82e828c13e971d2c5fa5cbd guix-build-bec911e37ac8/output/powerpc64le-linux-gnu/bitcoin-bec911e37ac8-powerpc64le-linux-gnu.tar.gz 543484396a47def1636d4e54d4a105c7093265c8896165a4140edec10aeef880 guix-build-bec911e37ac8/output/riscv64-linux-gnu/SHA256SUMS.part ac1f6e57016703f1319a3ef80014581aee96053e56525b8cd11ada2395496b86 guix-build-bec911e37ac8/output/riscv64-linux-gnu/bitcoin-bec911e37ac8-riscv64-linux-gnu-debug.tar.gz f9a2b65ed21f777524b078046c84b98239b0fbb92eae8d840bc7a25cab0eec6d guix-build-bec911e37ac8/output/riscv64-linux-gnu/bitcoin-bec911e37ac8-riscv64-linux-gnu.tar.gz a08ed0ee78ab1c4c815ba8368f1a21d3bd4327ce1104cb3793d63edd2bde1ef4 guix-build-bec911e37ac8/output/x86_64-apple-darwin/SHA256SUMS.part 74059397c6c8f0e899b60415d1aae04f2f7df18b8f39cea9f314e5e0c2e0ede6 guix-build-bec911e37ac8/output/x86_64-apple-darwin/bitcoin-bec911e37ac8-x86_64-apple-darwin-unsigned.dmg 6909ff6f59b78059d505f2b98e6fad63a4e3deb843566061c4cd6e94be1de066 guix-build-bec911e37ac8/output/x86_64-apple-darwin/bitcoin-bec911e37ac8-x86_64-apple-darwin-unsigned.tar.gz 21b45719d927422acc69662108e7255d8cd0b1d832493e70c622c1b1b3a3a314 guix-build-bec911e37ac8/output/x86_64-apple-darwin/bitcoin-bec911e37ac8-x86_64-apple-darwin.tar.gz b5fdee6fd2dbcdaa6302f388c7fcaa6d130ff91fd5760e19facf6e24c7216ef6 guix-build-bec911e37ac8/output/x86_64-linux-gnu/SHA256SUMS.part 56db24b0b0d2f463c8085a12502977c6d4f4ee5b85484e52522361f54ab3a6aa guix-build-bec911e37ac8/output/x86_64-linux-gnu/bitcoin-bec911e37ac8-x86_64-linux-gnu-debug.tar.gz 96dbe08e2ed0f68fe734dd6f0280c2d22af7b56c84debde424367054f118173f guix-build-bec911e37ac8/output/x86_64-linux-gnu/bitcoin-bec911e37ac8-x86_64-linux-gnu.tar.gz 70ac424b229befb2834a8a02ce27551e3ddde2d438497e8c11a3cedf0cea5d3c guix-build-bec911e37ac8/output/x86_64-w64-mingw32/SHA256SUMS.part bad144a599b28e8dc0018cc2fd1754543e79df39d651e58f565c197241f2b8cb guix-build-bec911e37ac8/output/x86_64-w64-mingw32/bitcoin-bec911e37ac8-win64-debug.zip 7ce2e72621eb070a8d23b7edbaaccc9f06257b82b9851c1cad4c61f08d2c7451 guix-build-bec911e37ac8/output/x86_64-w64-mingw32/bitcoin-bec911e37ac8-win64-setup-unsigned.exe 5f726ef8b478e3ac90b93cd3ae3c38a0e7bffa5f80306c46a7535518a73251c7 guix-build-bec911e37ac8/output/x86_64-w64-mingw32/bitcoin-bec911e37ac8-win64-unsigned.tar.gz 87cf1a23e948e471ed35c6a518813505c907c58788b55665829e7f12f33bd312 guix-build-bec911e37ac8/output/x86_64-w64-mingw32/bitcoin-bec911e37ac8-win64.zip ``` ACKs for top commit: hebasto: ACK bec911e37ac826d55b789428bc07280abab76443, I have reviewed the code and it looks OK, I agree it can be merged. jarolrod: ACK bec911e37ac826d55b789428bc07280abab76443 Tree-SHA512: 190381d9489ec6cc52bb9557750925c8574f1344eb6893095e9e31e66a579bd1bc283e8cbfcba52cec3fb072985895f929103b6f5351a23f908bdd0a04b474f1 Co-authored-by: fanquake --- src/leveldb/util/env_posix.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/leveldb/util/env_posix.cc b/src/leveldb/util/env_posix.cc index 18626b327c..fac41be6ce 100644 --- a/src/leveldb/util/env_posix.cc +++ b/src/leveldb/util/env_posix.cc @@ -49,7 +49,7 @@ constexpr const int kDefaultMmapLimit = (sizeof(void*) >= 8) ? 4096 : 0; int g_mmap_limit = kDefaultMmapLimit; // Common flags defined for all posix open operations -#if defined(HAVE_O_CLOEXEC) +#if HAVE_O_CLOEXEC constexpr const int kOpenBaseFlags = O_CLOEXEC; #else constexpr const int kOpenBaseFlags = 0; From e8afde2740defff43aa70a1d4da0a14bd9f9978b Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Fri, 23 Sep 2022 17:46:18 +0300 Subject: [PATCH 3/7] fix: Flush chainstate (and evodb) cache whenever evodb mem usage is getting too high (#5007) * fix: Flush chainstate (and evodb) cache whenever evodb mem usage is getting too high * Bump evodb cache to 64MiB --- src/init.cpp | 2 +- src/validation.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 4d525ed80b..d349a7fb3e 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1932,7 +1932,7 @@ bool AppInitMain(InitInterfaces& interfaces) nTotalCache -= nCoinDBCache; nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - int64_t nEvoDbCache = 1024 * 1024 * 16; // TODO + int64_t nEvoDbCache = 1024 * 1024 * 64; // TODO LogPrintf("Cache configuration:\n"); LogPrintf("* Using %.1f MiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { diff --git a/src/validation.cpp b/src/validation.cpp index da8863b8b7..cc6a38aa91 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2433,8 +2433,6 @@ CoinsCacheSizeState CChainState::GetCoinsCacheSizeState( int64_t nTotalSpace = max_coins_cache_size_bytes + std::max(max_mempool_size_bytes - nMempoolUsage, 0); - cacheSize += evoDb->GetMemoryUsage(); - //! No need to periodic flush if at least this much space still available. static constexpr int64_t MAX_BLOCK_COINSDB_USAGE_BYTES = 10 * 1024 * 1024; // 10MB int64_t large_threshold = @@ -2502,12 +2500,14 @@ bool CChainState::FlushStateToDisk( bool fCacheLarge = mode == FlushStateMode::PERIODIC && cache_state >= CoinsCacheSizeState::LARGE; // The cache is over the limit, we have to write now. bool fCacheCritical = mode == FlushStateMode::IF_NEEDED && cache_state >= CoinsCacheSizeState::CRITICAL; + // The evodb cache is too large + bool fEvoDbCacheCritical = mode == FlushStateMode::IF_NEEDED && evoDb != nullptr && evoDb->GetMemoryUsage() >= (64 << 20); // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. bool fPeriodicWrite = mode == FlushStateMode::PERIODIC && nNow > nLastWrite + DATABASE_WRITE_INTERVAL; // It's been very long since we flushed the cache. Do this infrequently, to optimize cache usage. bool fPeriodicFlush = mode == FlushStateMode::PERIODIC && nNow > nLastFlush + DATABASE_FLUSH_INTERVAL; // Combine all conditions that result in a full cache flush. - fDoFullFlush = (mode == FlushStateMode::ALWAYS) || fCacheLarge || fCacheCritical || fPeriodicFlush || fFlushForPrune; + fDoFullFlush = (mode == FlushStateMode::ALWAYS) || fCacheLarge || fCacheCritical || fEvoDbCacheCritical || fPeriodicFlush || fFlushForPrune; // Write blocks and block index to disk. if (fDoFullFlush || fPeriodicWrite) { // Depend on nMinDiskSpace to ensure we can write block index From 0b60096d8a23e6019ef5b0975515626ab7cb1b1e Mon Sep 17 00:00:00 2001 From: pasta Date: Thu, 29 Sep 2022 16:32:40 -0500 Subject: [PATCH 4/7] chore: bump version to 18.0.2 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 58665dd3aa..f93931632a 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ AC_PREREQ([2.69]) define(_CLIENT_VERSION_MAJOR, 18) define(_CLIENT_VERSION_MINOR, 0) -define(_CLIENT_VERSION_BUILD, 1) +define(_CLIENT_VERSION_BUILD, 2) define(_CLIENT_VERSION_RC, 0) define(_CLIENT_VERSION_IS_RELEASE, true) define(_COPYRIGHT_YEAR, 2022) From 96f4022a6a84cc7c9b7d092fb67212760b5cdc08 Mon Sep 17 00:00:00 2001 From: pasta Date: Thu, 29 Sep 2022 16:40:28 -0500 Subject: [PATCH 5/7] chore: archive release-nodes.md --- .../dash/release-notes-18.0.1.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/{release-notes.md => release-notes/dash/release-notes-18.0.1.md} (100%) diff --git a/doc/release-notes.md b/doc/release-notes/dash/release-notes-18.0.1.md similarity index 100% rename from doc/release-notes.md rename to doc/release-notes/dash/release-notes-18.0.1.md From 666ff7bff94ac90a21b9e440741a017f42c692c0 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kittywhiskers@users.noreply.github.com> Date: Sat, 10 Sep 2022 02:15:39 +0300 Subject: [PATCH 6/7] merge bitcoin#14193: Add missing mempool locks Co-authored-by: "UdjinM6 " --- src/governance/governance.cpp | 12 ++- src/governance/object.cpp | 5 +- src/net_processing.cpp | 2 +- src/rpc/governance.cpp | 2 +- src/rpc/mining.cpp | 3 +- src/test/validation_block_tests.cpp | 157 +++++++++++++++++++++++++++- src/txmempool.cpp | 20 ++-- src/txmempool.h | 36 +++---- src/validation.cpp | 18 ++-- src/validation.h | 7 +- 10 files changed, 206 insertions(+), 56 deletions(-) diff --git a/src/governance/governance.cpp b/src/governance/governance.cpp index 4a351f8257..b023ea707c 100644 --- a/src/governance/governance.cpp +++ b/src/governance/governance.cpp @@ -142,7 +142,8 @@ void CGovernanceManager::ProcessMessage(CNode* pfrom, const std::string& msg_typ return; } - LOCK2(cs_main, cs); + LOCK2(cs_main, ::mempool.cs); // Lock mempool because of GetTransaction deep inside + LOCK(cs); if (mapObjects.count(nHash) || mapPostponedObjects.count(nHash) || mapErasedGovernanceObjects.count(nHash)) { // TODO - print error code? what if it's GOVOBJ_ERROR_IMMATURE? @@ -261,7 +262,8 @@ void CGovernanceManager::AddGovernanceObject(CGovernanceObject& govobj, CConnman govobj.UpdateSentinelVariables(); //this sets local vars in object - LOCK2(cs_main, cs); + LOCK2(cs_main, ::mempool.cs); // Lock mempool because of GetTransaction deep inside + LOCK(cs); std::string strError; // MAKE SURE THIS OBJECT IS OK @@ -320,7 +322,8 @@ void CGovernanceManager::UpdateCachesAndClean() std::vector vecDirtyHashes = mmetaman.GetAndClearDirtyGovernanceObjectHashes(); - LOCK2(cs_main, cs); + LOCK2(cs_main, ::mempool.cs); // Lock mempool because of GetTransaction deep inside + LOCK(cs); for (const uint256& nHash : vecDirtyHashes) { auto it = mapObjects.find(nHash); @@ -835,7 +838,8 @@ void CGovernanceManager::CheckPostponedObjects(CConnman& connman) { if (!masternodeSync.IsSynced()) return; - LOCK2(cs_main, cs); + LOCK2(cs_main, ::mempool.cs); // Lock mempool because of GetTransaction deep inside + LOCK(cs); // Check postponed proposals for (auto it = mapPostponedObjects.begin(); it != mapPostponedObjects.end();) { diff --git a/src/governance/object.cpp b/src/governance/object.cpp index d8b3b67782..49ae94ec80 100644 --- a/src/governance/object.cpp +++ b/src/governance/object.cpp @@ -438,7 +438,7 @@ UniValue CGovernanceObject::ToJson() const void CGovernanceObject::UpdateLocalValidity() { - LOCK(cs_main); + AssertLockHeld(cs_main); // THIS DOES NOT CHECK COLLATERAL, THIS IS CHECKED UPON ORIGINAL ARRIVAL fCachedLocalValidity = IsValidLocally(strLocalValidityError, false); } @@ -526,6 +526,9 @@ CAmount CGovernanceObject::GetMinCollateralFee(bool fork_active) const bool CGovernanceObject::IsCollateralValid(std::string& strError, bool& fMissingConfirmations) const { + AssertLockHeld(cs_main); + AssertLockHeld(::mempool.cs); // because of GetTransaction + strError = ""; fMissingConfirmations = false; uint256 nExpectedHash = GetHash(); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 452cfa2fb0..a9470f2c93 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -4629,7 +4629,7 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // std::vector vInv; { - LOCK(pto->cs_inventory); + LOCK2(mempool.cs, pto->cs_inventory); size_t reserve = std::min(pto->setInventoryTxToSend.size(), INVENTORY_BROADCAST_MAX_PER_1MB_BLOCK * MaxBlockSize() / 1000000); reserve = std::max(reserve, pto->vInventoryBlockToSend.size()); diff --git a/src/rpc/governance.cpp b/src/rpc/governance.cpp index 996349e00c..22751687eb 100644 --- a/src/rpc/governance.cpp +++ b/src/rpc/governance.cpp @@ -392,7 +392,7 @@ static UniValue gobject_submit(const JSONRPCRequest& request) g_txindex->BlockUntilSyncedToCurrentChain(); } - LOCK(cs_main); + LOCK2(cs_main, ::mempool.cs); if (!govobj.IsValidLocally(strError, fMissingConfirmations, true) && !fMissingConfirmations) { LogPrintf("gobject(submit) -- Object submission rejected because object is not valid - hash = %s, strError = %s\n", strHash, strError); throw JSONRPCError(RPC_INTERNAL_ERROR, "Governance object is not valid - " + strHash + " - " + strError); diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index e28aacfe8a..bbad597799 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -501,7 +501,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) nTransactionsUpdatedLastLP = nTransactionsUpdatedLast; } - // Release the wallet and main lock while waiting + // Release lock while waiting LEAVE_CRITICAL_SECTION(cs_main); { checktxtime = std::chrono::steady_clock::now() + std::chrono::minutes(1); @@ -512,6 +512,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) if (g_best_block_cv.wait_until(lock, checktxtime) == std::cv_status::timeout) { // Timeout: Check transactions for update + // without holding ::mempool.cs to avoid deadlocks if (mempool.GetTransactionsUpdated() != nTransactionsUpdatedLastLP) break; checktxtime += std::chrono::seconds(10); diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 99d3c4beb8..f5d59fccb5 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -5,11 +5,13 @@ #include #include +#include #include #include #include #include #include +#include