mirror of
https://github.com/dashpay/dash.git
synced 2024-12-25 03:52:49 +01:00
merge bitcoin#21726: Improve Indices on pruned nodes via prune blockers
This commit is contained in:
Kittywhiskers Van Gogh
parent
c65ec190c5
commit
baf6e26eed
@ -62,9 +62,9 @@ bool BaseIndex::Init()
|
||||
LOCK(cs_main);
|
||||
CChain& active_chain = m_chainstate->m_chain;
|
||||
if (locator.IsNull()) {
|
||||
m_best_block_index = nullptr;
|
||||
SetBestBlockIndex(nullptr);
|
||||
} else {
|
||||
m_best_block_index = m_chainstate->FindForkInGlobalIndex(locator);
|
||||
SetBestBlockIndex(m_chainstate->FindForkInGlobalIndex(locator));
|
||||
}
|
||||
|
||||
// Note: this will latch to true immediately if the user starts up with an empty
|
||||
@ -76,11 +76,7 @@ bool BaseIndex::Init()
|
||||
if (!m_best_block_index) {
|
||||
// index is not built yet
|
||||
// make sure we have all block data back to the genesis
|
||||
const CBlockIndex* block = active_chain.Tip();
|
||||
while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) {
|
||||
block = block->pprev;
|
||||
}
|
||||
prune_violation = block != active_chain.Genesis();
|
||||
prune_violation = GetFirstStoredBlock(active_chain.Tip()) != active_chain.Genesis();
|
||||
}
|
||||
// in case the index has a best block set and is not fully synced
|
||||
// check if we have the required blocks to continue building the index
|
||||
@ -138,7 +134,7 @@ void BaseIndex::ThreadSync()
|
||||
std::chrono::steady_clock::time_point last_locator_write_time{0s};
|
||||
while (true) {
|
||||
if (m_interrupt) {
|
||||
m_best_block_index = pindex;
|
||||
SetBestBlockIndex(pindex);
|
||||
// No need to handle errors in Commit. If it fails, the error will be already be
|
||||
// logged. The best way to recover is to continue, as index cannot be corrupted by
|
||||
// a missed commit to disk for an advanced index state.
|
||||
@ -150,7 +146,7 @@ void BaseIndex::ThreadSync()
|
||||
LOCK(cs_main);
|
||||
const CBlockIndex* pindex_next = NextSyncBlock(pindex, m_chainstate->m_chain);
|
||||
if (!pindex_next) {
|
||||
m_best_block_index = pindex;
|
||||
SetBestBlockIndex(pindex);
|
||||
m_synced = true;
|
||||
// No need to handle errors in Commit. See rationale above.
|
||||
Commit();
|
||||
@ -172,7 +168,7 @@ void BaseIndex::ThreadSync()
|
||||
}
|
||||
|
||||
if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) {
|
||||
m_best_block_index = pindex;
|
||||
SetBestBlockIndex(pindex);
|
||||
last_locator_write_time = current_time;
|
||||
// No need to handle errors in Commit. See rationale above.
|
||||
Commit();
|
||||
@ -230,10 +226,10 @@ bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_ti
|
||||
// out of sync may be possible but a users fault.
|
||||
// In case we reorg beyond the pruned depth, ReadBlockFromDisk would
|
||||
// throw and lead to a graceful shutdown
|
||||
m_best_block_index = new_tip;
|
||||
SetBestBlockIndex(new_tip);
|
||||
if (!Commit()) {
|
||||
// If commit fails, revert the best block index to avoid corruption.
|
||||
m_best_block_index = current_tip;
|
||||
SetBestBlockIndex(current_tip);
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -274,7 +270,7 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const
|
||||
}
|
||||
|
||||
if (WriteBlock(*block, pindex)) {
|
||||
m_best_block_index = pindex;
|
||||
SetBestBlockIndex(pindex);
|
||||
} else {
|
||||
FatalError("%s: Failed to write block %s to index",
|
||||
__func__, pindex->GetBlockHash().ToString());
|
||||
@ -381,3 +377,14 @@ IndexSummary BaseIndex::GetSummary() const
|
||||
summary.best_block_height = m_best_block_index ? m_best_block_index.load()->nHeight : 0;
|
||||
return summary;
|
||||
}
|
||||
|
||||
void BaseIndex::SetBestBlockIndex(const CBlockIndex* block) {
|
||||
assert(!fPruneMode || AllowPrune());
|
||||
|
||||
m_best_block_index = block;
|
||||
if (AllowPrune() && block) {
|
||||
PruneLockInfo prune_lock;
|
||||
prune_lock.height_first = block->nHeight;
|
||||
WITH_LOCK(::cs_main, m_chainstate->m_blockman.UpdatePruneLock(GetName(), prune_lock));
|
||||
}
|
||||
}
|
||||
|
||||
@ -81,6 +81,9 @@ private:
|
||||
/// to a chain reorganization), the index must halt until Commit succeeds or else it could end up
|
||||
/// getting corrupted.
|
||||
bool Commit();
|
||||
|
||||
virtual bool AllowPrune() const = 0;
|
||||
|
||||
protected:
|
||||
CChainState* m_chainstate{nullptr};
|
||||
|
||||
@ -109,6 +112,9 @@ protected:
|
||||
/// Get the name of the index for display in logs.
|
||||
virtual const char* GetName() const = 0;
|
||||
|
||||
/// Update the internal best block index as well as the prune lock.
|
||||
void SetBestBlockIndex(const CBlockIndex* block);
|
||||
|
||||
public:
|
||||
/// Destructor interrupts sync thread if running and blocks until it exits.
|
||||
virtual ~BaseIndex();
|
||||
|
||||
@ -39,6 +39,8 @@ private:
|
||||
/** cache of block hash to filter header, to avoid disk access when responding to getcfcheckpt. */
|
||||
std::unordered_map<uint256, uint256, FilterHeaderHasher> m_headers_cache GUARDED_BY(m_cs_headers_cache);
|
||||
|
||||
bool AllowPrune() const override { return true; }
|
||||
|
||||
protected:
|
||||
bool Init() override;
|
||||
|
||||
|
||||
@ -36,6 +36,8 @@ private:
|
||||
|
||||
bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex);
|
||||
|
||||
bool AllowPrune() const override { return true; }
|
||||
|
||||
protected:
|
||||
bool Init() override;
|
||||
|
||||
|
||||
@ -20,6 +20,8 @@ protected:
|
||||
private:
|
||||
const std::unique_ptr<DB> m_db;
|
||||
|
||||
bool AllowPrune() const override { return false; }
|
||||
|
||||
protected:
|
||||
bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override;
|
||||
|
||||
|
||||
@ -538,7 +538,7 @@ void SetupServerArgs(ArgsManager& argsman)
|
||||
-GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
|
||||
argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
|
||||
argsman.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
|
||||
argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex, -coinstatsindex, -rescan and -disablegovernance=false. "
|
||||
argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex, -rescan and -disablegovernance=false. "
|
||||
"Warning: Reverting this setting requires re-downloading the entire blockchain. "
|
||||
"(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
|
||||
argsman.AddArg("-settings=<file>", strprintf("Specify path to dynamic settings data file. Can be disabled with -nosettings. File is written at runtime and not meant to be edited by users (use %s instead for custom settings). Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME, BITCOIN_SETTINGS_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
|
||||
@ -1159,8 +1159,6 @@ bool AppInitParameterInteraction(const ArgsManager& args)
|
||||
if (args.GetArg("-prune", 0)) {
|
||||
if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX))
|
||||
return InitError(_("Prune mode is incompatible with -txindex."));
|
||||
if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX))
|
||||
return InitError(_("Prune mode is incompatible with -coinstatsindex."));
|
||||
if (args.GetBoolArg("-reindex-chainstate", false)) {
|
||||
return InitError(_("Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead."));
|
||||
}
|
||||
|
||||
@ -24,6 +24,7 @@
|
||||
#include <walletinitinterface.h>
|
||||
|
||||
#include <map>
|
||||
#include <unordered_map>
|
||||
|
||||
std::atomic_bool fImporting(false);
|
||||
std::atomic_bool fReindex(false);
|
||||
@ -249,6 +250,11 @@ void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPr
|
||||
nLastBlockWeCanPrune, count);
|
||||
}
|
||||
|
||||
void BlockManager::UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) {
|
||||
AssertLockHeld(::cs_main);
|
||||
m_prune_locks[name] = lock_info;
|
||||
}
|
||||
|
||||
CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash)
|
||||
{
|
||||
AssertLockHeld(cs_main);
|
||||
@ -421,6 +427,16 @@ bool BlockManager::IsBlockPruned(const CBlockIndex* pblockindex)
|
||||
return (m_have_pruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0);
|
||||
}
|
||||
|
||||
const CBlockIndex* GetFirstStoredBlock(const CBlockIndex* start_block) {
|
||||
AssertLockHeld(::cs_main);
|
||||
assert(start_block);
|
||||
const CBlockIndex* last_block = start_block;
|
||||
while (last_block->pprev && (last_block->pprev->nStatus & BLOCK_HAVE_DATA)) {
|
||||
last_block = last_block->pprev;
|
||||
}
|
||||
return last_block;
|
||||
}
|
||||
|
||||
// If we're using -prune with -reindex, then delete block files that will be ignored by the
|
||||
// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile
|
||||
// is missing, do the same here to delete any later block files after a gap. Also delete all
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
#include <txdb.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
extern RecursiveMutex cs_main;
|
||||
@ -77,6 +78,10 @@ struct CBlockIndexHeightOnlyComparator {
|
||||
bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const;
|
||||
};
|
||||
|
||||
struct PruneLockInfo {
|
||||
int height_first{std::numeric_limits<int>::max()}; //! Height of earliest block that should be kept and not pruned
|
||||
};
|
||||
|
||||
/**
|
||||
* Maintains a tree of blocks (stored in `m_block_index`) which is consulted
|
||||
* to determine where the most-work tip is.
|
||||
@ -137,6 +142,14 @@ private:
|
||||
/** Dirty block file entries. */
|
||||
std::set<int> m_dirty_fileinfo;
|
||||
|
||||
/**
|
||||
* Map from external index name to oldest block that must not be pruned.
|
||||
*
|
||||
* @note Internally, only blocks at height (height_first - PRUNE_LOCK_BUFFER - 1) and
|
||||
* below will be pruned, but callers should avoid assuming any particular buffer size.
|
||||
*/
|
||||
std::unordered_map<std::string, PruneLockInfo> m_prune_locks GUARDED_BY(::cs_main);
|
||||
|
||||
public:
|
||||
BlockMap m_block_index GUARDED_BY(cs_main);
|
||||
PrevBlockMap m_prev_block_index GUARDED_BY(cs_main);
|
||||
@ -185,8 +198,14 @@ public:
|
||||
|
||||
//! Check whether the block associated with this index entry is pruned or not.
|
||||
bool IsBlockPruned(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
|
||||
|
||||
//! Create or update a prune lock identified by its name
|
||||
void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
|
||||
};
|
||||
|
||||
//! Find the first block that is not pruned
|
||||
const CBlockIndex* GetFirstStoredBlock(const CBlockIndex* start_block) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
|
||||
|
||||
void CleanupBlockRevFiles();
|
||||
|
||||
/** Open a block file (blk?????.dat) */
|
||||
|
||||
@ -1361,10 +1361,9 @@ static RPCHelpMan pruneblockchain()
|
||||
|
||||
PruneBlockFilesManual(active_chainstate, height);
|
||||
const CBlockIndex* block = CHECK_NONFATAL(active_chain.Tip());
|
||||
while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) {
|
||||
block = block->pprev;
|
||||
}
|
||||
return uint64_t(block->nHeight);
|
||||
const CBlockIndex* last_block = GetFirstStoredBlock(block);
|
||||
|
||||
return static_cast<uint64_t>(last_block->nHeight);
|
||||
},
|
||||
};
|
||||
}
|
||||
@ -1806,11 +1805,7 @@ RPCHelpMan getblockchaininfo()
|
||||
obj.pushKV("pruned", fPruneMode);
|
||||
if (fPruneMode) {
|
||||
const CBlockIndex* block = CHECK_NONFATAL(tip);
|
||||
while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) {
|
||||
block = block->pprev;
|
||||
}
|
||||
|
||||
obj.pushKV("pruneheight", block->nHeight);
|
||||
obj.pushKV("pruneheight", GetFirstStoredBlock(block)->nHeight);
|
||||
|
||||
// if 0, execution bypasses the whole if block.
|
||||
bool automatic_pruning{args.GetArg("-prune", 0) != 1};
|
||||
|
||||
@ -19,7 +19,6 @@
|
||||
#include <deploymentstatus.h>
|
||||
#include <flatfile.h>
|
||||
#include <hash.h>
|
||||
#include <index/blockfilterindex.h>
|
||||
#include <logging.h>
|
||||
#include <logging/timer.h>
|
||||
#include <node/blockstorage.h>
|
||||
@ -93,6 +92,12 @@ const std::vector<std::string> CHECKLEVEL_DOC {
|
||||
"level 4 tries to reconnect the blocks",
|
||||
"each level includes the checks of the previous levels",
|
||||
};
|
||||
/** The number of blocks to keep below the deepest prune lock.
|
||||
* There is nothing special about this number. It is higher than what we
|
||||
* expect to see in regular mainnet reorgs, but not so high that it would
|
||||
* noticeably interfere with the pruning mechanism.
|
||||
* */
|
||||
static constexpr int PRUNE_LOCK_BUFFER{10};
|
||||
|
||||
/**
|
||||
* Mutex to guard access to validation specific variables, such as reading
|
||||
@ -2368,12 +2373,24 @@ bool CChainState::FlushStateToDisk(
|
||||
CoinsCacheSizeState cache_state = GetCoinsCacheSizeState();
|
||||
LOCK(m_blockman.cs_LastBlockFile);
|
||||
if (fPruneMode && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && !fReindex) {
|
||||
// make sure we don't prune above the blockfilterindexes bestblocks
|
||||
// make sure we don't prune above any of the prune locks bestblocks
|
||||
// pruning is height-based
|
||||
int last_prune = m_chain.Height(); // last height we can prune
|
||||
ForEachBlockFilterIndex([&](BlockFilterIndex& index) {
|
||||
last_prune = std::max(1, std::min(last_prune, index.GetSummary().best_block_height));
|
||||
});
|
||||
int last_prune{m_chain.Height()}; // last height we can prune
|
||||
std::optional<std::string> limiting_lock; // prune lock that actually was the limiting factor, only used for logging
|
||||
|
||||
for (const auto& prune_lock : m_blockman.m_prune_locks) {
|
||||
if (prune_lock.second.height_first == std::numeric_limits<int>::max()) continue;
|
||||
// Remove the buffer and one additional block here to get actual height that is outside of the buffer
|
||||
const int lock_height{prune_lock.second.height_first - PRUNE_LOCK_BUFFER - 1};
|
||||
last_prune = std::max(1, std::min(last_prune, lock_height));
|
||||
if (last_prune == lock_height) {
|
||||
limiting_lock = prune_lock.first;
|
||||
}
|
||||
}
|
||||
|
||||
if (limiting_lock) {
|
||||
LogPrint(BCLog::PRUNE, "%s limited pruning to height %d\n", limiting_lock.value(), last_prune);
|
||||
}
|
||||
|
||||
if (nManualPruneHeight > 0) {
|
||||
LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune (manual)", BCLog::BENCHMARK);
|
||||
@ -2628,6 +2645,18 @@ bool CChainState::DisconnectTip(BlockValidationState& state, DisconnectedBlockTr
|
||||
dbTx->Commit();
|
||||
}
|
||||
LogPrint(BCLog::BENCHMARK, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * MILLI);
|
||||
|
||||
{
|
||||
// Prune locks that began at or after the tip should be moved backward so they get a chance to reorg
|
||||
const int max_height_first{pindexDelete->nHeight - 1};
|
||||
for (auto& prune_lock : m_blockman.m_prune_locks) {
|
||||
if (prune_lock.second.height_first <= max_height_first) continue;
|
||||
|
||||
prune_lock.second.height_first = max_height_first;
|
||||
LogPrint(BCLog::PRUNE, "%s prune lock moved back to %d\n", prune_lock.first, max_height_first);
|
||||
}
|
||||
}
|
||||
|
||||
// Write the chain state to disk, if necessary.
|
||||
if (!FlushStateToDisk(state, FlushStateMode::IF_NEEDED)) {
|
||||
return false;
|
||||
|
||||
@ -1,80 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2020 The Bitcoin Core developers
|
||||
# Distributed under the MIT software license, see the accompanying
|
||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
"""Test blockfilterindex in conjunction with prune."""
|
||||
from test_framework.test_framework import BitcoinTestFramework
|
||||
from test_framework.util import (
|
||||
assert_equal,
|
||||
assert_greater_than,
|
||||
assert_raises_rpc_error,
|
||||
)
|
||||
from test_framework.governance import EXPECTED_STDERR_NO_GOV_PRUNE
|
||||
|
||||
|
||||
class FeatureBlockfilterindexPruneTest(BitcoinTestFramework):
|
||||
def set_test_params(self):
|
||||
self.num_nodes = 1
|
||||
self.extra_args = [["-fastprune", "-prune=1", "-blockfilterindex=1", "-testactivationheight=v20@2000"]]
|
||||
|
||||
def sync_index(self, height):
|
||||
expected = {'basic block filter index': {'synced': True, 'best_block_height': height}}
|
||||
self.wait_until(lambda: self.nodes[0].getindexinfo() == expected)
|
||||
|
||||
def run_test(self):
|
||||
self.log.info("check if we can access a blockfilter when pruning is enabled but no blocks are actually pruned")
|
||||
self.sync_index(height=200)
|
||||
assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0)
|
||||
self.generate(self.nodes[0], 500)
|
||||
self.sync_index(height=700)
|
||||
|
||||
self.log.info("prune some blocks")
|
||||
pruneheight = self.nodes[0].pruneblockchain(400)
|
||||
# the prune heights used here and below are magic numbers that are determined by the
|
||||
# thresholds at which block files wrap, so they depend on disk serialization and default block file size.
|
||||
assert_equal(pruneheight, 366)
|
||||
|
||||
self.log.info("check if we can access the tips blockfilter when we have pruned some blocks")
|
||||
assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0)
|
||||
|
||||
self.log.info("check if we can access the blockfilter of a pruned block")
|
||||
assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getblockhash(2))['filter']), 0)
|
||||
|
||||
# mine and sync index up to a height that will later be the pruneheight
|
||||
self.generate(self.nodes[0], 298)
|
||||
self.sync_index(height=998)
|
||||
|
||||
self.log.info("start node without blockfilterindex")
|
||||
self.restart_node(0, extra_args=["-fastprune", "-prune=1", "-testactivationheight=v20@2000"], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
|
||||
|
||||
self.log.info("make sure accessing the blockfilters throws an error")
|
||||
assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic", self.nodes[0].getblockfilter, self.nodes[0].getblockhash(2))
|
||||
self.generate(self.nodes[0], 502)
|
||||
|
||||
self.log.info("prune exactly up to the blockfilterindexes best block while blockfilters are disabled")
|
||||
pruneheight_2 = self.nodes[0].pruneblockchain(1000)
|
||||
assert_equal(pruneheight_2, 932)
|
||||
self.restart_node(0, extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1", "-testactivationheight=v20@2000"], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
|
||||
self.log.info("make sure that we can continue with the partially synced index after having pruned up to the index height")
|
||||
self.sync_index(height=1500)
|
||||
|
||||
self.log.info("prune below the blockfilterindexes best block while blockfilters are disabled")
|
||||
self.restart_node(0, extra_args=["-fastprune", "-prune=1", "-testactivationheight=v20@2000"], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
|
||||
self.generate(self.nodes[0], 1000)
|
||||
pruneheight_3 = self.nodes[0].pruneblockchain(2000)
|
||||
assert_greater_than(pruneheight_3, pruneheight_2)
|
||||
self.stop_node(0, expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
|
||||
|
||||
self.log.info("make sure we get an init error when starting the node again with block filters")
|
||||
self.nodes[0].assert_start_raises_init_error(
|
||||
extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1"],
|
||||
expected_msg=f"{EXPECTED_STDERR_NO_GOV_PRUNE}\nError: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)",
|
||||
)
|
||||
|
||||
self.log.info("make sure the node starts again with the -reindex arg")
|
||||
self.start_node(0, extra_args=["-fastprune", "-prune=1", "-blockfilterindex", "-reindex"])
|
||||
self.stop_nodes(expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
FeatureBlockfilterindexPruneTest().main()
|
||||
161
test/functional/feature_index_prune.py
Executable file
161
test/functional/feature_index_prune.py
Executable file
@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2020 The Bitcoin Core developers
|
||||
# Distributed under the MIT software license, see the accompanying
|
||||
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
"""Test indices in conjunction with prune."""
|
||||
from test_framework.test_framework import BitcoinTestFramework
|
||||
from test_framework.util import (
|
||||
assert_equal,
|
||||
assert_greater_than,
|
||||
assert_raises_rpc_error,
|
||||
p2p_port,
|
||||
)
|
||||
from test_framework.governance import EXPECTED_STDERR_NO_GOV_PRUNE
|
||||
|
||||
DEPLOYMENT_ARG = "-testactivationheight=v20@3000"
|
||||
|
||||
class FeatureIndexPruneTest(BitcoinTestFramework):
|
||||
def set_test_params(self):
|
||||
self.num_nodes = 4
|
||||
self.extra_args = [
|
||||
["-fastprune", "-prune=1", "-blockfilterindex=1", DEPLOYMENT_ARG],
|
||||
["-fastprune", "-prune=1", "-coinstatsindex=1", DEPLOYMENT_ARG],
|
||||
["-fastprune", "-prune=1", "-blockfilterindex=1", "-coinstatsindex=1", DEPLOYMENT_ARG],
|
||||
[DEPLOYMENT_ARG]
|
||||
]
|
||||
|
||||
def sync_index(self, height):
|
||||
expected_filter = {
|
||||
'basic block filter index': {'synced': True, 'best_block_height': height},
|
||||
}
|
||||
self.wait_until(lambda: self.nodes[0].getindexinfo() == expected_filter)
|
||||
|
||||
expected_stats = {
|
||||
'coinstatsindex': {'synced': True, 'best_block_height': height}
|
||||
}
|
||||
self.wait_until(lambda: self.nodes[1].getindexinfo() == expected_stats)
|
||||
|
||||
expected = {**expected_filter, **expected_stats}
|
||||
self.wait_until(lambda: self.nodes[2].getindexinfo() == expected)
|
||||
|
||||
def reconnect_nodes(self):
|
||||
self.connect_nodes(0,1)
|
||||
self.connect_nodes(0,2)
|
||||
self.connect_nodes(0,3)
|
||||
|
||||
def mine_batches(self, blocks):
|
||||
n = blocks // 250
|
||||
for _ in range(n):
|
||||
self.generate(self.nodes[0], 250)
|
||||
self.generate(self.nodes[0], blocks % 250)
|
||||
self.sync_blocks()
|
||||
|
||||
def restart_without_indices(self):
|
||||
for i in range(3):
|
||||
self.restart_node(i, extra_args=["-fastprune", "-prune=1", DEPLOYMENT_ARG], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
|
||||
self.reconnect_nodes()
|
||||
|
||||
def run_test(self):
|
||||
filter_nodes = [self.nodes[0], self.nodes[2]]
|
||||
stats_nodes = [self.nodes[1], self.nodes[2]]
|
||||
|
||||
self.log.info("check if we can access blockfilters and coinstats when pruning is enabled but no blocks are actually pruned")
|
||||
self.sync_index(height=200)
|
||||
tip = self.nodes[0].getbestblockhash()
|
||||
for node in filter_nodes:
|
||||
assert_greater_than(len(node.getblockfilter(tip)['filter']), 0)
|
||||
for node in stats_nodes:
|
||||
assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash'])
|
||||
|
||||
self.mine_batches(500)
|
||||
self.sync_index(height=700)
|
||||
|
||||
self.log.info("prune some blocks")
|
||||
for node in self.nodes[:2]:
|
||||
with node.assert_debug_log(['limited pruning to height 689']):
|
||||
pruneheight_new = node.pruneblockchain(400)
|
||||
# the prune heights used here and below are magic numbers that are determined by the
|
||||
# thresholds at which block files wrap, so they depend on disk serialization and default block file size.
|
||||
assert_equal(pruneheight_new, 366)
|
||||
|
||||
self.log.info("check if we can access the tips blockfilter and coinstats when we have pruned some blocks")
|
||||
tip = self.nodes[0].getbestblockhash()
|
||||
for node in filter_nodes:
|
||||
assert_greater_than(len(node.getblockfilter(tip)['filter']), 0)
|
||||
for node in stats_nodes:
|
||||
assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash'])
|
||||
|
||||
self.log.info("check if we can access the blockfilter and coinstats of a pruned block")
|
||||
height_hash = self.nodes[0].getblockhash(2)
|
||||
for node in filter_nodes:
|
||||
assert_greater_than(len(node.getblockfilter(height_hash)['filter']), 0)
|
||||
for node in stats_nodes:
|
||||
assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=height_hash)['muhash'])
|
||||
|
||||
# mine and sync index up to a height that will later be the pruneheight
|
||||
self.generate(self.nodes[0], 298)
|
||||
self.sync_index(height=998)
|
||||
|
||||
self.restart_without_indices()
|
||||
|
||||
self.log.info("make sure trying to access the indices throws errors")
|
||||
for node in filter_nodes:
|
||||
msg = "Index is not enabled for filtertype basic"
|
||||
assert_raises_rpc_error(-1, msg, node.getblockfilter, height_hash)
|
||||
for node in stats_nodes:
|
||||
msg = "Querying specific block heights requires coinstatsindex"
|
||||
assert_raises_rpc_error(-8, msg, node.gettxoutsetinfo, "muhash", height_hash)
|
||||
|
||||
self.mine_batches(502)
|
||||
|
||||
self.log.info("prune exactly up to the indices best blocks while the indices are disabled")
|
||||
for i in range(3):
|
||||
pruneheight_2 = self.nodes[i].pruneblockchain(1000)
|
||||
assert_equal(pruneheight_2, 932)
|
||||
# Restart the nodes again with the indices activated
|
||||
self.restart_node(i, extra_args=self.extra_args[i], expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
|
||||
|
||||
self.log.info("make sure that we can continue with the partially synced indices after having pruned up to the index height")
|
||||
self.sync_index(height=1500)
|
||||
|
||||
self.log.info("prune further than the indices best blocks while the indices are disabled")
|
||||
self.restart_without_indices()
|
||||
self.mine_batches(1000)
|
||||
|
||||
for i in range(3):
|
||||
pruneheight_3 = self.nodes[i].pruneblockchain(2000)
|
||||
assert_greater_than(pruneheight_3, pruneheight_2)
|
||||
self.stop_node(i, expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
|
||||
|
||||
self.log.info("make sure we get an init error when starting the nodes again with the indices")
|
||||
filter_msg = "Error: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"
|
||||
stats_msg = "Error: coinstatsindex best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"
|
||||
for i, msg in enumerate([filter_msg, stats_msg, filter_msg]):
|
||||
self.nodes[i].assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=f"{EXPECTED_STDERR_NO_GOV_PRUNE}\n{msg}")
|
||||
|
||||
self.log.info("make sure the nodes start again with the indices and an additional -reindex arg")
|
||||
ip_port = "127.0.0.1:" + str(p2p_port(3))
|
||||
for i in range(3):
|
||||
# The nodes need to be reconnected to the non-pruning node upon restart, otherwise they will be stuck
|
||||
restart_args = self.extra_args[i]+["-reindex", f"-connect={ip_port}"]
|
||||
self.restart_node(i, extra_args=restart_args, expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE)
|
||||
|
||||
self.sync_blocks(timeout=300)
|
||||
|
||||
for node in self.nodes[:2]:
|
||||
with node.assert_debug_log(['limited pruning to height 2489']):
|
||||
pruneheight_new = node.pruneblockchain(2500)
|
||||
assert_equal(pruneheight_new, 2197)
|
||||
|
||||
self.log.info("ensure that prune locks don't prevent indices from failing in a reorg scenario")
|
||||
with self.nodes[0].assert_debug_log(['basic block filter index prune lock moved back to 2480']):
|
||||
self.nodes[3].invalidateblock(self.nodes[0].getblockhash(2480))
|
||||
self.generate(self.nodes[3], 30)
|
||||
self.sync_blocks()
|
||||
|
||||
for idx in range(self.num_nodes):
|
||||
self.nodes[idx].stop_node(expected_stderr=EXPECTED_STDERR_NO_GOV_PRUNE if idx != 3 else "")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
FeatureIndexPruneTest().main()
|
||||
@ -149,10 +149,6 @@ class PruneTest(BitcoinTestFramework):
|
||||
expected_msg='Error: Prune mode is incompatible with -txindex.',
|
||||
extra_args=['-prune=550', '-txindex'],
|
||||
)
|
||||
self.nodes[0].assert_start_raises_init_error(
|
||||
expected_msg='Error: Prune mode is incompatible with -coinstatsindex.',
|
||||
extra_args=['-prune=550', '-coinstatsindex'],
|
||||
)
|
||||
self.nodes[0].assert_start_raises_init_error(
|
||||
expected_msg='Error: Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead.',
|
||||
extra_args=['-prune=550', '-reindex-chainstate'],
|
||||
|
||||
@ -87,6 +87,7 @@ EXTENDED_SCRIPTS = [
|
||||
# Longest test should go first, to favor running tests in parallel
|
||||
'feature_pruning.py', # NOTE: Prune mode is incompatible with -txindex, should work with governance validation disabled though.
|
||||
'feature_dbcrash.py',
|
||||
'feature_index_prune.py',
|
||||
]
|
||||
|
||||
BASE_SCRIPTS = [
|
||||
@ -365,7 +366,6 @@ BASE_SCRIPTS = [
|
||||
'rpc_help.py',
|
||||
'feature_dirsymlinks.py',
|
||||
'feature_help.py',
|
||||
'feature_blockfilterindex_prune.py'
|
||||
# Don't append tests at the end to avoid merge conflicts
|
||||
# Put them in a random line within the section that fits their approximate run-time
|
||||
]
|
||||
|
||||
@ -11,8 +11,6 @@ export LC_ALL=C
|
||||
EXPECTED_CIRCULAR_DEPENDENCIES=(
|
||||
"chainparamsbase -> util/system -> chainparamsbase"
|
||||
"node/blockstorage -> validation -> node/blockstorage"
|
||||
"index/blockfilterindex -> node/blockstorage -> validation -> index/blockfilterindex"
|
||||
"index/base -> validation -> index/blockfilterindex -> index/base"
|
||||
"index/coinstatsindex -> node/coinstats -> index/coinstatsindex"
|
||||
"policy/fees -> txmempool -> policy/fees"
|
||||
"qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user