sikkienl/dash
1
0
Fork 0
mirror of https://github.com/dashpay/dash.git synced 2024-12-26 12:32:48 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge #7871: Manual block file pruning.

Browse Source 
afffeea fixup! Add pruneblockchain RPC to enable manual block file pruning. (Russell Yanofsky)
1fc4ec7 Add pruneblockchain RPC to enable manual block file pruning. (mrbandrews)
This commit is contained in:
Wladimir J. van der Laan 2017-01-11 14:16:11 +01:00
commit e2e624d9ce
No known key found for this signature in database
GPG Key ID: 74810B012346C9A6
8 changed files with 232 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
doc/man/bitcoin-qt.1
View File

@ -75,11 +75,13 @@ Specify pid file (default: bitcoind.pid)
.HP
\fB\-prune=\fR<n>
.IP
Reduce storage requirements by pruning (deleting) old blocks. This mode
is incompatible with \fB\-txindex\fR and \fB\-rescan\fR. Warning: Reverting
this setting requires re\-downloading the entire blockchain.
(default: 0 = disable pruning blocks, >550 = target size in MiB
to use for block files)
Reduce storage requirements by enabling pruning (deleting) of old blocks.
This allows the pruneblockchain RPC to be called to delete specific blocks,
and enables automatic pruning of old blocks if a target size in MiB is
provided. This mode is incompatible with \fB\-txindex\fR and \fB\-rescan\fR.
Warning: Reverting this setting requires re\-downloading the entire blockchain.
(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >550 =
automatically prune block files to stay under the specified target size in MiB)
.HP
\fB\-reindex\-chainstate\fR
.IP

12
doc/man/bitcoind.1
View File

@ -80,11 +80,13 @@ Specify pid file (default: bitcoind.pid)
.HP
\fB\-prune=\fR<n>
.IP
Reduce storage requirements by pruning (deleting) old blocks. This mode
is incompatible with \fB\-txindex\fR and \fB\-rescan\fR. Warning: Reverting
this setting requires re\-downloading the entire blockchain.
(default: 0 = disable pruning blocks, >550 = target size in MiB
to use for block files)
Reduce storage requirements by enabling pruning (deleting) of old blocks.
This allows the pruneblockchain RPC to be called to delete specific blocks,
and enables automatic pruning of old blocks if a target size in MiB is
provided. This mode is incompatible with \fB\-txindex\fR and \fB\-rescan\fR.
Warning: Reverting this setting requires re\-downloading the entire blockchain.
(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >550 =
automatically prune block files to stay under the specified target size in MiB)
.HP
\fB\-reindex\-chainstate\fR
.IP

128
qa/rpc-tests/pruning.py
View File

@ -25,7 +25,7 @@ class PruneTest(BitcoinTestFramework):
def __init__(self):
super().__init__()
self.setup_clean_chain = True
self.num_nodes = 3
self.num_nodes = 6
# Cache for utxos, as the listunspent may take a long time later in the test
self.utxo_cache_0 = []
@ -43,10 +43,22 @@ class PruneTest(BitcoinTestFramework):
self.nodes.append(start_node(2, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-prune=550"], timewait=900))
self.prunedir = self.options.tmpdir+"/node2/regtest/blocks/"
# Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later)
self.nodes.append(start_node(3, self.options.tmpdir, ["-debug=0","-maxreceivebuffer=20000","-blockmaxsize=999000"], timewait=900))
self.nodes.append(start_node(4, self.options.tmpdir, ["-debug=0","-maxreceivebuffer=20000","-blockmaxsize=999000"], timewait=900))
# Create nodes 5 to test wallet in prune mode, but do not connect
self.nodes.append(start_node(5, self.options.tmpdir, ["-debug=0", "-prune=550"]))
# Determine default relay fee
self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"]
connect_nodes(self.nodes[0], 1)
connect_nodes(self.nodes[1], 2)
connect_nodes(self.nodes[2], 0)
sync_blocks(self.nodes[0:3])
connect_nodes(self.nodes[0], 3)
connect_nodes(self.nodes[0], 4)
sync_blocks(self.nodes[0:5])
def create_big_chain(self):
# Start by creating some coinbases we can spend later
@ -57,7 +69,7 @@ class PruneTest(BitcoinTestFramework):
for i in range(645):
mine_large_block(self.nodes[0], self.utxo_cache_0)
sync_blocks(self.nodes[0:3])
sync_blocks(self.nodes[0:5])
def test_height_min(self):
if not os.path.isfile(self.prunedir+"blk00000.dat"):
@ -212,6 +224,103 @@ class PruneTest(BitcoinTestFramework):
# Verify we can now have the data for a block previously pruned
assert(self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight)
def manual_test(self, node_number, use_timestamp):
# at this point, node has 995 blocks and has not yet run in prune mode
node = self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-debug=0"], timewait=900)
assert_equal(node.getblockcount(), 995)
assert_raises_message(JSONRPCException, "not in prune mode", node.pruneblockchain, 500)
stop_node(node, node_number)
# now re-start in manual pruning mode
node = self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-debug=0","-prune=1"], timewait=900)
assert_equal(node.getblockcount(), 995)
def height(index):
if use_timestamp:
return node.getblockheader(node.getblockhash(index))["time"]
else:
return index
def has_block(index):
return os.path.isfile(self.options.tmpdir + "/node{}/regtest/blocks/blk{:05}.dat".format(node_number, index))
# should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000)
assert_raises_message(JSONRPCException, "Blockchain is too short for pruning", node.pruneblockchain, height(500))
# mine 6 blocks so we are at height 1001 (i.e., above PruneAfterHeight)
node.generate(6)
# negative and zero inputs should raise an exception
try:
node.pruneblockchain(-10)
raise AssertionError("pruneblockchain(-10) should have failed.")
except:
pass
try:
node.pruneblockchain(0)
raise AssertionError("pruneblockchain(0) should have failed.")
except:
pass
# height=100 too low to prune first block file so this is a no-op
node.pruneblockchain(height(100))
if not has_block(0):
raise AssertionError("blk00000.dat is missing when should still be there")
# height=500 should prune first file
node.pruneblockchain(height(500))
if has_block(0):
raise AssertionError("blk00000.dat is still there, should be pruned by now")
if not has_block(1):
raise AssertionError("blk00001.dat is missing when should still be there")
# height=650 should prune second file
node.pruneblockchain(height(650))
if has_block(1):
raise AssertionError("blk00001.dat is still there, should be pruned by now")
# height=1000 should not prune anything more, because tip-288 is in blk00002.dat.
node.pruneblockchain(height(1000))
if not has_block(2):
raise AssertionError("blk00002.dat is still there, should be pruned by now")
# advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat)
node.generate(288)
node.pruneblockchain(height(1000))
if has_block(2):
raise AssertionError("blk00002.dat is still there, should be pruned by now")
if has_block(3):
raise AssertionError("blk00003.dat is still there, should be pruned by now")
# stop node, start back up with auto-prune at 550MB, make sure still runs
stop_node(node, node_number)
self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-debug=0","-prune=550"], timewait=900)
print("Success")
def wallet_test(self):
# check that the pruning node's wallet is still in good shape
print("Stop and start pruning node to trigger wallet rescan")
try:
stop_node(self.nodes[2], 2)
start_node(2, self.options.tmpdir, ["-debug=1","-prune=550"])
print("Success")
except Exception as detail:
raise AssertionError("Wallet test: unable to re-start the pruning node")
# check that wallet loads loads successfully when restarting a pruned node after IBD.
# this was reported to fail in #7494.
print ("Syncing node 5 to test wallet")
connect_nodes(self.nodes[0], 5)
nds = [self.nodes[0], self.nodes[5]]
sync_blocks(nds)
try:
stop_node(self.nodes[5],5) #stop and start to trigger rescan
start_node(5, self.options.tmpdir, ["-debug=1","-prune=550"])
print ("Success")
except Exception as detail:
raise AssertionError("Wallet test: unable to re-start node5")
def run_test(self):
print("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)")
@ -226,6 +335,10 @@ class PruneTest(BitcoinTestFramework):
# Start by mining a simple chain that all nodes have
# N0=N1=N2 **...*(995)
# stop manual-pruning node with 995 blocks
stop_node(self.nodes[3],3)
stop_node(self.nodes[4],4)
print("Check that we haven't started pruning yet because we're below PruneAfterHeight")
self.test_height_min()
# Extend this chain past the PruneAfterHeight
@ -308,6 +421,15 @@ class PruneTest(BitcoinTestFramework):
#
# N1 doesn't change because 1033 on main chain (*) is invalid
print("Test manual pruning with block indices")
self.manual_test(3, use_timestamp=False)
print("Test manual pruning with timestamps")
self.manual_test(4, use_timestamp=True)
print("Test wallet re-scan")
self.wallet_test()
print("Done")
if __name__ == '__main__':

16
src/init.cpp
View File

@ -351,9 +351,9 @@ std::string HelpMessage(HelpMessageMode mode)
#ifndef WIN32
strUsage += HelpMessageOpt("-pid=<file>", strprintf(_("Specify pid file (default: %s)"), BITCOIN_PID_FILENAME));
#endif
strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by pruning (deleting) old blocks. This mode is incompatible with -txindex and -rescan. "
strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -rescan. "
"Warning: Reverting this setting requires re-downloading the entire blockchain. "
"(default: 0 = disable pruning blocks, >%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
"(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >%u = automatically prune block files to stay under the specified target size in MiB)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
strUsage += HelpMessageOpt("-reindex-chainstate", _("Rebuild chain state from the currently indexed blocks"));
strUsage += HelpMessageOpt("-reindex", _("Rebuild chain state and block index from the blk*.dat files on disk"));
#ifndef WIN32
@ -936,12 +936,16 @@ bool AppInitParameterInteraction()
nScriptCheckThreads = MAX_SCRIPTCHECK_THREADS;
// block pruning; get the amount of disk space (in MiB) to allot for block & undo files
int64_t nSignedPruneTarget = GetArg("-prune", 0) * 1024 * 1024;
if (nSignedPruneTarget < 0) {
int64_t nPruneArg = GetArg("-prune", 0);
if (nPruneArg < 0) {
return InitError(_("Prune cannot be configured with a negative value."));
}
nPruneTarget = (uint64_t) nSignedPruneTarget;
if (nPruneTarget) {
nPruneTarget = (uint64_t) nPruneArg * 1024 * 1024;
if (nPruneArg == 1) { // manual pruning: -prune=1
LogPrintf("Block pruning enabled. Use RPC call pruneblockchain(height) to manually prune block and undo files.\n");
nPruneTarget = std::numeric_limits<uint64_t>::max();
fPruneMode = true;
} else if (nPruneTarget) {
if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) {
return InitError(strprintf(_("Prune configured below the minimum of %d MiB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
}

41
src/rpc/blockchain.cpp
View File

@ -814,6 +814,46 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats)
return true;
}
UniValue pruneblockchain(const JSONRPCRequest& request)
{
if (request.fHelp || request.params.size() != 1)
throw runtime_error(
"pruneblockchain\n"
"\nArguments:\n"
"1. \"height\" (numeric, required) The block height to prune up to. May be set to a discrete height, or to a unix timestamp to prune based on block time.\n");
if (!fPruneMode)
throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Cannot prune blocks because node is not in prune mode.");
LOCK(cs_main);
int heightParam = request.params[0].get_int();
if (heightParam < 0)
throw JSONRPCError(RPC_INVALID_PARAMETER, "Negative block height.");
// Height value more than a billion is too high to be a block height, and
// too low to be a block time (corresponds to timestamp from Sep 2001).
if (heightParam > 1000000000) {
CBlockIndex* pindex = chainActive.FindLatestBefore(heightParam);
if (!pindex) {
throw JSONRPCError(RPC_INTERNAL_ERROR, "Could not find block before specified timestamp.");
}
heightParam = pindex->nHeight;
}
unsigned int height = (unsigned int) heightParam;
unsigned int chainHeight = (unsigned int) chainActive.Height();
if (chainHeight < Params().PruneAfterHeight())
throw JSONRPCError(RPC_INTERNAL_ERROR, "Blockchain is too short for pruning.");
else if (height > chainHeight)
throw JSONRPCError(RPC_INVALID_PARAMETER, "Blockchain is shorter than the attempted prune height.");
else if (height > chainHeight - MIN_BLOCKS_TO_KEEP)
LogPrint("rpc", "Attempt to prune blocks close to the tip. Retaining the minimum number of blocks.");
PruneBlockFilesManual(height);
return NullUniValue;
}
UniValue gettxoutsetinfo(const JSONRPCRequest& request)
{
if (request.fHelp || request.params.size() != 0)
@ -1384,6 +1424,7 @@ static const CRPCCommand commands[] =
{ "blockchain", "getrawmempool", &getrawmempool, true, {"verbose"} },
{ "blockchain", "gettxout", &gettxout, true, {"txid","n","include_mempool"} },
{ "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, true, {} },
{ "blockchain", "pruneblockchain", &pruneblockchain, true, {"height"} },
{ "blockchain", "verifychain", &verifychain, true, {"checklevel","nblocks"} },
{ "blockchain", "preciousblock", &preciousblock, true, {"blockhash"} },

1
src/rpc/client.cpp
View File

@ -103,6 +103,7 @@ static const CRPCConvertParam vRPCConvertParams[] =
{ "importmulti", 1, "options" },
{ "verifychain", 0, "checklevel" },
{ "verifychain", 1, "nblocks" },
{ "pruneblockchain", 0, "height" },
{ "keypoolrefill", 0, "newsize" },
{ "getrawmempool", 0, "verbose" },
{ "estimatefee", 0, "nblocks" },

44
src/validation.cpp
View File

@ -185,7 +185,8 @@ enum FlushStateMode {
};
// See definition for documentation
bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode);
bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int nManualPruneHeight=0);
void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight);
bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime)
{
@ -1934,7 +1935,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
* if they're too large, if it's been a while since the last write,
* or always and in all cases if we're in prune mode and are deleting files.
*/
bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int nManualPruneHeight) {
int64_t nMempoolUsage = mempool.DynamicMemoryUsage();
const CChainParams& chainparams = Params();
LOCK2(cs_main, cs_LastBlockFile);
@ -1944,9 +1945,13 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
std::set<int> setFilesToPrune;
bool fFlushForPrune = false;
try {
if (fPruneMode && fCheckForPruning && !fReindex) {
FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight());
fCheckForPruning = false;
if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) {
if (nManualPruneHeight > 0) {
FindFilesToPruneManual(setFilesToPrune, nManualPruneHeight);
} else {
FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight());
fCheckForPruning = false;
}
if (!setFilesToPrune.empty()) {
fFlushForPrune = true;
if (!fHavePruned) {
@ -3247,6 +3252,35 @@ void UnlinkPrunedFiles(std::set<int>& setFilesToPrune)
}
}
/* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */
void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight)
{
assert(fPruneMode && nManualPruneHeight > 0);
LOCK2(cs_main, cs_LastBlockFile);
if (chainActive.Tip() == NULL)
return;
// last block to prune is the lesser of (user-specified height, MIN_BLOCKS_TO_KEEP from the tip)
unsigned int nLastBlockWeCanPrune = min((unsigned)nManualPruneHeight, chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP);
int count=0;
for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) {
if (vinfoBlockFile[fileNumber].nSize == 0 || vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune)
continue;
PruneOneBlockFile(fileNumber);
setFilesToPrune.insert(fileNumber);
count++;
}
LogPrintf("Prune (Manual): prune_height=%d removed %d blk/rev pairs\n", nLastBlockWeCanPrune, count);
}
/* This function is called from the RPC code for pruneblockchain */
void PruneBlockFilesManual(int nManualPruneHeight)
{
CValidationState state;
FlushStateToDisk(state, FLUSH_STATE_NONE, nManualPruneHeight);
}
/* Calculate the block/rev files that should be deleted to remain under target*/
void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight)
{

2
src/validation.h
View File

@ -309,6 +309,8 @@ CBlockIndex * InsertBlockIndex(uint256 hash);
void FlushStateToDisk();
/** Prune block files and flush state to disk. */
void PruneAndFlush();
/** Prune block files up to a given height */
void PruneBlockFilesManual(int nPruneUpToHeight);
/** (try to) add transaction to memory pool **/
bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransactionRef &tx, bool fLimitFree,