sikkienl/dash
1
0
Fork 0
mirror of https://github.com/dashpay/dash.git synced 2024-12-25 03:52:49 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

bitcoin#15118: Refactor block file logic

Browse Source
This commit is contained in:
Kittywhiskers Van Gogh 2021-08-12 12:32:29 +05:30
parent 39e34e2b52
commit 4d3e3d7b07
13 changed files with 413 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Makefile.am
View File

@ -172,6 +172,7 @@ BITCOIN_CORE_H = \
governance/governance-votedb.h \
flat-database.h \
hdchain.h \
flatfile.h \
fs.h \
httprpc.h \
httpserver.h \
@ -336,6 +337,7 @@ libdash_server_a_SOURCES = \
evo/providertx.cpp \
evo/simplifiedmns.cpp \
evo/specialtx.cpp \
flatfile.cpp \
httprpc.cpp \
httpserver.cpp \
index/base.cpp \

1
src/Makefile.test.include
View File

@ -137,6 +137,7 @@ BITCOIN_TESTS =\
test/descriptor_tests.cpp \
test/evo_deterministicmns_tests.cpp \
test/evo_simplifiedmns_tests.cpp \
test/flatfile_tests.cpp \
test/fs_tests.cpp \
test/getarg_tests.cpp \
test/governance_validators_tests.cpp \

46
src/chain.h
View File

@ -8,6 +8,7 @@
#include <arith_uint256.h>
#include <consensus/params.h>
#include <flatfile.h>
#include <primitives/block.h>
#include <tinyformat.h>
#include <uint256.h>
@ -80,43 +81,6 @@ public:
}
};
struct CDiskBlockPos
{
int nFile;
unsigned int nPos;
SERIALIZE_METHODS(CDiskBlockPos, obj)
{
READWRITE(VARINT(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED), VARINT(obj.nPos));
}
CDiskBlockPos() {
SetNull();
}
CDiskBlockPos(int nFileIn, unsigned int nPosIn) {
nFile = nFileIn;
nPos = nPosIn;
}
friend bool operator==(const CDiskBlockPos &a, const CDiskBlockPos &b) {
return (a.nFile == b.nFile && a.nPos == b.nPos);
}
friend bool operator!=(const CDiskBlockPos &a, const CDiskBlockPos &b) {
return !(a == b);
}
void SetNull() { nFile = -1; nPos = 0; }
bool IsNull() const { return (nFile == -1); }
std::string ToString() const
{
return strprintf("CDiskBlockPos(nFile=%i, nPos=%i)", nFile, nPos);
}
};
enum BlockStatus: uint32_t {
//! Unused.
BLOCK_VALID_UNKNOWN = 0,
@ -227,8 +191,8 @@ public:
{
}
CDiskBlockPos GetBlockPos() const {
CDiskBlockPos ret;
FlatFilePos GetBlockPos() const {
FlatFilePos ret;
if (nStatus & BLOCK_HAVE_DATA) {
ret.nFile = nFile;
ret.nPos = nDataPos;
@ -236,8 +200,8 @@ public:
return ret;
}
CDiskBlockPos GetUndoPos() const {
CDiskBlockPos ret;
FlatFilePos GetUndoPos() const {
FlatFilePos ret;
if (nStatus & BLOCK_HAVE_UNDO) {
ret.nFile = nFile;
ret.nPos = nUndoPos;

6
src/coinjoin/coinjoin-client.cpp
View File

@ -35,7 +35,7 @@ void CCoinJoinClientQueueManager::ProcessMessage(CNode* pfrom, const std::string
if (!CCoinJoinClientOptions::IsEnabled()) return;
if (!masternodeSync.IsBlockchainSynced()) return;
if (!CheckDiskSpace()) {
if (!CheckDiskSpace(GetDataDir())) {
LogPrint(BCLog::COINJOIN, "CCoinJoinClientQueueManager::ProcessMessage -- Not enough disk space, disabling CoinJoin.\n");
return;
}
@ -128,7 +128,7 @@ void CCoinJoinClientManager::ProcessMessage(CNode* pfrom, const std::string& str
if (!CCoinJoinClientOptions::IsEnabled()) return;
if (!masternodeSync.IsBlockchainSynced()) return;
if (!CheckDiskSpace()) {
if (!CheckDiskSpace(GetDataDir())) {
ResetPool();
StopMixing();
LogPrint(BCLog::COINJOIN, "CCoinJoinClientManager::ProcessMessage -- Not enough disk space, disabling CoinJoin.\n");
@ -463,7 +463,7 @@ bool CCoinJoinClientSession::SendDenominate(const std::vector<std::pair<CTxDSIn,
return false;
}
if (!CheckDiskSpace()) {
if (!CheckDiskSpace(GetDataDir())) {
UnlockCoins();
keyHolderStorage.ReturnAll();
SetNull();

98
src/flatfile.cpp Normal file
View File

@ -0,0 +1,98 @@
// Copyright (c) 2009-2010 Satoshi Nakamoto
// Copyright (c) 2009-2019 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <stdexcept>
#include <flatfile.h>
#include <logging.h>
#include <tinyformat.h>
#include <util/system.h>
FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size) :
m_dir(std::move(dir)),
m_prefix(prefix),
m_chunk_size(chunk_size)
{
if (chunk_size == 0) {
throw std::invalid_argument("chunk_size must be positive");
}
}
std::string FlatFilePos::ToString() const
{
return strprintf("FlatFilePos(nFile=%i, nPos=%i)", nFile, nPos);
}
fs::path FlatFileSeq::FileName(const FlatFilePos& pos) const
{
return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile);
}
FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only)
{
if (pos.IsNull()) {
return nullptr;
}
fs::path path = FileName(pos);
fs::create_directories(path.parent_path());
FILE* file = fsbridge::fopen(path, read_only ? "rb": "rb+");
if (!file && !read_only)
file = fsbridge::fopen(path, "wb+");
if (!file) {
LogPrintf("Unable to open file %s\n", path.string());
return nullptr;
}
if (pos.nPos && fseek(file, pos.nPos, SEEK_SET)) {
LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string());
fclose(file);
return nullptr;
}
return file;
}
size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space)
{
out_of_space = false;
unsigned int n_old_chunks = (pos.nPos + m_chunk_size - 1) / m_chunk_size;
unsigned int n_new_chunks = (pos.nPos + add_size + m_chunk_size - 1) / m_chunk_size;
if (n_new_chunks > n_old_chunks) {
size_t old_size = pos.nPos;
size_t new_size = n_new_chunks * m_chunk_size;
size_t inc_size = new_size - old_size;
if (CheckDiskSpace(m_dir, inc_size)) {
FILE *file = Open(pos);
if (file) {
LogPrintf("Pre-allocating up to position 0x%x in %s%05u.dat\n", new_size, m_prefix, pos.nFile);
AllocateFileRange(file, pos.nPos, inc_size);
fclose(file);
return inc_size;
}
} else {
out_of_space = true;
}
}
return 0;
}
bool FlatFileSeq::Flush(const FlatFilePos& pos, bool finalize)
{
FILE* file = Open(FlatFilePos(pos.nFile, 0)); // Avoid fseek to nPos
if (!file) {
return error("%s: failed to open file %d", __func__, pos.nFile);
}
if (finalize && !TruncateFile(file, pos.nPos)) {
fclose(file);
return error("%s: failed to truncate file %d", __func__, pos.nFile);
}
if (!FileCommit(file)) {
fclose(file);
return error("%s: failed to commit file %d", __func__, pos.nFile);
}
fclose(file);
return true;
}

93
src/flatfile.h Normal file
View File

@ -0,0 +1,93 @@
// Copyright (c) 2009-2010 Satoshi Nakamoto
// Copyright (c) 2009-2019 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef BITCOIN_FLATFILE_H
#define BITCOIN_FLATFILE_H
#include <string>
#include <fs.h>
#include <serialize.h>
struct FlatFilePos
{
int nFile;
unsigned int nPos;
SERIALIZE_METHODS(FlatFilePos, obj)
{
READWRITE(VARINT(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED), VARINT(obj.nPos));
}
FlatFilePos() : nFile(-1), nPos(0) {}
FlatFilePos(int nFileIn, unsigned int nPosIn) :
nFile(nFileIn),
nPos(nPosIn)
{}
friend bool operator==(const FlatFilePos &a, const FlatFilePos &b) {
return (a.nFile == b.nFile && a.nPos == b.nPos);
}
friend bool operator!=(const FlatFilePos &a, const FlatFilePos &b) {
return !(a == b);
}
void SetNull() { nFile = -1; nPos = 0; }
bool IsNull() const { return (nFile == -1); }
std::string ToString() const;
};
/**
* FlatFileSeq represents a sequence of numbered files storing raw data. This class facilitates
* access to and efficient management of these files.
*/
class FlatFileSeq
{
private:
const fs::path m_dir;
const char* const m_prefix;
const size_t m_chunk_size;
public:
/**
* Constructor
*
* @param dir The base directory that all files live in.
* @param prefix A short prefix given to all file names.
* @param chunk_size Disk space is pre-allocated in multiples of this amount.
*/
FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size);
/** Get the name of the file at the given position. */
fs::path FileName(const FlatFilePos& pos) const;
/** Open a handle to the file at the given position. */
FILE* Open(const FlatFilePos& pos, bool read_only = false);
/**
* Allocate additional space in a file after the given starting position. The amount allocated
* will be the minimum multiple of the sequence chunk size greater than add_size.
*
* @param[in] pos The starting position that bytes will be allocated after.
* @param[in] add_size The minimum number of bytes to be allocated.
* @param[out] out_of_space Whether the allocation failed due to insufficient disk space.
* @return The number of bytes successfully allocated.
*/
size_t Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space);
/**
* Commit a file to disk, and optionally truncate off extra pre-allocated bytes if final.
*
* @param[in] pos The first unwritten position in the file to be flushed.
* @param[in] finalize True if no more data will be written to this file.
* @return true on success, false on failure.
*/
bool Flush(const FlatFilePos& pos, bool finalize = false);
};
#endif // BITCOIN_FLATFILE_H

8
src/index/txindex.cpp
View File

@ -16,17 +16,17 @@ constexpr char DB_TXINDEX_BLOCK = 'T';
std::unique_ptr<TxIndex> g_txindex;
struct CDiskTxPos : public CDiskBlockPos
struct CDiskTxPos : public FlatFilePos
{
unsigned int nTxOffset; // after header
SERIALIZE_METHODS(CDiskTxPos, obj)
{
READWRITEAS(CDiskBlockPos, obj);
READWRITEAS(FlatFilePos, obj);
READWRITE(VARINT(obj.nTxOffset));
}
CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
}
CDiskTxPos() {
@ -34,7 +34,7 @@ struct CDiskTxPos : public CDiskBlockPos
}
void SetNull() {
CDiskBlockPos::SetNull();
FlatFilePos::SetNull();
nTxOffset = 0;
}
};

8
src/init.cpp
View File

@ -814,8 +814,8 @@ static void ThreadImport(std::vector<fs::path> vImportFiles)
if (fReindex) {
int nFile = 0;
while (true) {
CDiskBlockPos pos(nFile, 0);
if (!fs::exists(GetBlockPosFilename(pos, "blk")))
FlatFilePos pos(nFile, 0);
if (!fs::exists(GetBlockPosFilename(pos)))
break; // No block files left to reindex
FILE *file = OpenBlockFile(pos, true);
if (!file)
@ -2405,11 +2405,11 @@ bool AppInitMain()
// ********************************************************* Step 11: import blocks
if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ false)) {
if (!CheckDiskSpace(GetDataDir())) {
InitError(strprintf(_("Error: Disk space is low for %s"), GetDataDir()));
return false;
}
if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ true)) {
if (!CheckDiskSpace(GetBlocksDir())) {
InitError(strprintf(_("Error: Disk space is low for %s"), GetBlocksDir()));
return false;
}

123
src/test/flatfile_tests.cpp Normal file
View File

@ -0,0 +1,123 @@
// Copyright (c) 2019 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <flatfile.h>
#include <test/test_dash.h>
#include <boost/test/unit_test.hpp>
BOOST_FIXTURE_TEST_SUITE(flatfile_tests, BasicTestingSetup)
BOOST_AUTO_TEST_CASE(flatfile_filename)
{
auto data_dir = SetDataDir("flatfile_test");
FlatFilePos pos(456, 789);
FlatFileSeq seq1(data_dir, "a", 16 * 1024);
BOOST_CHECK_EQUAL(seq1.FileName(pos), data_dir / "a00456.dat");
FlatFileSeq seq2(data_dir / "a", "b", 16 * 1024);
BOOST_CHECK_EQUAL(seq2.FileName(pos), data_dir / "a" / "b00456.dat");
}
BOOST_AUTO_TEST_CASE(flatfile_open)
{
auto data_dir = SetDataDir("flatfile_test");
FlatFileSeq seq(data_dir, "a", 16 * 1024);
std::string line1("A purely peer-to-peer version of electronic cash would allow online "
"payments to be sent directly from one party to another without going "
"through a financial institution.");
std::string line2("Digital signatures provide part of the solution, but the main benefits are "
"lost if a trusted third party is still required to prevent double-spending.");
size_t pos1 = 0;
size_t pos2 = pos1 + GetSerializeSize(line1, CLIENT_VERSION);
// Write first line to file.
{
CAutoFile file(seq.Open(FlatFilePos(0, pos1)), SER_DISK, CLIENT_VERSION);
file << LIMITED_STRING(line1, 256);
}
// Attempt to append to file opened in read-only mode.
{
CAutoFile file(seq.Open(FlatFilePos(0, pos2), true), SER_DISK, CLIENT_VERSION);
BOOST_CHECK_THROW(file << LIMITED_STRING(line2, 256), std::ios_base::failure);
}
// Append second line to file.
{
CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION);
file << LIMITED_STRING(line2, 256);
}
// Read text from file in read-only mode.
{
std::string text;
CAutoFile file(seq.Open(FlatFilePos(0, pos1), true), SER_DISK, CLIENT_VERSION);
file >> LIMITED_STRING(text, 256);
BOOST_CHECK_EQUAL(text, line1);
file >> LIMITED_STRING(text, 256);
BOOST_CHECK_EQUAL(text, line2);
}
// Read text from file with position offset.
{
std::string text;
CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION);
file >> LIMITED_STRING(text, 256);
BOOST_CHECK_EQUAL(text, line2);
}
// Ensure another file in the sequence has no data.
{
std::string text;
CAutoFile file(seq.Open(FlatFilePos(1, pos2)), SER_DISK, CLIENT_VERSION);
BOOST_CHECK_THROW(file >> LIMITED_STRING(text, 256), std::ios_base::failure);
}
}
BOOST_AUTO_TEST_CASE(flatfile_allocate)
{
auto data_dir = SetDataDir("flatfile_test");
FlatFileSeq seq(data_dir, "a", 100);
bool out_of_space;
BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 0), 1, out_of_space), 100);
BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 0))), 100);
BOOST_CHECK(!out_of_space);
BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 1, out_of_space), 0);
BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 100);
BOOST_CHECK(!out_of_space);
BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 2, out_of_space), 101);
BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 200);
BOOST_CHECK(!out_of_space);
}
BOOST_AUTO_TEST_CASE(flatfile_flush)
{
auto data_dir = SetDataDir("flatfile_test");
FlatFileSeq seq(data_dir, "a", 100);
bool out_of_space;
seq.Allocate(FlatFilePos(0, 0), 1, out_of_space);
// Flush without finalize should not truncate file.
seq.Flush(FlatFilePos(0, 1));
BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 100);
// Flush with finalize should truncate file.
seq.Flush(FlatFilePos(0, 1), true);
BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 1);
}
BOOST_AUTO_TEST_SUITE_END()

8
src/util/system.cpp
View File

@ -207,6 +207,14 @@ bool DirIsWritable(const fs::path& directory)
return true;
}
bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes)
{
constexpr uint64_t min_disk_space = 52428800; // 50 MiB
uint64_t free_bytes_available = fs::space(dir).available;
return free_bytes_available >= min_disk_space + additional_bytes;
}
/**
* Interpret a string argument as a boolean.
*

1
src/util/system.h
View File

@ -94,6 +94,7 @@ bool RenameOver(fs::path src, fs::path dest);
bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only=false);
void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name);
bool DirIsWritable(const fs::path& directory);
bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes = 0);
/** Release all directory locks. This is used for unit testing only, at runtime
* the global destructor will take care of the locks.

183
src/validation.cpp
View File

@ -16,6 +16,7 @@
#include <consensus/tx_verify.h>
#include <consensus/validation.h>
#include <cuckoocache.h>
#include <flatfile.h>
#include <hash.h>
#include <index/txindex.h>
#include <logging.h>
@ -192,7 +193,7 @@ public:
* that it doesn't descend from an invalid block, and then add it to mapBlockIndex.
*/
bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
// Block (dis)connection on a given view:
DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view);
@ -232,7 +233,7 @@ private:
void InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main);
void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
@ -340,7 +341,9 @@ static bool FlushStateToDisk(const CChainParams& chainParams, CValidationState &
static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight);
static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight);
bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr);
static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false);
static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false);
static FlatFileSeq BlockFileSeq();
static FlatFileSeq UndoFileSeq();
bool CheckFinalTx(const CTransaction &tx, int flags)
{
@ -1027,7 +1030,7 @@ bool GetTransaction(const uint256& hash, CTransactionRef& txOut, const Consensus
// CBlock and CBlockIndex
//
static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart)
static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart)
{
// Open history file to append
CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION);
@ -1048,7 +1051,7 @@ static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMes
return true;
}
bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams)
bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams)
{
block.SetNull();
@ -1074,7 +1077,7 @@ bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus:
bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams)
{
CDiskBlockPos blockPos;
FlatFilePos blockPos;
{
LOCK(cs_main);
blockPos = pindex->GetBlockPos();
@ -1547,7 +1550,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi
namespace {
bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart)
bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart)
{
// Open history file to append
CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION);
@ -1576,7 +1579,7 @@ bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint
static bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex *pindex)
{
CDiskBlockPos pos = pindex->GetUndoPos();
FlatFilePos pos = pindex->GetUndoPos();
if (pos.IsNull()) {
return error("%s: no undo data available", __func__);
}
@ -1838,37 +1841,24 @@ void static FlushBlockFile(bool fFinalize = false)
{
LOCK(cs_LastBlockFile);
CDiskBlockPos posOld(nLastBlockFile, 0);
FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize);
FlatFilePos undo_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nUndoSize);
bool status = true;
FILE *fileOld = OpenBlockFile(posOld);
if (fileOld) {
if (fFinalize)
status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nSize);
status &= FileCommit(fileOld);
fclose(fileOld);
}
fileOld = OpenUndoFile(posOld);
if (fileOld) {
if (fFinalize)
status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nUndoSize);
status &= FileCommit(fileOld);
fclose(fileOld);
}
status &= BlockFileSeq().Flush(block_pos_old, fFinalize);
status &= UndoFileSeq().Flush(undo_pos_old, fFinalize);
if (!status) {
AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error.");
}
}
static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize);
static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize);
static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams)
{
// Write undo information to disk
if (pindex->GetUndoPos().IsNull()) {
CDiskBlockPos _pos;
FlatFilePos _pos;
if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 40))
return error("ConnectBlock(): FindUndoPos failed");
if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart()))
@ -2516,8 +2506,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState &
// Write blocks and block index to disk.
if (fDoFullFlush || fPeriodicWrite) {
// Depend on nMinDiskSpace to ensure we can write block index
if (!CheckDiskSpace(0, true))
return state.Error("out of disk space");
if (!CheckDiskSpace(GetBlocksDir())) {
return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!"));
}
// First make sure all block and undo data is flushed to disk.
{
LOG_TIME_MILLIS("write block and undo data to disk", BCLog::BENCHMARK);
@ -2564,8 +2555,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState &
// twice (once in the log, and once in the tables). This is already
// an overestimation, as most will delete an existing entry or
// overwrite one. Still, use a conservative safety factor of 2.
if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize()))
return state.Error("out of disk space");
if (!CheckDiskSpace(GetDataDir(), 48 * 2 * 2 * pcoinsTip->GetCacheSize())) {
return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!"));
}
// Flush the chainstate (which may refer to block index entries).
if (!pcoinsTip->Flush())
return AbortNode(state, "Failed to write to coin database");
@ -3472,7 +3464,7 @@ CBlockIndex* CChainState::AddToBlockIndex(const CBlockHeader& block, enum BlockS
}
/** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */
void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos)
void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos)
{
pindexNew->nTx = block.vtx.size();
pindexNew->nChainTx = 0;
@ -3517,7 +3509,7 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi
}
}
static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false)
static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false)
{
LOCK(cs_LastBlockFile);
@ -3552,21 +3544,13 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int
vinfoBlockFile[nFile].nSize += nAddSize;
if (!fKnown) {
unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE;
unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE;
if (nNewChunks > nOldChunks) {
if (fPruneMode)
fCheckForPruning = true;
if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos, true)) {
FILE *file = OpenBlockFile(pos);
if (file) {
LogPrintf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile);
AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos);
fclose(file);
}
}
else
return error("out of disk space");
bool out_of_space;
size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space);
if (out_of_space) {
return AbortNode("Disk space is low!", _("Error: Disk space is low!"));
}
if (bytes_allocated != 0 && fPruneMode) {
fCheckForPruning = true;
}
}
@ -3574,32 +3558,23 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int
return true;
}
static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize)
static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize)
{
pos.nFile = nFile;
LOCK(cs_LastBlockFile);
unsigned int nNewSize;
pos.nPos = vinfoBlockFile[nFile].nUndoSize;
nNewSize = vinfoBlockFile[nFile].nUndoSize += nAddSize;
vinfoBlockFile[nFile].nUndoSize += nAddSize;
setDirtyFileInfo.insert(nFile);
unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE;
unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE;
if (nNewChunks > nOldChunks) {
if (fPruneMode)
fCheckForPruning = true;
if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos, true)) {
FILE *file = OpenUndoFile(pos);
if (file) {
LogPrintf("Pre-allocating up to position 0x%x in rev%05u.dat\n", nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile);
AllocateFileRange(file, pos.nPos, nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos);
fclose(file);
}
}
else
return state.Error("out of disk space");
bool out_of_space;
size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space);
if (out_of_space) {
return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!"));
}
if (bytes_allocated != 0 && fPruneMode) {
fCheckForPruning = true;
}
return true;
@ -3939,26 +3914,26 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidatio
}
/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */
static CDiskBlockPos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const CDiskBlockPos* dbp) {
static FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const FlatFilePos* dbp) {
unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION);
CDiskBlockPos blockPos;
FlatFilePos blockPos;
if (dbp != nullptr)
blockPos = *dbp;
if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != nullptr)) {
error("%s: FindBlockPos failed", __func__);
return CDiskBlockPos();
return FlatFilePos();
}
if (dbp == nullptr) {
if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) {
AbortNode("Failed to write block");
return CDiskBlockPos();
return FlatFilePos();
}
}
return blockPos;
}
/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */
bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock)
bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock)
{
boost::posix_time::ptime start = boost::posix_time::microsec_clock::local_time();
@ -4022,7 +3997,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CVali
// Write block to history file
if (fNewBlock) *fNewBlock = true;
try {
CDiskBlockPos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp);
FlatFilePos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp);
if (blockPos.IsNull()) {
state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__));
return false;
@ -4167,9 +4142,9 @@ void PruneOneBlockFile(const int fileNumber)
void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune)
{
for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
CDiskBlockPos pos(*it, 0);
fs::remove(GetBlockPosFilename(pos, "blk"));
fs::remove(GetBlockPosFilename(pos, "rev"));
FlatFilePos pos(*it, 0);
fs::remove(BlockFileSeq().FileName(pos));
fs::remove(UndoFileSeq().FileName(pos));
LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
}
}
@ -4277,52 +4252,28 @@ static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfte
nLastBlockWeCanPrune, count);
}
bool CheckDiskSpace(uint64_t nAdditionalBytes, bool blocks_dir)
static FlatFileSeq BlockFileSeq()
{
uint64_t nFreeBytesAvailable = fs::space(blocks_dir ? GetBlocksDir() : GetDataDir()).available;
// Check for nMinDiskSpace bytes (currently 50MB)
if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes)
return AbortNode("Disk space is low!", _("Error: Disk space is low!"));
return true;
return FlatFileSeq(GetBlocksDir(), "blk", BLOCKFILE_CHUNK_SIZE);
}
static FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly)
static FlatFileSeq UndoFileSeq()
{
if (pos.IsNull())
return nullptr;
fs::path path = GetBlockPosFilename(pos, prefix);
fs::create_directories(path.parent_path());
FILE* file = fsbridge::fopen(path, fReadOnly ? "rb": "rb+");
if (!file && !fReadOnly)
file = fsbridge::fopen(path, "wb+");
if (!file) {
LogPrintf("Unable to open file %s\n", path.string());
return nullptr;
}
if (pos.nPos) {
if (fseek(file, pos.nPos, SEEK_SET)) {
LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string());
fclose(file);
return nullptr;
}
}
return file;
return FlatFileSeq(GetBlocksDir(), "rev", UNDOFILE_CHUNK_SIZE);
}
FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) {
return OpenDiskFile(pos, "blk", fReadOnly);
FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) {
return BlockFileSeq().Open(pos, fReadOnly);
}
/** Open an undo file (rev?????.dat) */
static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) {
return OpenDiskFile(pos, "rev", fReadOnly);
static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly) {
return UndoFileSeq().Open(pos, fReadOnly);
}
fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix)
fs::path GetBlockPosFilename(const FlatFilePos &pos)
{
return GetBlocksDir() / strprintf("%s%05u.dat", prefix, pos.nFile);
return BlockFileSeq().FileName(pos);
}
CBlockIndex * CChainState::InsertBlockIndex(const uint256& hash)
@ -4435,7 +4386,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_RE
}
for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++)
{
CDiskBlockPos pos(*it, 0);
FlatFilePos pos(*it, 0);
if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) {
return false;
}
@ -4790,7 +4741,7 @@ bool LoadBlockIndex(const CChainParams& chainparams)
bool CChainState::AddGenesisBlock(const CChainParams& chainparams, const CBlock& block, CValidationState& state)
{
CDiskBlockPos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr);
FlatFilePos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr);
if (blockPos.IsNull())
return error("%s: writing genesis block to disk failed (%s)", __func__, FormatStateMessage(state));
CBlockIndex *pindex = AddToBlockIndex(block);
@ -4836,10 +4787,10 @@ bool LoadGenesisBlock(const CChainParams& chainparams)
return g_chainstate.LoadGenesisBlock(chainparams);
}
void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos* dbp)
void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp)
{
// Map of disk positions for blocks with unknown parent (only used for reindex)
static std::multimap<uint256, CDiskBlockPos> mapBlocksUnknownParent;
static std::multimap<uint256, FlatFilePos> mapBlocksUnknownParent;
int64_t nStart = GetTimeMillis();
int nLoaded = 0;
@ -4925,9 +4876,9 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB
while (!queue.empty()) {
uint256 head = queue.front();
queue.pop_front();
std::pair<std::multimap<uint256, CDiskBlockPos>::iterator, std::multimap<uint256, CDiskBlockPos>::iterator> range = mapBlocksUnknownParent.equal_range(head);
std::pair<std::multimap<uint256, FlatFilePos>::iterator, std::multimap<uint256, FlatFilePos>::iterator> range = mapBlocksUnknownParent.equal_range(head);
while (range.first != range.second) {
std::multimap<uint256, CDiskBlockPos>::iterator it = range.first;
std::multimap<uint256, FlatFilePos>::iterator it = range.first;
std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>();
if (ReadBlockFromDisk(*pblockrecursive, it->second, chainparams.GetConsensus()))
{

13
src/validation.h
View File

@ -167,9 +167,6 @@ extern arith_uint256 nMinimumChainWork;
/** Best header we've seen so far (used for getheaders queries' starting points). */
extern CBlockIndex *pindexBestHeader;
/** Minimum disk space required - used in CheckDiskSpace() */
static const uint64_t nMinDiskSpace = 52428800;
/** Pruning-related variables and constants */
/** True if any block files have ever been pruned. */
extern bool fHavePruned;
@ -214,14 +211,12 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons
*/
bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr, CBlockHeader* first_invalid = nullptr) LOCKS_EXCLUDED(cs_main);
/** Check whether enough disk space is available for an incoming block */
bool CheckDiskSpace(uint64_t nAdditionalBytes = 0, bool blocks_dir = false);
/** Open a block file (blk?????.dat) */
FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false);
FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false);
/** Translation to a filesystem path */
fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix);
fs::path GetBlockPosFilename(const FlatFilePos &pos);
/** Import blocks from an external file */
void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos* dbp = nullptr);
void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp = nullptr);
/** Ensures we have a genesis block in the block tree, possibly writing one to disk. */
bool LoadGenesisBlock(const CChainParams& chainparams);
/** Load the block tree and coins database from disk,
@ -374,7 +369,7 @@ void InitScriptExecutionCache();
/** Functions for disk access for blocks */
bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams);
bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams);
bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams);
/** Functions for validating blocks and updating the block tree */